001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.search.builder;
021
022import ca.uhn.fhir.context.ComboSearchParamType;
023import ca.uhn.fhir.context.FhirContext;
024import ca.uhn.fhir.context.FhirVersionEnum;
025import ca.uhn.fhir.context.RuntimeResourceDefinition;
026import ca.uhn.fhir.context.RuntimeSearchParam;
027import ca.uhn.fhir.i18n.Msg;
028import ca.uhn.fhir.interceptor.api.HookParams;
029import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
030import ca.uhn.fhir.interceptor.api.Pointcut;
031import ca.uhn.fhir.interceptor.model.RequestPartitionId;
032import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
033import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
034import ca.uhn.fhir.jpa.api.dao.IDao;
035import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
036import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
037import ca.uhn.fhir.jpa.config.HapiFhirLocalContainerEntityManagerFactoryBean;
038import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
039import ca.uhn.fhir.jpa.dao.BaseStorageDao;
040import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
041import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
042import ca.uhn.fhir.jpa.dao.IResultIterator;
043import ca.uhn.fhir.jpa.dao.ISearchBuilder;
044import ca.uhn.fhir.jpa.dao.data.IResourceSearchViewDao;
045import ca.uhn.fhir.jpa.dao.data.IResourceTagDao;
046import ca.uhn.fhir.jpa.dao.search.ResourceNotFoundInIndexException;
047import ca.uhn.fhir.jpa.entity.ResourceSearchView;
048import ca.uhn.fhir.jpa.interceptor.JpaPreResourceAccessDetails;
049import ca.uhn.fhir.jpa.model.config.PartitionSettings;
050import ca.uhn.fhir.jpa.model.dao.JpaPid;
051import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
052import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
053import ca.uhn.fhir.jpa.model.entity.ResourceTag;
054import ca.uhn.fhir.jpa.model.search.SearchBuilderLoadIncludesParameters;
055import ca.uhn.fhir.jpa.model.search.SearchRuntimeDetails;
056import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
057import ca.uhn.fhir.jpa.search.SearchConstants;
058import ca.uhn.fhir.jpa.search.builder.models.ResolvedSearchQueryExecutor;
059import ca.uhn.fhir.jpa.search.builder.sql.GeneratedSql;
060import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryBuilder;
061import ca.uhn.fhir.jpa.search.builder.sql.SearchQueryExecutor;
062import ca.uhn.fhir.jpa.search.builder.sql.SqlObjectFactory;
063import ca.uhn.fhir.jpa.search.lastn.IElasticsearchSvc;
064import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
065import ca.uhn.fhir.jpa.searchparam.util.Dstu3DistanceHelper;
066import ca.uhn.fhir.jpa.searchparam.util.JpaParamUtil;
067import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
068import ca.uhn.fhir.jpa.util.BaseIterator;
069import ca.uhn.fhir.jpa.util.CurrentThreadCaptureQueriesListener;
070import ca.uhn.fhir.jpa.util.QueryChunker;
071import ca.uhn.fhir.jpa.util.SqlQueryList;
072import ca.uhn.fhir.model.api.IQueryParameterType;
073import ca.uhn.fhir.model.api.Include;
074import ca.uhn.fhir.model.api.ResourceMetadataKeyEnum;
075import ca.uhn.fhir.model.valueset.BundleEntrySearchModeEnum;
076import ca.uhn.fhir.rest.api.Constants;
077import ca.uhn.fhir.rest.api.RestSearchParameterTypeEnum;
078import ca.uhn.fhir.rest.api.SearchContainedModeEnum;
079import ca.uhn.fhir.rest.api.SortOrderEnum;
080import ca.uhn.fhir.rest.api.SortSpec;
081import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
082import ca.uhn.fhir.rest.api.server.RequestDetails;
083import ca.uhn.fhir.rest.param.DateRangeParam;
084import ca.uhn.fhir.rest.param.ParameterUtil;
085import ca.uhn.fhir.rest.param.ReferenceParam;
086import ca.uhn.fhir.rest.param.StringParam;
087import ca.uhn.fhir.rest.param.TokenParam;
088import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
089import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
090import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
091import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
092import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
093import ca.uhn.fhir.util.StopWatch;
094import ca.uhn.fhir.util.StringUtil;
095import ca.uhn.fhir.util.UrlUtil;
096import com.google.common.collect.Streams;
097import com.healthmarketscience.sqlbuilder.Condition;
098import jakarta.annotation.Nonnull;
099import jakarta.annotation.Nullable;
100import jakarta.persistence.EntityManager;
101import jakarta.persistence.PersistenceContext;
102import jakarta.persistence.PersistenceContextType;
103import jakarta.persistence.Query;
104import jakarta.persistence.Tuple;
105import jakarta.persistence.TypedQuery;
106import jakarta.persistence.criteria.CriteriaBuilder;
107import org.apache.commons.lang3.StringUtils;
108import org.apache.commons.lang3.Validate;
109import org.apache.commons.lang3.math.NumberUtils;
110import org.apache.commons.lang3.tuple.Pair;
111import org.hl7.fhir.instance.model.api.IAnyResource;
112import org.hl7.fhir.instance.model.api.IBaseResource;
113import org.slf4j.Logger;
114import org.slf4j.LoggerFactory;
115import org.springframework.beans.factory.annotation.Autowired;
116import org.springframework.jdbc.core.JdbcTemplate;
117import org.springframework.jdbc.core.SingleColumnRowMapper;
118import org.springframework.transaction.support.TransactionSynchronizationManager;
119
120import java.util.ArrayList;
121import java.util.Collection;
122import java.util.Collections;
123import java.util.HashMap;
124import java.util.HashSet;
125import java.util.Iterator;
126import java.util.List;
127import java.util.Map;
128import java.util.Objects;
129import java.util.Set;
130import java.util.stream.Collectors;
131
132import static ca.uhn.fhir.jpa.model.util.JpaConstants.UNDESIRED_RESOURCE_LINKAGES_FOR_EVERYTHING_ON_PATIENT_INSTANCE;
133import static ca.uhn.fhir.jpa.search.builder.QueryStack.LOCATION_POSITION;
134import static ca.uhn.fhir.jpa.search.builder.QueryStack.SearchForIdsParams.with;
135import static org.apache.commons.lang3.StringUtils.defaultString;
136import static org.apache.commons.lang3.StringUtils.isBlank;
137import static org.apache.commons.lang3.StringUtils.isNotBlank;
138
139/**
140 * The SearchBuilder is responsible for actually forming the SQL query that handles
141 * searches for resources
142 */
143public class SearchBuilder implements ISearchBuilder<JpaPid> {
144
145        /**
146         * See loadResourcesByPid
147         * for an explanation of why we use the constant 800
148         */
149        // NB: keep public
150        @Deprecated
151        public static final int MAXIMUM_PAGE_SIZE = SearchConstants.MAX_PAGE_SIZE;
152
153        public static final int MAXIMUM_PAGE_SIZE_FOR_TESTING = 50;
154        public static final String RESOURCE_ID_ALIAS = "resource_id";
155        public static final String RESOURCE_VERSION_ALIAS = "resource_version";
156        private static final Logger ourLog = LoggerFactory.getLogger(SearchBuilder.class);
157        private static final JpaPid NO_MORE = JpaPid.fromId(-1L);
158        private static final String MY_TARGET_RESOURCE_PID = "myTargetResourcePid";
159        private static final String MY_SOURCE_RESOURCE_PID = "mySourceResourcePid";
160        private static final String MY_TARGET_RESOURCE_TYPE = "myTargetResourceType";
161        private static final String MY_SOURCE_RESOURCE_TYPE = "mySourceResourceType";
162        private static final String MY_TARGET_RESOURCE_VERSION = "myTargetResourceVersion";
163        public static boolean myUseMaxPageSize50ForTest = false;
164        protected final IInterceptorBroadcaster myInterceptorBroadcaster;
165        protected final IResourceTagDao myResourceTagDao;
166        private final String myResourceName;
167        private final Class<? extends IBaseResource> myResourceType;
168        private final HapiFhirLocalContainerEntityManagerFactoryBean myEntityManagerFactory;
169        private final SqlObjectFactory mySqlBuilderFactory;
170        private final HibernatePropertiesProvider myDialectProvider;
171        private final ISearchParamRegistry mySearchParamRegistry;
172        private final PartitionSettings myPartitionSettings;
173        private final DaoRegistry myDaoRegistry;
174        private final IResourceSearchViewDao myResourceSearchViewDao;
175        private final FhirContext myContext;
176        private final IIdHelperService<JpaPid> myIdHelperService;
177        private final JpaStorageSettings myStorageSettings;
178        private final IDao myCallingDao;
179
180        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
181        protected EntityManager myEntityManager;
182
183        private CriteriaBuilder myCriteriaBuilder;
184        private SearchParameterMap myParams;
185        private String mySearchUuid;
186        private int myFetchSize;
187        private Integer myMaxResultsToFetch;
188        private Set<JpaPid> myPidSet;
189        private boolean myHasNextIteratorQuery = false;
190        private RequestPartitionId myRequestPartitionId;
191
192        @Autowired(required = false)
193        private IFulltextSearchSvc myFulltextSearchSvc;
194
195        @Autowired(required = false)
196        private IElasticsearchSvc myIElasticsearchSvc;
197
198        @Autowired
199        private IJpaStorageResourceParser myJpaStorageResourceParser;
200
201        /**
202         * Constructor
203         */
204        public SearchBuilder(
205                        IDao theDao,
206                        String theResourceName,
207                        JpaStorageSettings theStorageSettings,
208                        HapiFhirLocalContainerEntityManagerFactoryBean theEntityManagerFactory,
209                        SqlObjectFactory theSqlBuilderFactory,
210                        HibernatePropertiesProvider theDialectProvider,
211                        ISearchParamRegistry theSearchParamRegistry,
212                        PartitionSettings thePartitionSettings,
213                        IInterceptorBroadcaster theInterceptorBroadcaster,
214                        IResourceTagDao theResourceTagDao,
215                        DaoRegistry theDaoRegistry,
216                        IResourceSearchViewDao theResourceSearchViewDao,
217                        FhirContext theContext,
218                        IIdHelperService theIdHelperService,
219                        Class<? extends IBaseResource> theResourceType) {
220                myCallingDao = theDao;
221                myResourceName = theResourceName;
222                myResourceType = theResourceType;
223                myStorageSettings = theStorageSettings;
224
225                myEntityManagerFactory = theEntityManagerFactory;
226                mySqlBuilderFactory = theSqlBuilderFactory;
227                myDialectProvider = theDialectProvider;
228                mySearchParamRegistry = theSearchParamRegistry;
229                myPartitionSettings = thePartitionSettings;
230                myInterceptorBroadcaster = theInterceptorBroadcaster;
231                myResourceTagDao = theResourceTagDao;
232                myDaoRegistry = theDaoRegistry;
233                myResourceSearchViewDao = theResourceSearchViewDao;
234                myContext = theContext;
235                myIdHelperService = theIdHelperService;
236        }
237
238        @Override
239        public void setMaxResultsToFetch(Integer theMaxResultsToFetch) {
240                myMaxResultsToFetch = theMaxResultsToFetch;
241        }
242
243        private void searchForIdsWithAndOr(
244                        SearchQueryBuilder theSearchSqlBuilder,
245                        QueryStack theQueryStack,
246                        @Nonnull SearchParameterMap theParams,
247                        RequestDetails theRequest) {
248                myParams = theParams;
249
250                // Remove any empty parameters
251                theParams.clean();
252
253                // For DSTU3, pull out near-distance first so when it comes time to evaluate near, we already know the distance
254                if (myContext.getVersion().getVersion() == FhirVersionEnum.DSTU3) {
255                        Dstu3DistanceHelper.setNearDistance(myResourceType, theParams);
256                }
257
258                // Attempt to lookup via composite unique key.
259                if (isCompositeUniqueSpCandidate()) {
260                        attemptComboUniqueSpProcessing(theQueryStack, theParams, theRequest);
261                }
262
263                SearchContainedModeEnum searchContainedMode = theParams.getSearchContainedMode();
264
265                // Handle _id and _tag last, since they can typically be tacked onto a different parameter
266                List<String> paramNames = myParams.keySet().stream()
267                                .filter(t -> !t.equals(IAnyResource.SP_RES_ID))
268                                .filter(t -> !t.equals(Constants.PARAM_TAG))
269                                .collect(Collectors.toList());
270                if (myParams.containsKey(IAnyResource.SP_RES_ID)) {
271                        paramNames.add(IAnyResource.SP_RES_ID);
272                }
273                if (myParams.containsKey(Constants.PARAM_TAG)) {
274                        paramNames.add(Constants.PARAM_TAG);
275                }
276
277                // Handle each parameter
278                for (String nextParamName : paramNames) {
279                        if (myParams.isLastN() && LastNParameterHelper.isLastNParameter(nextParamName, myContext)) {
280                                // Skip parameters for Subject, Patient, Code and Category for LastN as these will be filtered by
281                                // Elasticsearch
282                                continue;
283                        }
284                        List<List<IQueryParameterType>> andOrParams = myParams.get(nextParamName);
285                        Condition predicate = theQueryStack.searchForIdsWithAndOr(with().setResourceName(myResourceName)
286                                        .setParamName(nextParamName)
287                                        .setAndOrParams(andOrParams)
288                                        .setRequest(theRequest)
289                                        .setRequestPartitionId(myRequestPartitionId));
290                        if (predicate != null) {
291                                theSearchSqlBuilder.addPredicate(predicate);
292                        }
293                }
294        }
295
296        /**
297         * A search is a candidate for Composite Unique SP if unique indexes are enabled, there is no EverythingMode, and the
298         * parameters all have no modifiers.
299         */
300        private boolean isCompositeUniqueSpCandidate() {
301                return myStorageSettings.isUniqueIndexesEnabled()
302                                && myParams.getEverythingMode() == null
303                                && myParams.isAllParametersHaveNoModifier();
304        }
305
306        @SuppressWarnings("ConstantConditions")
307        @Override
308        public Long createCountQuery(
309                        SearchParameterMap theParams,
310                        String theSearchUuid,
311                        RequestDetails theRequest,
312                        @Nonnull RequestPartitionId theRequestPartitionId) {
313
314                assert theRequestPartitionId != null;
315                assert TransactionSynchronizationManager.isActualTransactionActive();
316
317                init(theParams, theSearchUuid, theRequestPartitionId);
318
319                if (checkUseHibernateSearch()) {
320                        long count = myFulltextSearchSvc.count(myResourceName, theParams.clone());
321                        return count;
322                }
323
324                List<ISearchQueryExecutor> queries = createQuery(theParams.clone(), null, null, null, true, theRequest, null);
325                if (queries.isEmpty()) {
326                        return 0L;
327                } else {
328                        return queries.get(0).next();
329                }
330        }
331
332        /**
333         * @param thePidSet May be null
334         */
335        @Override
336        public void setPreviouslyAddedResourcePids(@Nonnull List<JpaPid> thePidSet) {
337                myPidSet = new HashSet<>(thePidSet);
338        }
339
340        @SuppressWarnings("ConstantConditions")
341        @Override
342        public IResultIterator<JpaPid> createQuery(
343                        SearchParameterMap theParams,
344                        SearchRuntimeDetails theSearchRuntimeDetails,
345                        RequestDetails theRequest,
346                        @Nonnull RequestPartitionId theRequestPartitionId) {
347                assert theRequestPartitionId != null;
348                assert TransactionSynchronizationManager.isActualTransactionActive();
349
350                init(theParams, theSearchRuntimeDetails.getSearchUuid(), theRequestPartitionId);
351
352                if (myPidSet == null) {
353                        myPidSet = new HashSet<>();
354                }
355
356                return new QueryIterator(theSearchRuntimeDetails, theRequest);
357        }
358
359        private void init(SearchParameterMap theParams, String theSearchUuid, RequestPartitionId theRequestPartitionId) {
360                myCriteriaBuilder = myEntityManager.getCriteriaBuilder();
361                // we mutate the params.  Make a private copy.
362                myParams = theParams.clone();
363                mySearchUuid = theSearchUuid;
364                myRequestPartitionId = theRequestPartitionId;
365        }
366
367        private List<ISearchQueryExecutor> createQuery(
368                        SearchParameterMap theParams,
369                        SortSpec sort,
370                        Integer theOffset,
371                        Integer theMaximumResults,
372                        boolean theCountOnlyFlag,
373                        RequestDetails theRequest,
374                        SearchRuntimeDetails theSearchRuntimeDetails) {
375
376                ArrayList<ISearchQueryExecutor> queries = new ArrayList<>();
377
378                if (checkUseHibernateSearch()) {
379                        // we're going to run at least part of the search against the Fulltext service.
380
381                        // Ugh - we have two different return types for now
382                        ISearchQueryExecutor fulltextExecutor = null;
383                        List<JpaPid> fulltextMatchIds = null;
384                        int resultCount = 0;
385                        if (myParams.isLastN()) {
386                                fulltextMatchIds = executeLastNAgainstIndex(theMaximumResults);
387                                resultCount = fulltextMatchIds.size();
388                        } else if (myParams.getEverythingMode() != null) {
389                                fulltextMatchIds = queryHibernateSearchForEverythingPids(theRequest);
390                                resultCount = fulltextMatchIds.size();
391                        } else {
392                                fulltextExecutor = myFulltextSearchSvc.searchNotScrolled(
393                                                myResourceName, myParams, myMaxResultsToFetch, theRequest);
394                        }
395
396                        if (fulltextExecutor == null) {
397                                fulltextExecutor = SearchQueryExecutors.from(fulltextMatchIds);
398                        }
399
400                        if (theSearchRuntimeDetails != null) {
401                                theSearchRuntimeDetails.setFoundIndexMatchesCount(resultCount);
402                                HookParams params = new HookParams()
403                                                .add(RequestDetails.class, theRequest)
404                                                .addIfMatchesType(ServletRequestDetails.class, theRequest)
405                                                .add(SearchRuntimeDetails.class, theSearchRuntimeDetails);
406                                CompositeInterceptorBroadcaster.doCallHooks(
407                                                myInterceptorBroadcaster,
408                                                theRequest,
409                                                Pointcut.JPA_PERFTRACE_INDEXSEARCH_QUERY_COMPLETE,
410                                                params);
411                        }
412
413                        // can we skip the database entirely and return the pid list from here?
414                        boolean canSkipDatabase =
415                                        // if we processed an AND clause, and it returned nothing, then nothing can match.
416                                        !fulltextExecutor.hasNext()
417                                                        ||
418                                                        // Our hibernate search query doesn't respect partitions yet
419                                                        (!myPartitionSettings.isPartitioningEnabled()
420                                                                        &&
421                                                                        // were there AND terms left?  Then we still need the db.
422                                                                        theParams.isEmpty()
423                                                                        &&
424                                                                        // not every param is a param. :-(
425                                                                        theParams.getNearDistanceParam() == null
426                                                                        &&
427                                                                        // todo MB don't we support _lastUpdated and _offset now?
428                                                                        theParams.getLastUpdated() == null
429                                                                        && theParams.getEverythingMode() == null
430                                                                        && theParams.getOffset() == null);
431
432                        if (canSkipDatabase) {
433                                ourLog.trace("Query finished after HSearch.  Skip db query phase");
434                                if (theMaximumResults != null) {
435                                        fulltextExecutor = SearchQueryExecutors.limited(fulltextExecutor, theMaximumResults);
436                                }
437                                queries.add(fulltextExecutor);
438                        } else {
439                                ourLog.trace("Query needs db after HSearch.  Chunking.");
440                                // Finish the query in the database for the rest of the search parameters, sorting, partitioning, etc.
441                                // We break the pids into chunks that fit in the 1k limit for jdbc bind params.
442                                new QueryChunker<Long>()
443                                                .chunk(
444                                                                Streams.stream(fulltextExecutor).collect(Collectors.toList()),
445                                                                t -> doCreateChunkedQueries(
446                                                                                theParams, t, theOffset, sort, theCountOnlyFlag, theRequest, queries));
447                        }
448                } else {
449                        // do everything in the database.
450                        createChunkedQuery(
451                                        theParams, sort, theOffset, theMaximumResults, theCountOnlyFlag, theRequest, null, queries);
452                }
453
454                return queries;
455        }
456
457        /**
458         * Check to see if query should use Hibernate Search, and error if the query can't continue.
459         *
460         * @return true if the query should first be processed by Hibernate Search
461         * @throws InvalidRequestException if fulltext search is not enabled but the query requires it - _content or _text
462         */
463        private boolean checkUseHibernateSearch() {
464                boolean fulltextEnabled = (myFulltextSearchSvc != null) && !myFulltextSearchSvc.isDisabled();
465
466                if (!fulltextEnabled) {
467                        failIfUsed(Constants.PARAM_TEXT);
468                        failIfUsed(Constants.PARAM_CONTENT);
469                }
470
471                // someday we'll want a query planner to figure out if we _should_ or _must_ use the ft index, not just if we
472                // can.
473                return fulltextEnabled
474                                && myParams != null
475                                && myParams.getSearchContainedMode() == SearchContainedModeEnum.FALSE
476                                && myFulltextSearchSvc.supportsSomeOf(myParams);
477        }
478
479        private void failIfUsed(String theParamName) {
480                if (myParams.containsKey(theParamName)) {
481                        throw new InvalidRequestException(Msg.code(1192)
482                                        + "Fulltext search is not enabled on this service, can not process parameter: " + theParamName);
483                }
484        }
485
486        private List<JpaPid> executeLastNAgainstIndex(Integer theMaximumResults) {
487                // Can we use our hibernate search generated index on resource to support lastN?:
488                if (myStorageSettings.isAdvancedHSearchIndexing()) {
489                        if (myFulltextSearchSvc == null) {
490                                throw new InvalidRequestException(Msg.code(2027)
491                                                + "LastN operation is not enabled on this service, can not process this request");
492                        }
493                        return myFulltextSearchSvc.lastN(myParams, theMaximumResults).stream()
494                                        .map(lastNResourceId -> myIdHelperService.resolveResourcePersistentIds(
495                                                        myRequestPartitionId, myResourceName, String.valueOf(lastNResourceId)))
496                                        .collect(Collectors.toList());
497                } else {
498                        throw new InvalidRequestException(
499                                        Msg.code(2033) + "LastN operation is not enabled on this service, can not process this request");
500                }
501        }
502
503        private List<JpaPid> queryHibernateSearchForEverythingPids(RequestDetails theRequestDetails) {
504                JpaPid pid = null;
505                if (myParams.get(IAnyResource.SP_RES_ID) != null) {
506                        String idParamValue;
507                        IQueryParameterType idParam =
508                                        myParams.get(IAnyResource.SP_RES_ID).get(0).get(0);
509                        if (idParam instanceof TokenParam) {
510                                TokenParam idParm = (TokenParam) idParam;
511                                idParamValue = idParm.getValue();
512                        } else {
513                                StringParam idParm = (StringParam) idParam;
514                                idParamValue = idParm.getValue();
515                        }
516
517                        pid = myIdHelperService.resolveResourcePersistentIds(myRequestPartitionId, myResourceName, idParamValue);
518                }
519                List<JpaPid> pids = myFulltextSearchSvc.everything(myResourceName, myParams, pid, theRequestDetails);
520                return pids;
521        }
522
523        private void doCreateChunkedQueries(
524                        SearchParameterMap theParams,
525                        List<Long> thePids,
526                        Integer theOffset,
527                        SortSpec sort,
528                        boolean theCount,
529                        RequestDetails theRequest,
530                        ArrayList<ISearchQueryExecutor> theQueries) {
531                if (thePids.size() < getMaximumPageSize()) {
532                        normalizeIdListForLastNInClause(thePids);
533                }
534                createChunkedQuery(theParams, sort, theOffset, thePids.size(), theCount, theRequest, thePids, theQueries);
535        }
536
537        /**
538         * Combs through the params for any _id parameters and extracts the PIDs for them
539         *
540         * @param theTargetPids
541         */
542        private void extractTargetPidsFromIdParams(
543                        HashSet<Long> theTargetPids, List<ISearchQueryExecutor> theSearchQueryExecutors) {
544                // get all the IQueryParameterType objects
545                // for _id -> these should all be StringParam values
546                HashSet<String> ids = new HashSet<>();
547                List<List<IQueryParameterType>> params = myParams.get(IAnyResource.SP_RES_ID);
548                for (List<IQueryParameterType> paramList : params) {
549                        for (IQueryParameterType param : paramList) {
550                                if (param instanceof StringParam) {
551                                        // we expect all _id values to be StringParams
552                                        ids.add(((StringParam) param).getValue());
553                                } else if (param instanceof TokenParam) {
554                                        ids.add(((TokenParam) param).getValue());
555                                } else {
556                                        // we do not expect the _id parameter to be a non-string value
557                                        throw new IllegalArgumentException(
558                                                        Msg.code(1193) + "_id parameter must be a StringParam or TokenParam");
559                                }
560                        }
561                }
562
563                // fetch our target Pids
564                // this will throw if an id is not found
565                Map<String, JpaPid> idToPid = myIdHelperService.resolveResourcePersistentIds(
566                                myRequestPartitionId, myResourceName, new ArrayList<>(ids));
567
568                // add the pids to targetPids
569                for (JpaPid pid : idToPid.values()) {
570                        theTargetPids.add(pid.getId());
571                }
572
573                // add the target pids to our executors as the first
574                // results iterator to go through
575                theSearchQueryExecutors.add(new ResolvedSearchQueryExecutor(new ArrayList<>(theTargetPids)));
576        }
577
578        private void createChunkedQuery(
579                        SearchParameterMap theParams,
580                        SortSpec sort,
581                        Integer theOffset,
582                        Integer theMaximumResults,
583                        boolean theCountOnlyFlag,
584                        RequestDetails theRequest,
585                        List<Long> thePidList,
586                        List<ISearchQueryExecutor> theSearchQueryExecutors) {
587                String sqlBuilderResourceName = myParams.getEverythingMode() == null ? myResourceName : null;
588                SearchQueryBuilder sqlBuilder = new SearchQueryBuilder(
589                                myContext,
590                                myStorageSettings,
591                                myPartitionSettings,
592                                myRequestPartitionId,
593                                sqlBuilderResourceName,
594                                mySqlBuilderFactory,
595                                myDialectProvider,
596                                theCountOnlyFlag);
597                QueryStack queryStack3 = new QueryStack(
598                                theParams, myStorageSettings, myContext, sqlBuilder, mySearchParamRegistry, myPartitionSettings);
599
600                if (theParams.keySet().size() > 1
601                                || theParams.getSort() != null
602                                || theParams.keySet().contains(Constants.PARAM_HAS)
603                                || isPotentiallyContainedReferenceParameterExistsAtRoot(theParams)) {
604                        List<RuntimeSearchParam> activeComboParams =
605                                        mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
606                        if (activeComboParams.isEmpty()) {
607                                sqlBuilder.setNeedResourceTableRoot(true);
608                        }
609                }
610
611                JdbcTemplate jdbcTemplate = new JdbcTemplate(myEntityManagerFactory.getDataSource());
612                jdbcTemplate.setFetchSize(myFetchSize);
613                if (theMaximumResults != null) {
614                        jdbcTemplate.setMaxRows(theMaximumResults);
615                }
616
617                if (myParams.getEverythingMode() != null) {
618                        HashSet<Long> targetPids = new HashSet<>();
619                        if (myParams.get(IAnyResource.SP_RES_ID) != null) {
620                                // will add an initial search executor for
621                                // _id params
622                                extractTargetPidsFromIdParams(targetPids, theSearchQueryExecutors);
623                        } else {
624                                // For Everything queries, we make the query root by the ResourceLink table, since this query
625                                // is basically a reverse-include search. For type/Everything (as opposed to instance/Everything)
626                                // the one problem with this approach is that it doesn't catch Patients that have absolutely
627                                // nothing linked to them. So we do one additional query to make sure we catch those too.
628                                SearchQueryBuilder fetchPidsSqlBuilder = new SearchQueryBuilder(
629                                                myContext,
630                                                myStorageSettings,
631                                                myPartitionSettings,
632                                                myRequestPartitionId,
633                                                myResourceName,
634                                                mySqlBuilderFactory,
635                                                myDialectProvider,
636                                                theCountOnlyFlag);
637                                GeneratedSql allTargetsSql = fetchPidsSqlBuilder.generate(theOffset, myMaxResultsToFetch);
638                                String sql = allTargetsSql.getSql();
639                                Object[] args = allTargetsSql.getBindVariables().toArray(new Object[0]);
640
641                                List<Long> output = jdbcTemplate.query(sql, args, new SingleColumnRowMapper<>(Long.class));
642
643                                // we add a search executor to fetch unlinked patients first
644                                theSearchQueryExecutors.add(new ResolvedSearchQueryExecutor(output));
645                        }
646
647                        List<String> typeSourceResources = new ArrayList<>();
648                        if (myParams.get(Constants.PARAM_TYPE) != null) {
649                                typeSourceResources.addAll(extractTypeSourceResourcesFromParams());
650                        }
651
652                        queryStack3.addPredicateEverythingOperation(
653                                        myResourceName, typeSourceResources, targetPids.toArray(new Long[0]));
654                } else {
655                        /*
656                         * If we're doing a filter, always use the resource table as the root - This avoids the possibility of
657                         * specific filters with ORs as their root from working around the natural resource type / deletion
658                         * status / partition IDs built into queries.
659                         */
660                        if (theParams.containsKey(Constants.PARAM_FILTER)) {
661                                Condition partitionIdPredicate = sqlBuilder
662                                                .getOrCreateResourceTablePredicateBuilder()
663                                                .createPartitionIdPredicate(myRequestPartitionId);
664                                if (partitionIdPredicate != null) {
665                                        sqlBuilder.addPredicate(partitionIdPredicate);
666                                }
667                        }
668
669                        // Normal search
670                        searchForIdsWithAndOr(sqlBuilder, queryStack3, myParams, theRequest);
671                }
672
673                // If we haven't added any predicates yet, we're doing a search for all resources. Make sure we add the
674                // partition ID predicate in that case.
675                if (!sqlBuilder.haveAtLeastOnePredicate()) {
676                        Condition partitionIdPredicate = sqlBuilder
677                                        .getOrCreateResourceTablePredicateBuilder()
678                                        .createPartitionIdPredicate(myRequestPartitionId);
679                        if (partitionIdPredicate != null) {
680                                sqlBuilder.addPredicate(partitionIdPredicate);
681                        }
682                }
683
684                // Add PID list predicate for full text search and/or lastn operation
685                if (thePidList != null && thePidList.size() > 0) {
686                        sqlBuilder.addResourceIdsPredicate(thePidList);
687                }
688
689                // Last updated
690                DateRangeParam lu = myParams.getLastUpdated();
691                if (lu != null && !lu.isEmpty()) {
692                        Condition lastUpdatedPredicates = sqlBuilder.addPredicateLastUpdated(lu);
693                        sqlBuilder.addPredicate(lastUpdatedPredicates);
694                }
695
696                /*
697                 * Exclude the pids already in the previous iterator. This is an optimization, as opposed
698                 * to something needed to guarantee correct results.
699                 *
700                 * Why do we need it? Suppose for example, a query like:
701                 *    Observation?category=foo,bar,baz
702                 * And suppose you have many resources that have all 3 of these category codes. In this case
703                 * the SQL query will probably return the same PIDs multiple times, and if this happens enough
704                 * we may exhaust the query results without getting enough distinct results back. When that
705                 * happens we re-run the query with a larger limit. Excluding results we already know about
706                 * tries to ensure that we get new unique results.
707                 *
708                 * The challenge with that though is that lots of DBs have an issue with too many
709                 * parameters in one query. So we only do this optimization if there aren't too
710                 * many results.
711                 */
712                if (myHasNextIteratorQuery) {
713                        if (myPidSet.size() + sqlBuilder.countBindVariables() < 900) {
714                                sqlBuilder.excludeResourceIdsPredicate(myPidSet);
715                        }
716                }
717
718                /*
719                 * If offset is present, we want deduplicate the results by using GROUP BY
720                 */
721                if (theOffset != null) {
722                        queryStack3.addGrouping();
723                        queryStack3.setUseAggregate(true);
724                }
725
726                /*
727                 * Sort
728                 *
729                 * If we have a sort, we wrap the criteria search (the search that actually
730                 * finds the appropriate resources) in an outer search which is then sorted
731                 */
732                if (sort != null) {
733                        assert !theCountOnlyFlag;
734
735                        createSort(queryStack3, sort, theParams);
736                }
737
738                /*
739                 * Now perform the search
740                 */
741                GeneratedSql generatedSql = sqlBuilder.generate(theOffset, myMaxResultsToFetch);
742                if (!generatedSql.isMatchNothing()) {
743                        SearchQueryExecutor executor =
744                                        mySqlBuilderFactory.newSearchQueryExecutor(generatedSql, myMaxResultsToFetch);
745                        theSearchQueryExecutors.add(executor);
746                }
747        }
748
749        private Collection<String> extractTypeSourceResourcesFromParams() {
750
751                List<List<IQueryParameterType>> listOfList = myParams.get(Constants.PARAM_TYPE);
752
753                // first off, let's flatten the list of list
754                List<IQueryParameterType> iQueryParameterTypesList =
755                                listOfList.stream().flatMap(List::stream).collect(Collectors.toList());
756
757                // then, extract all elements of each CSV into one big list
758                List<String> resourceTypes = iQueryParameterTypesList.stream()
759                                .map(param -> ((StringParam) param).getValue())
760                                .map(csvString -> List.of(csvString.split(",")))
761                                .flatMap(List::stream)
762                                .collect(Collectors.toList());
763
764                Set<String> knownResourceTypes = myContext.getResourceTypes();
765
766                // remove leading/trailing whitespaces if any and remove duplicates
767                Set<String> retVal = new HashSet<>();
768
769                for (String type : resourceTypes) {
770                        String trimmed = type.trim();
771                        if (!knownResourceTypes.contains(trimmed)) {
772                                throw new ResourceNotFoundException(
773                                                Msg.code(2197) + "Unknown resource type '" + trimmed + "' in _type parameter.");
774                        }
775                        retVal.add(trimmed);
776                }
777
778                return retVal;
779        }
780
781        private boolean isPotentiallyContainedReferenceParameterExistsAtRoot(SearchParameterMap theParams) {
782                return myStorageSettings.isIndexOnContainedResources()
783                                && theParams.values().stream()
784                                                .flatMap(Collection::stream)
785                                                .flatMap(Collection::stream)
786                                                .anyMatch(t -> t instanceof ReferenceParam);
787        }
788
789        private List<Long> normalizeIdListForLastNInClause(List<Long> lastnResourceIds) {
790                /*
791                The following is a workaround to a known issue involving Hibernate. If queries are used with "in" clauses with large and varying
792                numbers of parameters, this can overwhelm Hibernate's QueryPlanCache and deplete heap space. See the following link for more info:
793                https://stackoverflow.com/questions/31557076/spring-hibernate-query-plan-cache-memory-usage.
794
795                Normalizing the number of parameters in the "in" clause stabilizes the size of the QueryPlanCache, so long as the number of
796                arguments never exceeds the maximum specified below.
797                */
798                int listSize = lastnResourceIds.size();
799
800                if (listSize > 1 && listSize < 10) {
801                        padIdListWithPlaceholders(lastnResourceIds, 10);
802                } else if (listSize > 10 && listSize < 50) {
803                        padIdListWithPlaceholders(lastnResourceIds, 50);
804                } else if (listSize > 50 && listSize < 100) {
805                        padIdListWithPlaceholders(lastnResourceIds, 100);
806                } else if (listSize > 100 && listSize < 200) {
807                        padIdListWithPlaceholders(lastnResourceIds, 200);
808                } else if (listSize > 200 && listSize < 500) {
809                        padIdListWithPlaceholders(lastnResourceIds, 500);
810                } else if (listSize > 500 && listSize < 800) {
811                        padIdListWithPlaceholders(lastnResourceIds, 800);
812                }
813
814                return lastnResourceIds;
815        }
816
817        private void padIdListWithPlaceholders(List<Long> theIdList, int preferredListSize) {
818                while (theIdList.size() < preferredListSize) {
819                        theIdList.add(-1L);
820                }
821        }
822
823        private void createSort(QueryStack theQueryStack, SortSpec theSort, SearchParameterMap theParams) {
824                if (theSort == null || isBlank(theSort.getParamName())) {
825                        return;
826                }
827
828                boolean ascending = (theSort.getOrder() == null) || (theSort.getOrder() == SortOrderEnum.ASC);
829
830                if (IAnyResource.SP_RES_ID.equals(theSort.getParamName())) {
831
832                        theQueryStack.addSortOnResourceId(ascending);
833
834                } else if (Constants.PARAM_PID.equals(theSort.getParamName())) {
835
836                        theQueryStack.addSortOnResourcePID(ascending);
837
838                } else if (Constants.PARAM_LASTUPDATED.equals(theSort.getParamName())) {
839
840                        theQueryStack.addSortOnLastUpdated(ascending);
841
842                } else {
843
844                        RuntimeSearchParam param = null;
845
846                        if (param == null) {
847                                // do we have a composition param defined for the whole chain?
848                                param = mySearchParamRegistry.getActiveSearchParam(myResourceName, theSort.getParamName());
849                        }
850
851                        /*
852                         * If we have a sort like _sort=subject.name and we  have an
853                         * uplifted refchain for that combination we can do it more efficiently
854                         * by using the index associated with the uplifted refchain. In this case,
855                         * we need to find the actual target search parameter (corresponding
856                         * to "name" in this example) so that we know what datatype it is.
857                         */
858                        String paramName = theSort.getParamName();
859                        if (param == null && myStorageSettings.isIndexOnUpliftedRefchains()) {
860                                String[] chains = StringUtils.split(paramName, '.');
861                                if (chains.length == 2) {
862
863                                        // Given: Encounter?_sort=Patient:subject.name
864                                        String referenceParam = chains[0]; // subject
865                                        String referenceParamTargetType = null; // Patient
866                                        String targetParam = chains[1]; // name
867
868                                        int colonIdx = referenceParam.indexOf(':');
869                                        if (colonIdx > -1) {
870                                                referenceParamTargetType = referenceParam.substring(0, colonIdx);
871                                                referenceParam = referenceParam.substring(colonIdx + 1);
872                                        }
873                                        RuntimeSearchParam outerParam =
874                                                        mySearchParamRegistry.getActiveSearchParam(myResourceName, referenceParam);
875                                        if (outerParam == null) {
876                                                throwInvalidRequestExceptionForUnknownSortParameter(myResourceName, referenceParam);
877                                        }
878
879                                        if (outerParam.hasUpliftRefchain(targetParam)) {
880                                                for (String nextTargetType : outerParam.getTargets()) {
881                                                        if (referenceParamTargetType != null && !referenceParamTargetType.equals(nextTargetType)) {
882                                                                continue;
883                                                        }
884                                                        RuntimeSearchParam innerParam =
885                                                                        mySearchParamRegistry.getActiveSearchParam(nextTargetType, targetParam);
886                                                        if (innerParam != null) {
887                                                                param = innerParam;
888                                                                break;
889                                                        }
890                                                }
891                                        }
892                                }
893                        }
894
895                        int colonIdx = paramName.indexOf(':');
896                        String referenceTargetType = null;
897                        if (colonIdx > -1) {
898                                referenceTargetType = paramName.substring(0, colonIdx);
899                                paramName = paramName.substring(colonIdx + 1);
900                        }
901
902                        int dotIdx = paramName.indexOf('.');
903                        String chainName = null;
904                        if (param == null && dotIdx > -1) {
905                                chainName = paramName.substring(dotIdx + 1);
906                                paramName = paramName.substring(0, dotIdx);
907                                if (chainName.contains(".")) {
908                                        String msg = myContext
909                                                        .getLocalizer()
910                                                        .getMessageSanitized(
911                                                                        BaseStorageDao.class,
912                                                                        "invalidSortParameterTooManyChains",
913                                                                        paramName + "." + chainName);
914                                        throw new InvalidRequestException(Msg.code(2286) + msg);
915                                }
916                        }
917
918                        if (param == null) {
919                                param = mySearchParamRegistry.getActiveSearchParam(myResourceName, paramName);
920                        }
921
922                        if (param == null) {
923                                throwInvalidRequestExceptionForUnknownSortParameter(getResourceName(), paramName);
924                        }
925
926                        if (isNotBlank(chainName) && param.getParamType() != RestSearchParameterTypeEnum.REFERENCE) {
927                                throw new InvalidRequestException(
928                                                Msg.code(2285) + "Invalid chain, " + paramName + " is not a reference SearchParameter");
929                        }
930
931                        switch (param.getParamType()) {
932                                case STRING:
933                                        theQueryStack.addSortOnString(myResourceName, paramName, ascending);
934                                        break;
935                                case DATE:
936                                        theQueryStack.addSortOnDate(myResourceName, paramName, ascending);
937                                        break;
938                                case REFERENCE:
939                                        theQueryStack.addSortOnResourceLink(
940                                                        myResourceName, referenceTargetType, paramName, chainName, ascending, theParams);
941                                        break;
942                                case TOKEN:
943                                        theQueryStack.addSortOnToken(myResourceName, paramName, ascending);
944                                        break;
945                                case NUMBER:
946                                        theQueryStack.addSortOnNumber(myResourceName, paramName, ascending);
947                                        break;
948                                case URI:
949                                        theQueryStack.addSortOnUri(myResourceName, paramName, ascending);
950                                        break;
951                                case QUANTITY:
952                                        theQueryStack.addSortOnQuantity(myResourceName, paramName, ascending);
953                                        break;
954                                case COMPOSITE:
955                                        List<RuntimeSearchParam> compositeList =
956                                                        JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, param);
957                                        if (compositeList == null) {
958                                                throw new InvalidRequestException(Msg.code(1195) + "The composite _sort parameter " + paramName
959                                                                + " is not defined by the resource " + myResourceName);
960                                        }
961                                        if (compositeList.size() != 2) {
962                                                throw new InvalidRequestException(Msg.code(1196) + "The composite _sort parameter " + paramName
963                                                                + " must have 2 composite types declared in parameter annotation, found "
964                                                                + compositeList.size());
965                                        }
966                                        RuntimeSearchParam left = compositeList.get(0);
967                                        RuntimeSearchParam right = compositeList.get(1);
968
969                                        createCompositeSort(theQueryStack, left.getParamType(), left.getName(), ascending);
970                                        createCompositeSort(theQueryStack, right.getParamType(), right.getName(), ascending);
971
972                                        break;
973                                case SPECIAL:
974                                        if (LOCATION_POSITION.equals(param.getPath())) {
975                                                theQueryStack.addSortOnCoordsNear(paramName, ascending, theParams);
976                                                break;
977                                        }
978                                        throw new InvalidRequestException(
979                                                        Msg.code(2306) + "This server does not support _sort specifications of type "
980                                                                        + param.getParamType() + " - Can't serve _sort=" + paramName);
981
982                                case HAS:
983                                default:
984                                        throw new InvalidRequestException(
985                                                        Msg.code(1197) + "This server does not support _sort specifications of type "
986                                                                        + param.getParamType() + " - Can't serve _sort=" + paramName);
987                        }
988                }
989
990                // Recurse
991                createSort(theQueryStack, theSort.getChain(), theParams);
992        }
993
994        private void throwInvalidRequestExceptionForUnknownSortParameter(String theResourceName, String theParamName) {
995                Collection<String> validSearchParameterNames =
996                                mySearchParamRegistry.getValidSearchParameterNamesIncludingMeta(theResourceName);
997                String msg = myContext
998                                .getLocalizer()
999                                .getMessageSanitized(
1000                                                BaseStorageDao.class,
1001                                                "invalidSortParameter",
1002                                                theParamName,
1003                                                theResourceName,
1004                                                validSearchParameterNames);
1005                throw new InvalidRequestException(Msg.code(1194) + msg);
1006        }
1007
1008        private void createCompositeSort(
1009                        QueryStack theQueryStack,
1010                        RestSearchParameterTypeEnum theParamType,
1011                        String theParamName,
1012                        boolean theAscending) {
1013
1014                switch (theParamType) {
1015                        case STRING:
1016                                theQueryStack.addSortOnString(myResourceName, theParamName, theAscending);
1017                                break;
1018                        case DATE:
1019                                theQueryStack.addSortOnDate(myResourceName, theParamName, theAscending);
1020                                break;
1021                        case TOKEN:
1022                                theQueryStack.addSortOnToken(myResourceName, theParamName, theAscending);
1023                                break;
1024                        case QUANTITY:
1025                                theQueryStack.addSortOnQuantity(myResourceName, theParamName, theAscending);
1026                                break;
1027                        case NUMBER:
1028                        case REFERENCE:
1029                        case COMPOSITE:
1030                        case URI:
1031                        case HAS:
1032                        case SPECIAL:
1033                        default:
1034                                throw new InvalidRequestException(
1035                                                Msg.code(1198) + "Don't know how to handle composite parameter with type of " + theParamType
1036                                                                + " on _sort=" + theParamName);
1037                }
1038        }
1039
1040        private void doLoadPids(
1041                        Collection<JpaPid> thePids,
1042                        Collection<JpaPid> theIncludedPids,
1043                        List<IBaseResource> theResourceListToPopulate,
1044                        boolean theForHistoryOperation,
1045                        Map<JpaPid, Integer> thePosition) {
1046
1047                Map<Long, Long> resourcePidToVersion = null;
1048                for (JpaPid next : thePids) {
1049                        if (next.getVersion() != null && myStorageSettings.isRespectVersionsForSearchIncludes()) {
1050                                if (resourcePidToVersion == null) {
1051                                        resourcePidToVersion = new HashMap<>();
1052                                }
1053                                resourcePidToVersion.put((next).getId(), next.getVersion());
1054                        }
1055                }
1056
1057                List<Long> versionlessPids = JpaPid.toLongList(thePids);
1058                if (versionlessPids.size() < getMaximumPageSize()) {
1059                        versionlessPids = normalizeIdListForLastNInClause(versionlessPids);
1060                }
1061
1062                // -- get the resource from the searchView
1063                Collection<ResourceSearchView> resourceSearchViewList =
1064                                myResourceSearchViewDao.findByResourceIds(versionlessPids);
1065
1066                // -- preload all tags with tag definition if any
1067                Map<Long, Collection<ResourceTag>> tagMap = getResourceTagMap(resourceSearchViewList);
1068
1069                for (IBaseResourceEntity next : resourceSearchViewList) {
1070                        if (next.getDeleted() != null) {
1071                                continue;
1072                        }
1073
1074                        Class<? extends IBaseResource> resourceType =
1075                                        myContext.getResourceDefinition(next.getResourceType()).getImplementingClass();
1076
1077                        JpaPid resourceId = JpaPid.fromId(next.getResourceId());
1078
1079                        /*
1080                         * If a specific version is requested via an include, we'll replace the current version
1081                         * with the specific desired version. This is not the most efficient thing, given that
1082                         * we're loading the current version and then turning around and throwing it away again.
1083                         * This could be optimized and probably should be, but it's not critical given that
1084                         * this only applies to includes, which don't tend to be massive in numbers.
1085                         */
1086                        if (resourcePidToVersion != null) {
1087                                Long version = resourcePidToVersion.get(next.getResourceId());
1088                                resourceId.setVersion(version);
1089                                if (version != null && !version.equals(next.getVersion())) {
1090                                        IFhirResourceDao<? extends IBaseResource> dao = myDaoRegistry.getResourceDao(resourceType);
1091                                        next = (IBaseResourceEntity)
1092                                                        dao.readEntity(next.getIdDt().withVersion(Long.toString(version)), null);
1093                                }
1094                        }
1095
1096                        IBaseResource resource = null;
1097                        if (next != null) {
1098                                resource = myJpaStorageResourceParser.toResource(
1099                                                resourceType, next, tagMap.get(next.getId()), theForHistoryOperation);
1100                        }
1101                        if (resource == null) {
1102                                ourLog.warn(
1103                                                "Unable to find resource {}/{}/_history/{} in database",
1104                                                next.getResourceType(),
1105                                                next.getIdDt().getIdPart(),
1106                                                next.getVersion());
1107                                continue;
1108                        }
1109
1110                        Integer index = thePosition.get(resourceId);
1111                        if (index == null) {
1112                                ourLog.warn("Got back unexpected resource PID {}", resourceId);
1113                                continue;
1114                        }
1115
1116                        if (theIncludedPids.contains(resourceId)) {
1117                                ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.INCLUDE);
1118                        } else {
1119                                ResourceMetadataKeyEnum.ENTRY_SEARCH_MODE.put(resource, BundleEntrySearchModeEnum.MATCH);
1120                        }
1121
1122                        theResourceListToPopulate.set(index, resource);
1123                }
1124        }
1125
1126        private Map<Long, Collection<ResourceTag>> getResourceTagMap(
1127                        Collection<? extends IBaseResourceEntity> theResourceSearchViewList) {
1128
1129                List<Long> idList = new ArrayList<>(theResourceSearchViewList.size());
1130
1131                // -- find all resource has tags
1132                for (IBaseResourceEntity resource : theResourceSearchViewList) {
1133                        if (resource.isHasTags()) idList.add(resource.getId());
1134                }
1135
1136                return getPidToTagMap(idList);
1137        }
1138
1139        @Nonnull
1140        private Map<Long, Collection<ResourceTag>> getPidToTagMap(List<Long> thePidList) {
1141                Map<Long, Collection<ResourceTag>> tagMap = new HashMap<>();
1142
1143                // -- no tags
1144                if (thePidList.size() == 0) return tagMap;
1145
1146                // -- get all tags for the idList
1147                Collection<ResourceTag> tagList = myResourceTagDao.findByResourceIds(thePidList);
1148
1149                // -- build the map, key = resourceId, value = list of ResourceTag
1150                JpaPid resourceId;
1151                Collection<ResourceTag> tagCol;
1152                for (ResourceTag tag : tagList) {
1153
1154                        resourceId = JpaPid.fromId(tag.getResourceId());
1155                        tagCol = tagMap.get(resourceId.getId());
1156                        if (tagCol == null) {
1157                                tagCol = new ArrayList<>();
1158                                tagCol.add(tag);
1159                                tagMap.put(resourceId.getId(), tagCol);
1160                        } else {
1161                                tagCol.add(tag);
1162                        }
1163                }
1164
1165                return tagMap;
1166        }
1167
1168        @Override
1169        public void loadResourcesByPid(
1170                        Collection<JpaPid> thePids,
1171                        Collection<JpaPid> theIncludedPids,
1172                        List<IBaseResource> theResourceListToPopulate,
1173                        boolean theForHistoryOperation,
1174                        RequestDetails theDetails) {
1175                if (thePids.isEmpty()) {
1176                        ourLog.debug("The include pids are empty");
1177                        // return;
1178                }
1179
1180                // Dupes will cause a crash later anyhow, but this is expensive so only do it
1181                // when running asserts
1182                assert new HashSet<>(thePids).size() == thePids.size() : "PID list contains duplicates: " + thePids;
1183
1184                Map<JpaPid, Integer> position = new HashMap<>();
1185                for (JpaPid next : thePids) {
1186                        position.put(next, theResourceListToPopulate.size());
1187                        theResourceListToPopulate.add(null);
1188                }
1189
1190                // Can we fast track this loading by checking elastic search?
1191                if (isLoadingFromElasticSearchSupported(thePids)) {
1192                        try {
1193                                theResourceListToPopulate.addAll(loadResourcesFromElasticSearch(thePids));
1194                                return;
1195
1196                        } catch (ResourceNotFoundInIndexException theE) {
1197                                // some resources were not found in index, so we will inform this and resort to JPA search
1198                                ourLog.warn(
1199                                                "Some resources were not found in index. Make sure all resources were indexed. Resorting to database search.");
1200                        }
1201                }
1202
1203                // We only chunk because some jdbc drivers can't handle long param lists.
1204                new QueryChunker<JpaPid>()
1205                                .chunk(
1206                                                thePids,
1207                                                t -> doLoadPids(
1208                                                                t, theIncludedPids, theResourceListToPopulate, theForHistoryOperation, position));
1209        }
1210
1211        /**
1212         * Check if we can load the resources from Hibernate Search instead of the database.
1213         * We assume this is faster.
1214         * <p>
1215         * Hibernate Search only stores the current version, and only if enabled.
1216         *
1217         * @param thePids the pids to check for versioned references
1218         * @return can we fetch from Hibernate Search?
1219         */
1220        private boolean isLoadingFromElasticSearchSupported(Collection<JpaPid> thePids) {
1221                // is storage enabled?
1222                return myStorageSettings.isStoreResourceInHSearchIndex()
1223                                && myStorageSettings.isAdvancedHSearchIndexing()
1224                                &&
1225                                // we don't support history
1226                                thePids.stream().noneMatch(p -> p.getVersion() != null)
1227                                &&
1228                                // skip the complexity for metadata in dstu2
1229                                myContext.getVersion().getVersion().isEqualOrNewerThan(FhirVersionEnum.DSTU3);
1230        }
1231
1232        private List<IBaseResource> loadResourcesFromElasticSearch(Collection<JpaPid> thePids) {
1233                // Do we use the fulltextsvc via hibernate-search to load resources or be backwards compatible with older ES
1234                // only impl
1235                // to handle lastN?
1236                if (myStorageSettings.isAdvancedHSearchIndexing() && myStorageSettings.isStoreResourceInHSearchIndex()) {
1237                        List<Long> pidList = thePids.stream().map(pid -> (pid).getId()).collect(Collectors.toList());
1238
1239                        List<IBaseResource> resources = myFulltextSearchSvc.getResources(pidList);
1240                        return resources;
1241                } else if (!Objects.isNull(myParams) && myParams.isLastN()) {
1242                        // legacy LastN implementation
1243                        return myIElasticsearchSvc.getObservationResources(thePids);
1244                } else {
1245                        return Collections.emptyList();
1246                }
1247        }
1248
1249        /**
1250         * THIS SHOULD RETURN HASHSET and not just Set because we add to it later
1251         * so it can't be Collections.emptySet() or some such thing.
1252         * The JpaPid returned will have resource type populated.
1253         */
1254        @Override
1255        public Set<JpaPid> loadIncludes(
1256                        FhirContext theContext,
1257                        EntityManager theEntityManager,
1258                        Collection<JpaPid> theMatches,
1259                        Collection<Include> theIncludes,
1260                        boolean theReverseMode,
1261                        DateRangeParam theLastUpdated,
1262                        String theSearchIdOrDescription,
1263                        RequestDetails theRequest,
1264                        Integer theMaxCount) {
1265                SearchBuilderLoadIncludesParameters<JpaPid> parameters = new SearchBuilderLoadIncludesParameters<>();
1266                parameters.setFhirContext(theContext);
1267                parameters.setEntityManager(theEntityManager);
1268                parameters.setMatches(theMatches);
1269                parameters.setIncludeFilters(theIncludes);
1270                parameters.setReverseMode(theReverseMode);
1271                parameters.setLastUpdated(theLastUpdated);
1272                parameters.setSearchIdOrDescription(theSearchIdOrDescription);
1273                parameters.setRequestDetails(theRequest);
1274                parameters.setMaxCount(theMaxCount);
1275                return loadIncludes(parameters);
1276        }
1277
1278        @Override
1279        public Set<JpaPid> loadIncludes(SearchBuilderLoadIncludesParameters<JpaPid> theParameters) {
1280                Collection<JpaPid> matches = theParameters.getMatches();
1281                Collection<Include> currentIncludes = theParameters.getIncludeFilters();
1282                boolean reverseMode = theParameters.isReverseMode();
1283                EntityManager entityManager = theParameters.getEntityManager();
1284                Integer maxCount = theParameters.getMaxCount();
1285                FhirContext fhirContext = theParameters.getFhirContext();
1286                DateRangeParam lastUpdated = theParameters.getLastUpdated();
1287                RequestDetails request = theParameters.getRequestDetails();
1288                String searchIdOrDescription = theParameters.getSearchIdOrDescription();
1289                List<String> desiredResourceTypes = theParameters.getDesiredResourceTypes();
1290                boolean hasDesiredResourceTypes = desiredResourceTypes != null && !desiredResourceTypes.isEmpty();
1291                if (CompositeInterceptorBroadcaster.hasHooks(
1292                                Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, theParameters.getRequestDetails())) {
1293                        CurrentThreadCaptureQueriesListener.startCapturing();
1294                }
1295                if (matches.size() == 0) {
1296                        return new HashSet<>();
1297                }
1298                if (currentIncludes == null || currentIncludes.isEmpty()) {
1299                        return new HashSet<>();
1300                }
1301                String searchPidFieldName = reverseMode ? MY_TARGET_RESOURCE_PID : MY_SOURCE_RESOURCE_PID;
1302                String findPidFieldName = reverseMode ? MY_SOURCE_RESOURCE_PID : MY_TARGET_RESOURCE_PID;
1303                String findResourceTypeFieldName = reverseMode ? MY_SOURCE_RESOURCE_TYPE : MY_TARGET_RESOURCE_TYPE;
1304                String findVersionFieldName = null;
1305                if (!reverseMode && myStorageSettings.isRespectVersionsForSearchIncludes()) {
1306                        findVersionFieldName = MY_TARGET_RESOURCE_VERSION;
1307                }
1308
1309                List<JpaPid> nextRoundMatches = new ArrayList<>(matches);
1310                HashSet<JpaPid> allAdded = new HashSet<>();
1311                HashSet<JpaPid> original = new HashSet<>(matches);
1312                ArrayList<Include> includes = new ArrayList<>(currentIncludes);
1313
1314                int roundCounts = 0;
1315                StopWatch w = new StopWatch();
1316
1317                boolean addedSomeThisRound;
1318                do {
1319                        roundCounts++;
1320
1321                        HashSet<JpaPid> pidsToInclude = new HashSet<>();
1322
1323                        for (Iterator<Include> iter = includes.iterator(); iter.hasNext(); ) {
1324                                Include nextInclude = iter.next();
1325                                if (nextInclude.isRecurse() == false) {
1326                                        iter.remove();
1327                                }
1328
1329                                // Account for _include=*
1330                                boolean matchAll = "*".equals(nextInclude.getValue());
1331
1332                                // Account for _include=[resourceType]:*
1333                                String wantResourceType = null;
1334                                if (!matchAll) {
1335                                        if ("*".equals(nextInclude.getParamName())) {
1336                                                wantResourceType = nextInclude.getParamType();
1337                                                matchAll = true;
1338                                        }
1339                                }
1340
1341                                if (matchAll) {
1342                                        StringBuilder sqlBuilder = new StringBuilder();
1343                                        sqlBuilder.append("SELECT r.").append(findPidFieldName);
1344                                        sqlBuilder.append(", r.").append(findResourceTypeFieldName);
1345                                        if (findVersionFieldName != null) {
1346                                                sqlBuilder.append(", r.").append(findVersionFieldName);
1347                                        }
1348                                        sqlBuilder.append(" FROM ResourceLink r WHERE ");
1349
1350                                        sqlBuilder.append("r.");
1351                                        sqlBuilder.append(searchPidFieldName); // (rev mode) target_resource_id | source_resource_id
1352                                        sqlBuilder.append(" IN (:target_pids)");
1353
1354                                        /*
1355                                         * We need to set the resource type in 2 cases only:
1356                                         * 1) we are in $everything mode
1357                                         *              (where we only want to fetch specific resource types, regardless of what is
1358                                         *              available to fetch)
1359                                         * 2) we are doing revincludes
1360                                         *
1361                                         *      Technically if the request is a qualified star (e.g. _include=Observation:*) we
1362                                         * should always be checking the source resource type on the resource link. We don't
1363                                         * actually index that column though by default, so in order to try and be efficient
1364                                         * we don't actually include it for includes (but we do for revincludes). This is
1365                                         * because for an include, it doesn't really make sense to include a different
1366                                         * resource type than the one you are searching on.
1367                                         */
1368                                        if (wantResourceType != null
1369                                                        && (reverseMode || (myParams != null && myParams.getEverythingMode() != null))) {
1370                                                // because mySourceResourceType is not part of the HFJ_RES_LINK
1371                                                // index, this might not be the most optimal performance.
1372                                                // but it is for an $everything operation (and maybe we should update the index)
1373                                                sqlBuilder.append(" AND r.mySourceResourceType = :want_resource_type");
1374                                        } else {
1375                                                wantResourceType = null;
1376                                        }
1377
1378                                        // When calling $everything on a Patient instance, we don't want to recurse into new Patient
1379                                        // resources
1380                                        // (e.g. via Provenance, List, or Group) when in an $everything operation
1381                                        if (myParams != null
1382                                                        && myParams.getEverythingMode() == SearchParameterMap.EverythingModeEnum.PATIENT_INSTANCE) {
1383                                                sqlBuilder.append(" AND r.myTargetResourceType != 'Patient'");
1384                                                sqlBuilder.append(UNDESIRED_RESOURCE_LINKAGES_FOR_EVERYTHING_ON_PATIENT_INSTANCE.stream()
1385                                                                .collect(Collectors.joining("', '", " AND r.mySourceResourceType NOT IN ('", "')")));
1386                                        }
1387                                        if (hasDesiredResourceTypes) {
1388                                                sqlBuilder.append(" AND r.myTargetResourceType IN (:desired_target_resource_types)");
1389                                        }
1390
1391                                        String sql = sqlBuilder.toString();
1392                                        List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize());
1393                                        for (Collection<JpaPid> nextPartition : partitions) {
1394                                                TypedQuery<?> q = entityManager.createQuery(sql, Object[].class);
1395                                                q.setParameter("target_pids", JpaPid.toLongList(nextPartition));
1396                                                if (wantResourceType != null) {
1397                                                        q.setParameter("want_resource_type", wantResourceType);
1398                                                }
1399                                                if (maxCount != null) {
1400                                                        q.setMaxResults(maxCount);
1401                                                }
1402                                                if (hasDesiredResourceTypes) {
1403                                                        q.setParameter("desired_target_resource_types", desiredResourceTypes);
1404                                                }
1405                                                List<?> results = q.getResultList();
1406                                                for (Object nextRow : results) {
1407                                                        if (nextRow == null) {
1408                                                                // This can happen if there are outgoing references which are canonical or point to
1409                                                                // other servers
1410                                                                continue;
1411                                                        }
1412
1413                                                        Long version = null;
1414                                                        Long resourceLink = (Long) ((Object[]) nextRow)[0];
1415                                                        String resourceType = (String) ((Object[]) nextRow)[1];
1416                                                        if (findVersionFieldName != null) {
1417                                                                version = (Long) ((Object[]) nextRow)[2];
1418                                                        }
1419
1420                                                        if (resourceLink != null) {
1421                                                                JpaPid pid =
1422                                                                                JpaPid.fromIdAndVersionAndResourceType(resourceLink, version, resourceType);
1423                                                                pidsToInclude.add(pid);
1424                                                        }
1425                                                }
1426                                        }
1427                                } else {
1428                                        List<String> paths;
1429
1430                                        // Start replace
1431                                        RuntimeSearchParam param;
1432                                        String resType = nextInclude.getParamType();
1433                                        if (isBlank(resType)) {
1434                                                continue;
1435                                        }
1436                                        RuntimeResourceDefinition def = fhirContext.getResourceDefinition(resType);
1437                                        if (def == null) {
1438                                                ourLog.warn("Unknown resource type in include/revinclude=" + nextInclude.getValue());
1439                                                continue;
1440                                        }
1441
1442                                        String paramName = nextInclude.getParamName();
1443                                        if (isNotBlank(paramName)) {
1444                                                param = mySearchParamRegistry.getActiveSearchParam(resType, paramName);
1445                                        } else {
1446                                                param = null;
1447                                        }
1448                                        if (param == null) {
1449                                                ourLog.warn("Unknown param name in include/revinclude=" + nextInclude.getValue());
1450                                                continue;
1451                                        }
1452
1453                                        paths = param.getPathsSplitForResourceType(resType);
1454                                        // end replace
1455
1456                                        Set<String> targetResourceTypes = computeTargetResourceTypes(nextInclude, param);
1457
1458                                        for (String nextPath : paths) {
1459                                                String findPidFieldSqlColumn = findPidFieldName.equals(MY_SOURCE_RESOURCE_PID)
1460                                                                ? "src_resource_id"
1461                                                                : "target_resource_id";
1462                                                String fieldsToLoad = "r." + findPidFieldSqlColumn + " AS " + RESOURCE_ID_ALIAS;
1463                                                if (findVersionFieldName != null) {
1464                                                        fieldsToLoad += ", r.target_resource_version AS " + RESOURCE_VERSION_ALIAS;
1465                                                }
1466
1467                                                // Query for includes lookup has 2 cases
1468                                                // Case 1: Where target_resource_id is available in hfj_res_link table for local references
1469                                                // Case 2: Where target_resource_id is null in hfj_res_link table and referred by a canonical
1470                                                // url in target_resource_url
1471
1472                                                // Case 1:
1473                                                Map<String, Object> localReferenceQueryParams = new HashMap<>();
1474
1475                                                String searchPidFieldSqlColumn = searchPidFieldName.equals(MY_TARGET_RESOURCE_PID)
1476                                                                ? "target_resource_id"
1477                                                                : "src_resource_id";
1478                                                StringBuilder localReferenceQuery =
1479                                                                new StringBuilder("SELECT " + fieldsToLoad + " FROM hfj_res_link r "
1480                                                                                + " WHERE r.src_path = :src_path AND "
1481                                                                                + " r.target_resource_id IS NOT NULL AND "
1482                                                                                + " r."
1483                                                                                + searchPidFieldSqlColumn + " IN (:target_pids) ");
1484                                                localReferenceQueryParams.put("src_path", nextPath);
1485                                                // we loop over target_pids later.
1486                                                if (targetResourceTypes != null) {
1487                                                        if (targetResourceTypes.size() == 1) {
1488                                                                localReferenceQuery.append(" AND r.target_resource_type = :target_resource_type ");
1489                                                                localReferenceQueryParams.put(
1490                                                                                "target_resource_type",
1491                                                                                targetResourceTypes.iterator().next());
1492                                                        } else {
1493                                                                localReferenceQuery.append(" AND r.target_resource_type in (:target_resource_types) ");
1494                                                                localReferenceQueryParams.put("target_resource_types", targetResourceTypes);
1495                                                        }
1496                                                }
1497
1498                                                // Case 2:
1499                                                Pair<String, Map<String, Object>> canonicalQuery = buildCanonicalUrlQuery(
1500                                                                findVersionFieldName, searchPidFieldSqlColumn, targetResourceTypes);
1501
1502                                                // @formatter:on
1503
1504                                                String sql = localReferenceQuery + " UNION " + canonicalQuery.getLeft();
1505
1506                                                List<Collection<JpaPid>> partitions = partition(nextRoundMatches, getMaximumPageSize());
1507                                                for (Collection<JpaPid> nextPartition : partitions) {
1508                                                        Query q = entityManager.createNativeQuery(sql, Tuple.class);
1509                                                        q.setParameter("target_pids", JpaPid.toLongList(nextPartition));
1510                                                        localReferenceQueryParams.forEach(q::setParameter);
1511                                                        canonicalQuery.getRight().forEach(q::setParameter);
1512
1513                                                        if (maxCount != null) {
1514                                                                q.setMaxResults(maxCount);
1515                                                        }
1516                                                        @SuppressWarnings("unchecked")
1517                                                        List<Tuple> results = q.getResultList();
1518                                                        for (Tuple result : results) {
1519                                                                if (result != null) {
1520                                                                        Long resourceId =
1521                                                                                        NumberUtils.createLong(String.valueOf(result.get(RESOURCE_ID_ALIAS)));
1522                                                                        Long resourceVersion = null;
1523                                                                        if (findVersionFieldName != null && result.get(RESOURCE_VERSION_ALIAS) != null) {
1524                                                                                resourceVersion = NumberUtils.createLong(
1525                                                                                                String.valueOf(result.get(RESOURCE_VERSION_ALIAS)));
1526                                                                        }
1527                                                                        pidsToInclude.add(JpaPid.fromIdAndVersion(resourceId, resourceVersion));
1528                                                                }
1529                                                        }
1530                                                }
1531                                        }
1532                                }
1533                        }
1534
1535                        nextRoundMatches.clear();
1536                        for (JpaPid next : pidsToInclude) {
1537                                if (!original.contains(next) && !allAdded.contains(next)) {
1538                                        nextRoundMatches.add(next);
1539                                }
1540                        }
1541
1542                        addedSomeThisRound = allAdded.addAll(pidsToInclude);
1543
1544                        if (maxCount != null && allAdded.size() >= maxCount) {
1545                                break;
1546                        }
1547
1548                } while (!includes.isEmpty() && !nextRoundMatches.isEmpty() && addedSomeThisRound);
1549
1550                allAdded.removeAll(original);
1551
1552                ourLog.info(
1553                                "Loaded {} {} in {} rounds and {} ms for search {}",
1554                                allAdded.size(),
1555                                reverseMode ? "_revincludes" : "_includes",
1556                                roundCounts,
1557                                w.getMillisAndRestart(),
1558                                searchIdOrDescription);
1559
1560                if (CompositeInterceptorBroadcaster.hasHooks(
1561                                Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, request)) {
1562                        callRawSqlHookWithCurrentThreadQueries(request);
1563                }
1564                // Interceptor call: STORAGE_PREACCESS_RESOURCES
1565                // This can be used to remove results from the search result details before
1566                // the user has a chance to know that they were in the results
1567                if (!allAdded.isEmpty()) {
1568
1569                        if (CompositeInterceptorBroadcaster.hasHooks(
1570                                        Pointcut.STORAGE_PREACCESS_RESOURCES, myInterceptorBroadcaster, request)) {
1571                                List<JpaPid> includedPidList = new ArrayList<>(allAdded);
1572                                JpaPreResourceAccessDetails accessDetails =
1573                                                new JpaPreResourceAccessDetails(includedPidList, () -> this);
1574                                HookParams params = new HookParams()
1575                                                .add(IPreResourceAccessDetails.class, accessDetails)
1576                                                .add(RequestDetails.class, request)
1577                                                .addIfMatchesType(ServletRequestDetails.class, request);
1578                                CompositeInterceptorBroadcaster.doCallHooks(
1579                                                myInterceptorBroadcaster, request, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
1580
1581                                for (int i = includedPidList.size() - 1; i >= 0; i--) {
1582                                        if (accessDetails.isDontReturnResourceAtIndex(i)) {
1583                                                JpaPid value = includedPidList.remove(i);
1584                                                if (value != null) {
1585                                                        allAdded.remove(value);
1586                                                }
1587                                        }
1588                                }
1589                        }
1590                }
1591
1592                return allAdded;
1593        }
1594
1595        /**
1596         * Given a
1597         * @param request
1598         */
1599        private void callRawSqlHookWithCurrentThreadQueries(RequestDetails request) {
1600                SqlQueryList capturedQueries = CurrentThreadCaptureQueriesListener.getCurrentQueueAndStopCapturing();
1601                HookParams params = new HookParams()
1602                                .add(RequestDetails.class, request)
1603                                .addIfMatchesType(ServletRequestDetails.class, request)
1604                                .add(SqlQueryList.class, capturedQueries);
1605                CompositeInterceptorBroadcaster.doCallHooks(
1606                                myInterceptorBroadcaster, request, Pointcut.JPA_PERFTRACE_RAW_SQL, params);
1607        }
1608
1609        @Nullable
1610        private static Set<String> computeTargetResourceTypes(Include nextInclude, RuntimeSearchParam param) {
1611                String targetResourceType = defaultString(nextInclude.getParamTargetType(), null);
1612                boolean haveTargetTypesDefinedByParam = param.hasTargets();
1613                Set<String> targetResourceTypes;
1614                if (targetResourceType != null) {
1615                        targetResourceTypes = Set.of(targetResourceType);
1616                } else if (haveTargetTypesDefinedByParam) {
1617                        targetResourceTypes = param.getTargets();
1618                } else {
1619                        // all types!
1620                        targetResourceTypes = null;
1621                }
1622                return targetResourceTypes;
1623        }
1624
1625        @Nonnull
1626        private Pair<String, Map<String, Object>> buildCanonicalUrlQuery(
1627                        String theVersionFieldName, String thePidFieldSqlColumn, Set<String> theTargetResourceTypes) {
1628                String fieldsToLoadFromSpidxUriTable = "rUri.res_id";
1629                if (theVersionFieldName != null) {
1630                        // canonical-uri references aren't versioned, but we need to match the column count for the UNION
1631                        fieldsToLoadFromSpidxUriTable += ", NULL";
1632                }
1633                // The logical join will be by hfj_spidx_uri on sp_name='uri' and sp_uri=target_resource_url.
1634                // But sp_name isn't indexed, so we use hash_identity instead.
1635                if (theTargetResourceTypes == null) {
1636                        // hash_identity includes the resource type.  So a null wildcard must be replaced with a list of all types.
1637                        theTargetResourceTypes = myDaoRegistry.getRegisteredDaoTypes();
1638                }
1639                assert !theTargetResourceTypes.isEmpty();
1640
1641                Set<Long> identityHashesForTypes = theTargetResourceTypes.stream()
1642                                .map(type -> BaseResourceIndexedSearchParam.calculateHashIdentity(
1643                                                myPartitionSettings, myRequestPartitionId, type, "url"))
1644                                .collect(Collectors.toSet());
1645
1646                Map<String, Object> canonicalUriQueryParams = new HashMap<>();
1647                StringBuilder canonicalUrlQuery = new StringBuilder(
1648                                "SELECT " + fieldsToLoadFromSpidxUriTable + " FROM hfj_res_link r " + " JOIN hfj_spidx_uri rUri ON ( ");
1649                // join on hash_identity and sp_uri - indexed in IDX_SP_URI_HASH_IDENTITY_V2
1650                if (theTargetResourceTypes.size() == 1) {
1651                        canonicalUrlQuery.append("   rUri.hash_identity = :uri_identity_hash ");
1652                        canonicalUriQueryParams.put(
1653                                        "uri_identity_hash", identityHashesForTypes.iterator().next());
1654                } else {
1655                        canonicalUrlQuery.append("   rUri.hash_identity in (:uri_identity_hashes) ");
1656                        canonicalUriQueryParams.put("uri_identity_hashes", identityHashesForTypes);
1657                }
1658
1659                canonicalUrlQuery.append("  AND r.target_resource_url = rUri.sp_uri  )" + " WHERE r.src_path = :src_path AND "
1660                                + " r.target_resource_id IS NULL AND "
1661                                + " r."
1662                                + thePidFieldSqlColumn + " IN (:target_pids) ");
1663                return Pair.of(canonicalUrlQuery.toString(), canonicalUriQueryParams);
1664        }
1665
1666        private List<Collection<JpaPid>> partition(Collection<JpaPid> theNextRoundMatches, int theMaxLoad) {
1667                if (theNextRoundMatches.size() <= theMaxLoad) {
1668                        return Collections.singletonList(theNextRoundMatches);
1669                } else {
1670
1671                        List<Collection<JpaPid>> retVal = new ArrayList<>();
1672                        Collection<JpaPid> current = null;
1673                        for (JpaPid next : theNextRoundMatches) {
1674                                if (current == null) {
1675                                        current = new ArrayList<>(theMaxLoad);
1676                                        retVal.add(current);
1677                                }
1678
1679                                current.add(next);
1680
1681                                if (current.size() >= theMaxLoad) {
1682                                        current = null;
1683                                }
1684                        }
1685
1686                        return retVal;
1687                }
1688        }
1689
1690        private void attemptComboUniqueSpProcessing(
1691                        QueryStack theQueryStack3, @Nonnull SearchParameterMap theParams, RequestDetails theRequest) {
1692                RuntimeSearchParam comboParam = null;
1693                List<String> comboParamNames = null;
1694                List<RuntimeSearchParam> exactMatchParams =
1695                                mySearchParamRegistry.getActiveComboSearchParams(myResourceName, theParams.keySet());
1696                if (exactMatchParams.size() > 0) {
1697                        comboParam = exactMatchParams.get(0);
1698                        comboParamNames = new ArrayList<>(theParams.keySet());
1699                }
1700
1701                if (comboParam == null) {
1702                        List<RuntimeSearchParam> candidateComboParams =
1703                                        mySearchParamRegistry.getActiveComboSearchParams(myResourceName);
1704                        for (RuntimeSearchParam nextCandidate : candidateComboParams) {
1705                                List<String> nextCandidateParamNames =
1706                                                JpaParamUtil.resolveComponentParameters(mySearchParamRegistry, nextCandidate).stream()
1707                                                                .map(t -> t.getName())
1708                                                                .collect(Collectors.toList());
1709                                if (theParams.keySet().containsAll(nextCandidateParamNames)) {
1710                                        comboParam = nextCandidate;
1711                                        comboParamNames = nextCandidateParamNames;
1712                                        break;
1713                                }
1714                        }
1715                }
1716
1717                if (comboParam != null) {
1718                        // Since we're going to remove elements below
1719                        theParams.values().forEach(nextAndList -> ensureSubListsAreWritable(nextAndList));
1720
1721                        StringBuilder sb = new StringBuilder();
1722                        sb.append(myResourceName);
1723                        sb.append("?");
1724
1725                        boolean first = true;
1726
1727                        Collections.sort(comboParamNames);
1728                        for (String nextParamName : comboParamNames) {
1729                                List<List<IQueryParameterType>> nextValues = theParams.get(nextParamName);
1730
1731                                // TODO Hack to fix weird IOOB on the next stanza until James comes back and makes sense of this.
1732                                if (nextValues.isEmpty()) {
1733                                        ourLog.error(
1734                                                        "query parameter {} is unexpectedly empty. Encountered while considering {} index for {}",
1735                                                        nextParamName,
1736                                                        comboParam.getName(),
1737                                                        theRequest.getCompleteUrl());
1738                                        sb = null;
1739                                        break;
1740                                }
1741
1742                                if (nextValues.get(0).size() != 1) {
1743                                        sb = null;
1744                                        break;
1745                                }
1746
1747                                // Reference params are only eligible for using a composite index if they
1748                                // are qualified
1749                                RuntimeSearchParam nextParamDef =
1750                                                mySearchParamRegistry.getActiveSearchParam(myResourceName, nextParamName);
1751                                if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.REFERENCE) {
1752                                        ReferenceParam param = (ReferenceParam) nextValues.get(0).get(0);
1753                                        if (isBlank(param.getResourceType())) {
1754                                                sb = null;
1755                                                break;
1756                                        }
1757                                }
1758
1759                                List<? extends IQueryParameterType> nextAnd = nextValues.remove(0);
1760                                IQueryParameterType nextOr = nextAnd.remove(0);
1761                                String nextOrValue = nextOr.getValueAsQueryToken(myContext);
1762
1763                                if (comboParam.getComboSearchParamType() == ComboSearchParamType.NON_UNIQUE) {
1764                                        if (nextParamDef.getParamType() == RestSearchParameterTypeEnum.STRING) {
1765                                                nextOrValue = StringUtil.normalizeStringForSearchIndexing(nextOrValue);
1766                                        }
1767                                }
1768
1769                                if (first) {
1770                                        first = false;
1771                                } else {
1772                                        sb.append('&');
1773                                }
1774
1775                                nextParamName = UrlUtil.escapeUrlParam(nextParamName);
1776                                nextOrValue = UrlUtil.escapeUrlParam(nextOrValue);
1777
1778                                sb.append(nextParamName).append('=').append(nextOrValue);
1779                        }
1780
1781                        if (sb != null) {
1782                                String indexString = sb.toString();
1783                                ourLog.debug(
1784                                                "Checking for {} combo index for query: {}", comboParam.getComboSearchParamType(), indexString);
1785
1786                                // Interceptor broadcast: JPA_PERFTRACE_INFO
1787                                StorageProcessingMessage msg = new StorageProcessingMessage()
1788                                                .setMessage("Using " + comboParam.getComboSearchParamType() + " index for query for search: "
1789                                                                + indexString);
1790                                HookParams params = new HookParams()
1791                                                .add(RequestDetails.class, theRequest)
1792                                                .addIfMatchesType(ServletRequestDetails.class, theRequest)
1793                                                .add(StorageProcessingMessage.class, msg);
1794                                CompositeInterceptorBroadcaster.doCallHooks(
1795                                                myInterceptorBroadcaster, theRequest, Pointcut.JPA_PERFTRACE_INFO, params);
1796
1797                                switch (comboParam.getComboSearchParamType()) {
1798                                        case UNIQUE:
1799                                                theQueryStack3.addPredicateCompositeUnique(indexString, myRequestPartitionId);
1800                                                break;
1801                                        case NON_UNIQUE:
1802                                                theQueryStack3.addPredicateCompositeNonUnique(indexString, myRequestPartitionId);
1803                                                break;
1804                                }
1805
1806                                // Remove any empty parameters remaining after this
1807                                theParams.clean();
1808                        }
1809                }
1810        }
1811
1812        private <T> void ensureSubListsAreWritable(List<List<T>> theListOfLists) {
1813                for (int i = 0; i < theListOfLists.size(); i++) {
1814                        List<T> oldSubList = theListOfLists.get(i);
1815                        if (!(oldSubList instanceof ArrayList)) {
1816                                List<T> newSubList = new ArrayList<>(oldSubList);
1817                                theListOfLists.set(i, newSubList);
1818                        }
1819                }
1820        }
1821
1822        @Override
1823        public void setFetchSize(int theFetchSize) {
1824                myFetchSize = theFetchSize;
1825        }
1826
1827        public SearchParameterMap getParams() {
1828                return myParams;
1829        }
1830
1831        public CriteriaBuilder getBuilder() {
1832                return myCriteriaBuilder;
1833        }
1834
1835        public Class<? extends IBaseResource> getResourceType() {
1836                return myResourceType;
1837        }
1838
1839        public String getResourceName() {
1840                return myResourceName;
1841        }
1842
1843        /**
1844         * IncludesIterator, used to recursively fetch resources from the provided list of PIDs
1845         */
1846        public class IncludesIterator extends BaseIterator<JpaPid> implements Iterator<JpaPid> {
1847
1848                private final RequestDetails myRequest;
1849                private final Set<JpaPid> myCurrentPids;
1850                private Iterator<JpaPid> myCurrentIterator;
1851                private JpaPid myNext;
1852
1853                IncludesIterator(Set<JpaPid> thePidSet, RequestDetails theRequest) {
1854                        myCurrentPids = new HashSet<>(thePidSet);
1855                        myCurrentIterator = null;
1856                        myRequest = theRequest;
1857                }
1858
1859                private void fetchNext() {
1860                        while (myNext == null) {
1861
1862                                if (myCurrentIterator == null) {
1863                                        Set<Include> includes = new HashSet<>();
1864                                        if (myParams.containsKey(Constants.PARAM_TYPE)) {
1865                                                for (List<IQueryParameterType> typeList : myParams.get(Constants.PARAM_TYPE)) {
1866                                                        for (IQueryParameterType type : typeList) {
1867                                                                String queryString = ParameterUtil.unescape(type.getValueAsQueryToken(myContext));
1868                                                                for (String resourceType : queryString.split(",")) {
1869                                                                        String rt = resourceType.trim();
1870                                                                        if (isNotBlank(rt)) {
1871                                                                                includes.add(new Include(rt + ":*", true));
1872                                                                        }
1873                                                                }
1874                                                        }
1875                                                }
1876                                        }
1877                                        if (includes.isEmpty()) {
1878                                                includes.add(new Include("*", true));
1879                                        }
1880                                        Set<JpaPid> newPids = loadIncludes(
1881                                                        myContext,
1882                                                        myEntityManager,
1883                                                        myCurrentPids,
1884                                                        includes,
1885                                                        false,
1886                                                        getParams().getLastUpdated(),
1887                                                        mySearchUuid,
1888                                                        myRequest,
1889                                                        null);
1890                                        myCurrentIterator = newPids.iterator();
1891                                }
1892
1893                                if (myCurrentIterator.hasNext()) {
1894                                        myNext = myCurrentIterator.next();
1895                                } else {
1896                                        myNext = NO_MORE;
1897                                }
1898                        }
1899                }
1900
1901                @Override
1902                public boolean hasNext() {
1903                        fetchNext();
1904                        return !NO_MORE.equals(myNext);
1905                }
1906
1907                @Override
1908                public JpaPid next() {
1909                        fetchNext();
1910                        JpaPid retVal = myNext;
1911                        myNext = null;
1912                        return retVal;
1913                }
1914        }
1915
1916        /**
1917         * Basic Query iterator, used to fetch the results of a query.
1918         */
1919        private final class QueryIterator extends BaseIterator<JpaPid> implements IResultIterator<JpaPid> {
1920
1921                private final SearchRuntimeDetails mySearchRuntimeDetails;
1922                private final RequestDetails myRequest;
1923                private final boolean myHaveRawSqlHooks;
1924                private final boolean myHavePerfTraceFoundIdHook;
1925                private final SortSpec mySort;
1926                private final Integer myOffset;
1927                private boolean myFirst = true;
1928                private IncludesIterator myIncludesIterator;
1929                /**
1930                 * The next JpaPid value of the next result in this query.
1931                 * Will not be null if fetched using getNext()
1932                 */
1933                private JpaPid myNext;
1934                /**
1935                 * The current query result iterator running sql and supplying PIDs
1936                 * @see #myQueryList
1937                 */
1938                private ISearchQueryExecutor myResultsIterator;
1939
1940                private boolean myFetchIncludesForEverythingOperation;
1941                /**
1942                 * The count of resources skipped because they were seen in earlier results
1943                 */
1944                private int mySkipCount = 0;
1945                /**
1946                 * The count of resources that are new in this search
1947                 * (ie, not cached in previous searches)
1948                 */
1949                private int myNonSkipCount = 0;
1950
1951                /**
1952                 * The list of queries to use to find all results.
1953                 * Normal JPA queries will normally have a single entry.
1954                 * Queries that involve Hibernate Search/Elastisearch may have
1955                 * multiple queries because of chunking.
1956                 * The $everything operation also jams some extra results in.
1957                 */
1958                private List<ISearchQueryExecutor> myQueryList = new ArrayList<>();
1959
1960                private QueryIterator(SearchRuntimeDetails theSearchRuntimeDetails, RequestDetails theRequest) {
1961                        mySearchRuntimeDetails = theSearchRuntimeDetails;
1962                        mySort = myParams.getSort();
1963                        myOffset = myParams.getOffset();
1964                        myRequest = theRequest;
1965
1966                        // everything requires fetching recursively all related resources
1967                        if (myParams.getEverythingMode() != null) {
1968                                myFetchIncludesForEverythingOperation = true;
1969                        }
1970
1971                        myHavePerfTraceFoundIdHook = CompositeInterceptorBroadcaster.hasHooks(
1972                                        Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID, myInterceptorBroadcaster, myRequest);
1973                        myHaveRawSqlHooks = CompositeInterceptorBroadcaster.hasHooks(
1974                                        Pointcut.JPA_PERFTRACE_RAW_SQL, myInterceptorBroadcaster, myRequest);
1975                }
1976
1977                private void fetchNext() {
1978                        try {
1979                                if (myHaveRawSqlHooks) {
1980                                        CurrentThreadCaptureQueriesListener.startCapturing();
1981                                }
1982
1983                                // If we don't have a query yet, create one
1984                                if (myResultsIterator == null) {
1985                                        if (myMaxResultsToFetch == null) {
1986                                                if (myParams.getLoadSynchronousUpTo() != null) {
1987                                                        myMaxResultsToFetch = myParams.getLoadSynchronousUpTo();
1988                                                } else if (myParams.getOffset() != null && myParams.getCount() != null) {
1989                                                        myMaxResultsToFetch = myParams.getCount();
1990                                                } else {
1991                                                        myMaxResultsToFetch = myStorageSettings.getFetchSizeDefaultMaximum();
1992                                                }
1993                                        }
1994
1995                                        /*
1996                                         * assigns the results iterator
1997                                         * and populates the myQueryList.
1998                                         */
1999                                        initializeIteratorQuery(myOffset, myMaxResultsToFetch);
2000                                }
2001
2002                                if (myNext == null) {
2003                                        // no next means we need a new query (if one is available)
2004                                        while (myResultsIterator.hasNext() || !myQueryList.isEmpty()) {
2005                                                // Update iterator with next chunk if necessary.
2006                                                if (!myResultsIterator.hasNext()) {
2007                                                        retrieveNextIteratorQuery();
2008
2009                                                        // if our new results iterator is also empty
2010                                                        // we're done here
2011                                                        if (!myResultsIterator.hasNext()) {
2012                                                                break;
2013                                                        }
2014                                                }
2015
2016                                                Long nextLong = myResultsIterator.next();
2017                                                if (myHavePerfTraceFoundIdHook) {
2018                                                        HookParams params = new HookParams()
2019                                                                        .add(Integer.class, System.identityHashCode(this))
2020                                                                        .add(Object.class, nextLong);
2021                                                        CompositeInterceptorBroadcaster.doCallHooks(
2022                                                                        myInterceptorBroadcaster,
2023                                                                        myRequest,
2024                                                                        Pointcut.JPA_PERFTRACE_SEARCH_FOUND_ID,
2025                                                                        params);
2026                                                }
2027
2028                                                if (nextLong != null) {
2029                                                        JpaPid next = JpaPid.fromId(nextLong);
2030                                                        if (myPidSet.add(next)) {
2031                                                                myNext = next;
2032                                                                myNonSkipCount++;
2033                                                                break;
2034                                                        } else {
2035                                                                mySkipCount++;
2036                                                        }
2037                                                }
2038
2039                                                if (!myResultsIterator.hasNext()) {
2040                                                        if (myMaxResultsToFetch != null && (mySkipCount + myNonSkipCount == myMaxResultsToFetch)) {
2041                                                                if (mySkipCount > 0 && myNonSkipCount == 0) {
2042
2043                                                                        sendProcessingMsgAndFirePerformanceHook();
2044
2045                                                                        myMaxResultsToFetch += 1000;
2046                                                                        initializeIteratorQuery(myOffset, myMaxResultsToFetch);
2047                                                                }
2048                                                        }
2049                                                }
2050                                        }
2051                                }
2052
2053                                if (myNext == null) {
2054                                        // if we got here, it means the current JpaPid has already been processed,
2055                                        // and we will decide (here) if we need to fetch related resources recursively
2056                                        if (myFetchIncludesForEverythingOperation) {
2057                                                myIncludesIterator = new IncludesIterator(myPidSet, myRequest);
2058                                                myFetchIncludesForEverythingOperation = false;
2059                                        }
2060                                        if (myIncludesIterator != null) {
2061                                                while (myIncludesIterator.hasNext()) {
2062                                                        JpaPid next = myIncludesIterator.next();
2063                                                        if (next != null)
2064                                                                if (myPidSet.add(next)) {
2065                                                                        myNext = next;
2066                                                                        break;
2067                                                                }
2068                                                }
2069                                                if (myNext == null) {
2070                                                        myNext = NO_MORE;
2071                                                }
2072                                        } else {
2073                                                myNext = NO_MORE;
2074                                        }
2075                                }
2076
2077                                mySearchRuntimeDetails.setFoundMatchesCount(myPidSet.size());
2078
2079                        } finally {
2080                                // search finished - fire hooks
2081                                if (myHaveRawSqlHooks) {
2082                                        callRawSqlHookWithCurrentThreadQueries(myRequest);
2083                                }
2084                        }
2085
2086                        if (myFirst) {
2087                                HookParams params = new HookParams()
2088                                                .add(RequestDetails.class, myRequest)
2089                                                .addIfMatchesType(ServletRequestDetails.class, myRequest)
2090                                                .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
2091                                CompositeInterceptorBroadcaster.doCallHooks(
2092                                                myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_FIRST_RESULT_LOADED, params);
2093                                myFirst = false;
2094                        }
2095
2096                        if (NO_MORE.equals(myNext)) {
2097                                HookParams params = new HookParams()
2098                                                .add(RequestDetails.class, myRequest)
2099                                                .addIfMatchesType(ServletRequestDetails.class, myRequest)
2100                                                .add(SearchRuntimeDetails.class, mySearchRuntimeDetails);
2101                                CompositeInterceptorBroadcaster.doCallHooks(
2102                                                myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_SEARCH_SELECT_COMPLETE, params);
2103                        }
2104                }
2105
2106                private void sendProcessingMsgAndFirePerformanceHook() {
2107                        StorageProcessingMessage message = new StorageProcessingMessage();
2108                        String msg = "Pass completed with no matching results seeking rows "
2109                                        + myPidSet.size() + "-" + mySkipCount
2110                                        + ". This indicates an inefficient query! Retrying with new max count of "
2111                                        + myMaxResultsToFetch;
2112                        ourLog.warn(msg);
2113                        message.setMessage(msg);
2114                        HookParams params = new HookParams()
2115                                        .add(RequestDetails.class, myRequest)
2116                                        .addIfMatchesType(ServletRequestDetails.class, myRequest)
2117                                        .add(StorageProcessingMessage.class, message);
2118                        CompositeInterceptorBroadcaster.doCallHooks(
2119                                        myInterceptorBroadcaster, myRequest, Pointcut.JPA_PERFTRACE_WARNING, params);
2120                }
2121
2122                private void initializeIteratorQuery(Integer theOffset, Integer theMaxResultsToFetch) {
2123                        if (myQueryList.isEmpty()) {
2124                                // Capture times for Lucene/Elasticsearch queries as well
2125                                mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
2126                                myQueryList = createQuery(
2127                                                myParams, mySort, theOffset, theMaxResultsToFetch, false, myRequest, mySearchRuntimeDetails);
2128                        }
2129
2130                        mySearchRuntimeDetails.setQueryStopwatch(new StopWatch());
2131
2132                        retrieveNextIteratorQuery();
2133
2134                        mySkipCount = 0;
2135                        myNonSkipCount = 0;
2136                }
2137
2138                private void retrieveNextIteratorQuery() {
2139                        close();
2140                        if (myQueryList != null && myQueryList.size() > 0) {
2141                                myResultsIterator = myQueryList.remove(0);
2142                                myHasNextIteratorQuery = true;
2143                        } else {
2144                                myResultsIterator = SearchQueryExecutor.emptyExecutor();
2145                                myHasNextIteratorQuery = false;
2146                        }
2147                }
2148
2149                @Override
2150                public boolean hasNext() {
2151                        if (myNext == null) {
2152                                fetchNext();
2153                        }
2154                        return !NO_MORE.equals(myNext);
2155                }
2156
2157                @Override
2158                public JpaPid next() {
2159                        fetchNext();
2160                        JpaPid retVal = myNext;
2161                        myNext = null;
2162                        Validate.isTrue(!NO_MORE.equals(retVal), "No more elements");
2163                        return retVal;
2164                }
2165
2166                @Override
2167                public int getSkippedCount() {
2168                        return mySkipCount;
2169                }
2170
2171                @Override
2172                public int getNonSkippedCount() {
2173                        return myNonSkipCount;
2174                }
2175
2176                @Override
2177                public Collection<JpaPid> getNextResultBatch(long theBatchSize) {
2178                        Collection<JpaPid> batch = new ArrayList<>();
2179                        while (this.hasNext() && batch.size() < theBatchSize) {
2180                                batch.add(this.next());
2181                        }
2182                        return batch;
2183                }
2184
2185                @Override
2186                public void close() {
2187                        if (myResultsIterator != null) {
2188                                myResultsIterator.close();
2189                        }
2190                        myResultsIterator = null;
2191                }
2192        }
2193
2194        public static int getMaximumPageSize() {
2195                if (myUseMaxPageSize50ForTest) {
2196                        return MAXIMUM_PAGE_SIZE_FOR_TESTING;
2197                } else {
2198                        return MAXIMUM_PAGE_SIZE;
2199                }
2200        }
2201
2202        public static void setMaxPageSize50ForTest(boolean theIsTest) {
2203                myUseMaxPageSize50ForTest = theIsTest;
2204        }
2205}