Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.search;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.interceptor.api.HookParams;
024import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
025import ca.uhn.fhir.interceptor.api.Pointcut;
026import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
027import ca.uhn.fhir.interceptor.model.RequestPartitionId;
028import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
029import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
030import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
031import ca.uhn.fhir.jpa.dao.HistoryBuilder;
032import ca.uhn.fhir.jpa.dao.HistoryBuilderFactory;
033import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
034import ca.uhn.fhir.jpa.dao.ISearchBuilder;
035import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
036import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
037import ca.uhn.fhir.jpa.entity.Search;
038import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
039import ca.uhn.fhir.jpa.model.dao.JpaPid;
040import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
041import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
042import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
043import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
044import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum;
045import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
046import ca.uhn.fhir.jpa.util.MemoryCacheService;
047import ca.uhn.fhir.jpa.util.QueryParameterUtils;
048import ca.uhn.fhir.model.primitive.InstantDt;
049import ca.uhn.fhir.rest.api.server.IBundleProvider;
050import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
051import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
052import ca.uhn.fhir.rest.api.server.RequestDetails;
053import ca.uhn.fhir.rest.api.server.SimplePreResourceAccessDetails;
054import ca.uhn.fhir.rest.api.server.SimplePreResourceShowDetails;
055import ca.uhn.fhir.rest.server.interceptor.ServerInterceptorUtil;
056import ca.uhn.fhir.rest.server.method.ResponsePage;
057import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
058import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
059import com.google.common.annotations.VisibleForTesting;
060import jakarta.annotation.Nonnull;
061import jakarta.persistence.EntityManager;
062import jakarta.persistence.PersistenceContext;
063import org.hl7.fhir.instance.model.api.IBaseResource;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066import org.springframework.beans.factory.annotation.Autowired;
067
068import java.util.ArrayList;
069import java.util.Collections;
070import java.util.List;
071import java.util.Optional;
072import java.util.Set;
073import java.util.function.Function;
074
075public class PersistedJpaBundleProvider implements IBundleProvider {
076
077        private static final Logger ourLog = LoggerFactory.getLogger(PersistedJpaBundleProvider.class);
078
079        /*
080         * Autowired fields
081         */
082        protected final RequestDetails myRequest;
083
084        @Autowired
085        protected HapiTransactionService myTxService;
086
087        @PersistenceContext
088        private EntityManager myEntityManager;
089
090        @Autowired
091        private IInterceptorBroadcaster myInterceptorBroadcaster;
092
093        @Autowired
094        private SearchBuilderFactory<JpaPid> mySearchBuilderFactory;
095
096        @Autowired
097        private HistoryBuilderFactory myHistoryBuilderFactory;
098
099        @Autowired
100        private DaoRegistry myDaoRegistry;
101
102        @Autowired
103        private FhirContext myContext;
104
105        @Autowired
106        private ISearchCoordinatorSvc<JpaPid> mySearchCoordinatorSvc;
107
108        @Autowired
109        private ISearchCacheSvc mySearchCacheSvc;
110
111        @Autowired
112        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
113
114        @Autowired
115        private JpaStorageSettings myStorageSettings;
116
117        @Autowired
118        private MemoryCacheService myMemoryCacheService;
119
120        @Autowired
121        private IJpaStorageResourceParser myJpaStorageResourceParser;
122        /*
123         * Non autowired fields (will be different for every instance
124         * of this class, since it's a prototype
125         */
126        private Search mySearchEntity;
127        private final String myUuid;
128        private SearchCacheStatusEnum myCacheStatus;
129        private RequestPartitionId myRequestPartitionId;
130
131        /**
132         * Constructor
133         */
134        public PersistedJpaBundleProvider(RequestDetails theRequest, String theSearchUuid) {
135                myRequest = theRequest;
136                myUuid = theSearchUuid;
137        }
138
139        /**
140         * Constructor
141         */
142        public PersistedJpaBundleProvider(RequestDetails theRequest, Search theSearch) {
143                myRequest = theRequest;
144                mySearchEntity = theSearch;
145                myUuid = theSearch.getUuid();
146        }
147
148        @VisibleForTesting
149        public void setRequestPartitionHelperSvcForUnitTest(IRequestPartitionHelperSvc theRequestPartitionHelperSvc) {
150                myRequestPartitionHelperSvc = theRequestPartitionHelperSvc;
151        }
152
153        @VisibleForTesting
154        public Search getSearchEntityForTesting() {
155                return getSearchEntity();
156        }
157
158        protected Search getSearchEntity() {
159                return mySearchEntity;
160        }
161
162        // Note: Leave as protected, HSPC depends on this
163        @SuppressWarnings("WeakerAccess")
164        protected void setSearchEntity(Search theSearchEntity) {
165                mySearchEntity = theSearchEntity;
166        }
167
168        /**
169         * Perform a history search
170         */
171        private List<IBaseResource> doHistoryInTransaction(Integer theOffset, int theFromIndex, int theToIndex) {
172
173                HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
174                                mySearchEntity.getResourceType(),
175                                mySearchEntity.getResourceId(),
176                                mySearchEntity.getLastUpdatedLow(),
177                                mySearchEntity.getLastUpdatedHigh());
178
179                RequestPartitionId partitionId = getRequestPartitionId();
180                List<ResourceHistoryTable> results = historyBuilder.fetchEntities(
181                                partitionId, theOffset, theFromIndex, theToIndex, mySearchEntity.getHistorySearchStyle());
182
183                List<IBaseResource> retVal = new ArrayList<>();
184                for (ResourceHistoryTable next : results) {
185                        BaseHasResource resource;
186                        resource = next;
187
188                        retVal.add(myJpaStorageResourceParser.toResource(resource, true));
189                }
190
191                IInterceptorBroadcaster compositeBroadcaster =
192                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, myRequest);
193
194                // Interceptor call: STORAGE_PREACCESS_RESOURCES
195                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_PREACCESS_RESOURCES)) {
196                        SimplePreResourceAccessDetails accessDetails = new SimplePreResourceAccessDetails(retVal);
197                        HookParams params = new HookParams()
198                                        .add(IPreResourceAccessDetails.class, accessDetails)
199                                        .add(RequestDetails.class, myRequest)
200                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
201                        compositeBroadcaster.callHooks(Pointcut.STORAGE_PREACCESS_RESOURCES, params);
202
203                        for (int i = retVal.size() - 1; i >= 0; i--) {
204                                if (accessDetails.isDontReturnResourceAtIndex(i)) {
205                                        retVal.remove(i);
206                                }
207                        }
208                }
209
210                // Interceptor broadcast: STORAGE_PRESHOW_RESOURCES
211                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_PRESHOW_RESOURCES)) {
212                        SimplePreResourceShowDetails showDetails = new SimplePreResourceShowDetails(retVal);
213                        HookParams params = new HookParams()
214                                        .add(IPreResourceShowDetails.class, showDetails)
215                                        .add(RequestDetails.class, myRequest)
216                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
217                        compositeBroadcaster.callHooks(Pointcut.STORAGE_PRESHOW_RESOURCES, params);
218                        retVal = showDetails.toList();
219                }
220
221                return retVal;
222        }
223
224        @Nonnull
225        protected final RequestPartitionId getRequestPartitionId() {
226                if (myRequestPartitionId == null) {
227                        ReadPartitionIdRequestDetails details;
228                        if (mySearchEntity == null) {
229                                details = ReadPartitionIdRequestDetails.forSearchUuid(myUuid);
230                        } else if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
231                                details = ReadPartitionIdRequestDetails.forHistory(mySearchEntity.getResourceType(), null);
232                        } else {
233                                SearchParameterMap params =
234                                                mySearchEntity.getSearchParameterMap().orElse(null);
235                                details = ReadPartitionIdRequestDetails.forSearchType(mySearchEntity.getResourceType(), params, null);
236                        }
237                        myRequestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequest(myRequest, details);
238                }
239                return myRequestPartitionId;
240        }
241
242        public void setRequestPartitionId(RequestPartitionId theRequestPartitionId) {
243                myRequestPartitionId = theRequestPartitionId;
244        }
245
246        protected List<IBaseResource> doSearchOrEverything(
247                        final int theFromIndex,
248                        final int theToIndex,
249                        @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
250                if (mySearchEntity.getTotalCount() != null && mySearchEntity.getNumFound() <= 0) {
251                        // No resources to fetch (e.g. we did a _summary=count search)
252                        return Collections.emptyList();
253                }
254                String resourceName = mySearchEntity.getResourceType();
255                Class<? extends IBaseResource> resourceType =
256                                myContext.getResourceDefinition(resourceName).getImplementingClass();
257
258                final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(resourceName, resourceType);
259
260                RequestPartitionId requestPartitionId = getRequestPartitionId();
261                // we request 1 more resource than we need
262                // this is so we can be sure of when we hit the last page
263                // (when doing offset searches)
264                final List<JpaPid> pidsSubList = mySearchCoordinatorSvc.getResources(
265                                myUuid, theFromIndex, theToIndex + 1, myRequest, requestPartitionId);
266                // max list size should be either the entire list, or from - to length
267                int maxSize = Math.min(theToIndex - theFromIndex, pidsSubList.size());
268                theResponsePageBuilder.setTotalRequestedResourcesFetched(pidsSubList.size());
269
270                List<JpaPid> firstBatchOfPids = pidsSubList.subList(0, maxSize);
271                List<IBaseResource> resources = myTxService
272                                .withRequest(myRequest)
273                                .withRequestPartitionId(requestPartitionId)
274                                .execute(() -> {
275                                        return toResourceList(sb, firstBatchOfPids, theResponsePageBuilder);
276                                });
277
278                return resources;
279        }
280
281        /**
282         * Returns false if the entity can't be found
283         */
284        public boolean ensureSearchEntityLoaded() {
285                if (mySearchEntity == null) {
286                        Optional<Search> searchOpt = myTxService
287                                        .withRequest(myRequest)
288                                        .withRequestPartitionId(myRequestPartitionId)
289                                        .execute(() -> mySearchCacheSvc.fetchByUuid(myUuid, myRequestPartitionId));
290                        if (!searchOpt.isPresent()) {
291                                return false;
292                        }
293
294                        setSearchEntity(searchOpt.get());
295
296                        ourLog.trace(
297                                        "Retrieved search with version {} and total {}",
298                                        mySearchEntity.getVersion(),
299                                        mySearchEntity.getTotalCount());
300
301                        return true;
302                }
303
304                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
305                        if (mySearchEntity.getTotalCount() == null) {
306                                calculateHistoryCount();
307                        }
308                }
309
310                return true;
311        }
312
313        /**
314         * Note that this method is called outside a DB transaction, and uses a loading cache
315         * (assuming the default {@literal COUNT_CACHED} mode) so this effectively throttles
316         * access to the database by preventing multiple concurrent DB calls for an expensive
317         * count operation.
318         */
319        private void calculateHistoryCount() {
320                MemoryCacheService.HistoryCountKey key;
321                if (mySearchEntity.getResourceId() != null) {
322                        key = MemoryCacheService.HistoryCountKey.forInstance(mySearchEntity.getResourceId());
323                } else if (mySearchEntity.getResourceType() != null) {
324                        key = MemoryCacheService.HistoryCountKey.forType(mySearchEntity.getResourceType());
325                } else {
326                        key = MemoryCacheService.HistoryCountKey.forSystem();
327                }
328
329                Function<MemoryCacheService.HistoryCountKey, Integer> supplier = k -> myTxService
330                                .withRequest(myRequest)
331                                .withRequestPartitionId(getRequestPartitionId())
332                                .execute(() -> {
333                                        HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
334                                                        mySearchEntity.getResourceType(),
335                                                        mySearchEntity.getResourceId(),
336                                                        mySearchEntity.getLastUpdatedLow(),
337                                                        mySearchEntity.getLastUpdatedHigh());
338                                        Long count = historyBuilder.fetchCount(getRequestPartitionId());
339                                        return count.intValue();
340                                });
341
342                boolean haveOffset = mySearchEntity.getLastUpdatedLow() != null || mySearchEntity.getLastUpdatedHigh() != null;
343
344                switch (myStorageSettings.getHistoryCountMode()) {
345                        case COUNT_ACCURATE: {
346                                int count = supplier.apply(key);
347                                mySearchEntity.setTotalCount(count);
348                                break;
349                        }
350                        case CACHED_ONLY_WITHOUT_OFFSET: {
351                                if (!haveOffset) {
352                                        int count = myMemoryCacheService.get(MemoryCacheService.CacheEnum.HISTORY_COUNT, key, supplier);
353                                        mySearchEntity.setTotalCount(count);
354                                }
355                                break;
356                        }
357                        case COUNT_DISABLED: {
358                                break;
359                        }
360                }
361        }
362
363        @Override
364        public InstantDt getPublished() {
365                ensureSearchEntityLoaded();
366                return new InstantDt(mySearchEntity.getCreated());
367        }
368
369        @Nonnull
370        @Override
371        public List<IBaseResource> getResources(int theFromIndex, int theToIndex) {
372                return getResources(theFromIndex, theToIndex, new ResponsePage.ResponsePageBuilder());
373        }
374
375        @Override
376        public List<IBaseResource> getResources(
377                        int theFromIndex, int theToIndex, @Nonnull ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
378                boolean entityLoaded = ensureSearchEntityLoaded();
379                assert entityLoaded;
380                assert mySearchEntity != null;
381                assert mySearchEntity.getSearchType() != null;
382
383                switch (mySearchEntity.getSearchType()) {
384                        case HISTORY:
385                                return myTxService
386                                                .withRequest(myRequest)
387                                                .withRequestPartitionId(getRequestPartitionId())
388                                                .execute(() -> doHistoryInTransaction(mySearchEntity.getOffset(), theFromIndex, theToIndex));
389                        case SEARCH:
390                        case EVERYTHING:
391                        default:
392                                List<IBaseResource> retVal = doSearchOrEverything(theFromIndex, theToIndex, theResponsePageBuilder);
393                                /*
394                                 * If we got fewer resources back than we asked for, it's possible that the search
395                                 * completed. If that's the case, the cached version of the search entity is probably
396                                 * no longer valid so let's force a reload if it gets asked for again (most likely
397                                 * because someone is calling size() on us)
398                                 */
399                                if (retVal.size() < theToIndex - theFromIndex) {
400                                        mySearchEntity = null;
401                                }
402                                return retVal;
403                }
404        }
405
406        @Override
407        public String getUuid() {
408                return myUuid;
409        }
410
411        public SearchCacheStatusEnum getCacheStatus() {
412                return myCacheStatus;
413        }
414
415        void setCacheStatus(SearchCacheStatusEnum theSearchCacheStatusEnum) {
416                myCacheStatus = theSearchCacheStatusEnum;
417        }
418
419        @Override
420        public Integer preferredPageSize() {
421                ensureSearchEntityLoaded();
422                return mySearchEntity.getPreferredPageSize();
423        }
424
425        public void setContext(FhirContext theContext) {
426                myContext = theContext;
427        }
428
429        public void setEntityManager(EntityManager theEntityManager) {
430                myEntityManager = theEntityManager;
431        }
432
433        @VisibleForTesting
434        public void setSearchCoordinatorSvcForUnitTest(ISearchCoordinatorSvc theSearchCoordinatorSvc) {
435                mySearchCoordinatorSvc = theSearchCoordinatorSvc;
436        }
437
438        @VisibleForTesting
439        public void setTxServiceForUnitTest(HapiTransactionService theTxManager) {
440                myTxService = theTxManager;
441        }
442
443        @Override
444        public Integer size() {
445                ensureSearchEntityLoaded();
446                QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity);
447
448                Integer size = mySearchEntity.getTotalCount();
449                if (size != null) {
450                        return Math.max(0, size);
451                }
452
453                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
454                        return null;
455                } else {
456                        return mySearchCoordinatorSvc
457                                        .getSearchTotal(myUuid, myRequest, myRequestPartitionId)
458                                        .orElse(null);
459                }
460        }
461
462        protected boolean hasIncludes() {
463                ensureSearchEntityLoaded();
464                return !mySearchEntity.getIncludes().isEmpty();
465        }
466
467        // Note: Leave as protected, HSPC depends on this
468        @SuppressWarnings("WeakerAccess")
469        protected List<IBaseResource> toResourceList(
470                        ISearchBuilder theSearchBuilder,
471                        List<JpaPid> thePids,
472                        ResponsePage.ResponsePageBuilder theResponsePageBuilder) {
473                List<JpaPid> includedPidList = new ArrayList<>();
474                if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) {
475                        Integer maxIncludes = myStorageSettings.getMaximumIncludesToLoadPerPage();
476
477                        // Load non-iterate _revincludes
478                        Set<JpaPid> nonIterateRevIncludedPids = theSearchBuilder.loadIncludes(
479                                        myContext,
480                                        myEntityManager,
481                                        thePids,
482                                        mySearchEntity.toRevIncludesList(false),
483                                        true,
484                                        mySearchEntity.getLastUpdated(),
485                                        myUuid,
486                                        myRequest,
487                                        maxIncludes);
488                        if (maxIncludes != null) {
489                                maxIncludes -= nonIterateRevIncludedPids.size();
490                        }
491                        thePids.addAll(nonIterateRevIncludedPids);
492                        includedPidList.addAll(nonIterateRevIncludedPids);
493
494                        // Load non-iterate _includes
495                        Set<JpaPid> nonIterateIncludedPids = theSearchBuilder.loadIncludes(
496                                        myContext,
497                                        myEntityManager,
498                                        thePids,
499                                        mySearchEntity.toIncludesList(false),
500                                        false,
501                                        mySearchEntity.getLastUpdated(),
502                                        myUuid,
503                                        myRequest,
504                                        maxIncludes);
505                        if (maxIncludes != null) {
506                                maxIncludes -= nonIterateIncludedPids.size();
507                        }
508                        thePids.addAll(nonIterateIncludedPids);
509                        includedPidList.addAll(nonIterateIncludedPids);
510
511                        // Load iterate _revinclude
512                        Set<JpaPid> iterateRevIncludedPids = theSearchBuilder.loadIncludes(
513                                        myContext,
514                                        myEntityManager,
515                                        thePids,
516                                        mySearchEntity.toRevIncludesList(true),
517                                        true,
518                                        mySearchEntity.getLastUpdated(),
519                                        myUuid,
520                                        myRequest,
521                                        maxIncludes);
522                        if (maxIncludes != null) {
523                                maxIncludes -= iterateRevIncludedPids.size();
524                        }
525                        thePids.addAll(iterateRevIncludedPids);
526                        includedPidList.addAll(iterateRevIncludedPids);
527
528                        // Load iterate _includes
529                        Set<JpaPid> iterateIncludedPids = theSearchBuilder.loadIncludes(
530                                        myContext,
531                                        myEntityManager,
532                                        thePids,
533                                        mySearchEntity.toIncludesList(true),
534                                        false,
535                                        mySearchEntity.getLastUpdated(),
536                                        myUuid,
537                                        myRequest,
538                                        maxIncludes);
539                        thePids.addAll(iterateIncludedPids);
540                        includedPidList.addAll(iterateIncludedPids);
541                }
542
543                // Execute the query and make sure we return distinct results
544                List<IBaseResource> resources = new ArrayList<>();
545                theSearchBuilder.loadResourcesByPid(thePids, includedPidList, resources, false, myRequest);
546
547                // we will send the resource list to our interceptors
548                // this can (potentially) change the results being returned.
549                int precount = resources.size();
550                resources = ServerInterceptorUtil.fireStoragePreshowResource(resources, myRequest, myInterceptorBroadcaster);
551                // we only care about omitted results from this page
552                theResponsePageBuilder.setOmittedResourceCount(precount - resources.size());
553                theResponsePageBuilder.setResources(resources);
554                theResponsePageBuilder.setIncludedResourceCount(includedPidList.size());
555
556                return resources;
557        }
558
559        public void setInterceptorBroadcaster(IInterceptorBroadcaster theInterceptorBroadcaster) {
560                myInterceptorBroadcaster = theInterceptorBroadcaster;
561        }
562
563        @VisibleForTesting
564        public void setSearchCacheSvcForUnitTest(ISearchCacheSvc theSearchCacheSvc) {
565                mySearchCacheSvc = theSearchCacheSvc;
566        }
567
568        @VisibleForTesting
569        public void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) {
570                myDaoRegistry = theDaoRegistry;
571        }
572
573        @VisibleForTesting
574        public void setStorageSettingsForUnitTest(JpaStorageSettings theStorageSettings) {
575                myStorageSettings = theStorageSettings;
576        }
577
578        @VisibleForTesting
579        public void setSearchBuilderFactoryForUnitTest(SearchBuilderFactory theSearchBuilderFactory) {
580                mySearchBuilderFactory = theSearchBuilderFactory;
581        }
582}