Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA - Search Parameters
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.searchparam.extractor;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.RuntimeResourceDefinition;
024import ca.uhn.fhir.context.RuntimeSearchParam;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.HookParams;
027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
028import ca.uhn.fhir.interceptor.api.Pointcut;
029import ca.uhn.fhir.interceptor.model.RequestPartitionId;
030import ca.uhn.fhir.jpa.model.config.PartitionSettings;
031import ca.uhn.fhir.jpa.model.cross.IResourceLookup;
032import ca.uhn.fhir.jpa.model.dao.JpaPid;
033import ca.uhn.fhir.jpa.model.entity.BasePartitionable;
034import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam;
035import ca.uhn.fhir.jpa.model.entity.IResourceIndexComboSearchParameter;
036import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique;
037import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboTokenNonUnique;
038import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
039import ca.uhn.fhir.jpa.model.entity.ResourceLink;
040import ca.uhn.fhir.jpa.model.entity.ResourceTable;
041import ca.uhn.fhir.jpa.model.entity.SearchParamPresentEntity;
042import ca.uhn.fhir.jpa.model.entity.StorageSettings;
043import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage;
044import ca.uhn.fhir.parser.DataFormatException;
045import ca.uhn.fhir.rest.api.server.RequestDetails;
046import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
047import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
048import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
049import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
050import ca.uhn.fhir.rest.server.util.ISearchParamRegistry;
051import ca.uhn.fhir.rest.server.util.ResourceSearchParams;
052import ca.uhn.fhir.util.FhirTerser;
053import com.google.common.annotations.VisibleForTesting;
054import jakarta.annotation.Nonnull;
055import jakarta.annotation.Nullable;
056import org.apache.commons.lang3.StringUtils;
057import org.hl7.fhir.instance.model.api.IBaseReference;
058import org.hl7.fhir.instance.model.api.IBaseResource;
059import org.hl7.fhir.instance.model.api.IIdType;
060import org.hl7.fhir.r4.model.IdType;
061import org.springframework.beans.factory.annotation.Autowired;
062
063import java.util.ArrayList;
064import java.util.Collection;
065import java.util.Date;
066import java.util.HashMap;
067import java.util.HashSet;
068import java.util.List;
069import java.util.Map;
070import java.util.Optional;
071import java.util.Set;
072import java.util.stream.Collectors;
073
074import static org.apache.commons.lang3.StringUtils.isBlank;
075import static org.apache.commons.lang3.StringUtils.isNotBlank;
076
077public class SearchParamExtractorService {
078        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchParamExtractorService.class);
079
080        @Autowired
081        private ISearchParamExtractor mySearchParamExtractor;
082
083        @Autowired
084        private IInterceptorBroadcaster myInterceptorBroadcaster;
085
086        @Autowired
087        private StorageSettings myStorageSettings;
088
089        @Autowired
090        private FhirContext myContext;
091
092        @Autowired
093        private ISearchParamRegistry mySearchParamRegistry;
094
095        @Autowired
096        private PartitionSettings myPartitionSettings;
097
098        @Autowired(required = false)
099        private IResourceLinkResolver myResourceLinkResolver;
100
101        private SearchParamExtractionUtil mySearchParamExtractionUtil;
102
103        @VisibleForTesting
104        public void setSearchParamExtractor(ISearchParamExtractor theSearchParamExtractor) {
105                mySearchParamExtractor = theSearchParamExtractor;
106        }
107
108        public void extractFromResource(
109                        RequestPartitionId theRequestPartitionId,
110                        RequestDetails theRequestDetails,
111                        ResourceIndexedSearchParams theParams,
112                        ResourceTable theEntity,
113                        IBaseResource theResource,
114                        TransactionDetails theTransactionDetails,
115                        boolean theFailOnInvalidReference) {
116                extractFromResource(
117                                theRequestPartitionId,
118                                theRequestDetails,
119                                theParams,
120                                ResourceIndexedSearchParams.withSets(),
121                                theEntity,
122                                theResource,
123                                theTransactionDetails,
124                                theFailOnInvalidReference,
125                                ISearchParamExtractor.ALL_PARAMS);
126        }
127
128        /**
129         * This method is responsible for scanning a resource for all of the search parameter instances.
130         * I.e. for all search parameters defined for
131         * a given resource type, it extracts the associated indexes and populates
132         * {@literal theParams}.
133         */
134        public void extractFromResource(
135                        RequestPartitionId theRequestPartitionId,
136                        RequestDetails theRequestDetails,
137                        ResourceIndexedSearchParams theNewParams,
138                        ResourceIndexedSearchParams theExistingParams,
139                        ResourceTable theEntity,
140                        IBaseResource theResource,
141                        TransactionDetails theTransactionDetails,
142                        boolean theFailOnInvalidReference,
143                        @Nonnull ISearchParamExtractor.ISearchParamFilter theSearchParamFilter) {
144                // All search parameter types except Reference
145                ResourceIndexedSearchParams normalParams = ResourceIndexedSearchParams.withSets();
146                getExtractionUtil()
147                                .extractSearchIndexParameters(theRequestDetails, normalParams, theResource, theSearchParamFilter);
148                mergeParams(normalParams, theNewParams);
149
150                boolean indexOnContainedResources = myStorageSettings.isIndexOnContainedResources();
151                ISearchParamExtractor.SearchParamSet<PathAndRef> indexedReferences =
152                                mySearchParamExtractor.extractResourceLinks(theResource, indexOnContainedResources);
153                SearchParamExtractorService.handleWarnings(theRequestDetails, myInterceptorBroadcaster, indexedReferences);
154
155                if (indexOnContainedResources) {
156                        ResourceIndexedSearchParams containedParams = ResourceIndexedSearchParams.withSets();
157                        extractSearchIndexParametersForContainedResources(
158                                        theRequestDetails, containedParams, theResource, theEntity, indexedReferences);
159                        mergeParams(containedParams, theNewParams);
160                }
161
162                if (myStorageSettings.isIndexOnUpliftedRefchains()) {
163                        ResourceIndexedSearchParams containedParams = ResourceIndexedSearchParams.withSets();
164                        extractSearchIndexParametersForUpliftedRefchains(
165                                        theRequestDetails,
166                                        containedParams,
167                                        theEntity,
168                                        theRequestPartitionId,
169                                        theTransactionDetails,
170                                        indexedReferences);
171                        mergeParams(containedParams, theNewParams);
172                }
173
174                // Do this after, because we add to strings during both string and token processing, and contained resource if
175                // any
176                populateResourceTables(theNewParams, theEntity);
177
178                // Reference search parameters
179                extractResourceLinks(
180                                theRequestPartitionId,
181                                theExistingParams,
182                                theNewParams,
183                                theEntity,
184                                theResource,
185                                theTransactionDetails,
186                                theFailOnInvalidReference,
187                                theRequestDetails,
188                                indexedReferences);
189
190                if (indexOnContainedResources) {
191                        extractResourceLinksForContainedResources(
192                                        theRequestPartitionId,
193                                        theNewParams,
194                                        theEntity,
195                                        theResource,
196                                        theTransactionDetails,
197                                        theFailOnInvalidReference,
198                                        theRequestDetails);
199                }
200
201                // Missing (:missing) Indexes - These are indexes to satisfy the :missing
202                // modifier
203                if (myStorageSettings.getIndexMissingFields() == StorageSettings.IndexEnabledEnum.ENABLED) {
204
205                        // References
206                        Map<String, Boolean> presenceMap = getReferenceSearchParamPresenceMap(theEntity, theNewParams);
207                        presenceMap.forEach((key, value) -> {
208                                SearchParamPresentEntity present = new SearchParamPresentEntity();
209                                present.setPartitionSettings(myPartitionSettings);
210                                present.setResource(theEntity);
211                                present.setParamName(key);
212                                present.setPresent(value);
213                                present.setPartitionId(theEntity.getPartitionId());
214                                present.calculateHashes();
215                                theNewParams.mySearchParamPresentEntities.add(present);
216                        });
217
218                        // Everything else
219                        ResourceSearchParams activeSearchParams =
220                                        mySearchParamRegistry.getActiveSearchParams(theEntity.getResourceType());
221                        theNewParams.findMissingSearchParams(myPartitionSettings, myStorageSettings, theEntity, activeSearchParams);
222                }
223
224                extractSearchParamComboUnique(theEntity, theNewParams);
225
226                extractSearchParamComboNonUnique(theEntity, theNewParams);
227
228                theNewParams.setUpdatedTime(theTransactionDetails.getTransactionDate());
229        }
230
231        private SearchParamExtractionUtil getExtractionUtil() {
232                if (mySearchParamExtractionUtil == null) {
233                        mySearchParamExtractionUtil = new SearchParamExtractionUtil(
234                                        myContext, myStorageSettings, mySearchParamExtractor, myInterceptorBroadcaster);
235                }
236                return mySearchParamExtractionUtil;
237        }
238
239        @Nonnull
240        private Map<String, Boolean> getReferenceSearchParamPresenceMap(
241                        ResourceTable entity, ResourceIndexedSearchParams newParams) {
242                Map<String, Boolean> retval = new HashMap<>();
243
244                for (String nextKey : newParams.getPopulatedResourceLinkParameters()) {
245                        retval.put(nextKey, Boolean.TRUE);
246                }
247
248                ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams(entity.getResourceType());
249                activeSearchParams.getReferenceSearchParamNames().forEach(key -> retval.putIfAbsent(key, Boolean.FALSE));
250                return retval;
251        }
252
253        @VisibleForTesting
254        public void setStorageSettings(StorageSettings theStorageSettings) {
255                myStorageSettings = theStorageSettings;
256        }
257
258        /**
259         * Extract search parameter indexes for contained resources. E.g. if we
260         * are storing a Patient with a contained Organization, we might extract
261         * a String index on the Patient with paramName="organization.name" and
262         * value="Org Name"
263         */
264        private void extractSearchIndexParametersForContainedResources(
265                        RequestDetails theRequestDetails,
266                        ResourceIndexedSearchParams theParams,
267                        IBaseResource theResource,
268                        ResourceTable theEntity,
269                        ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) {
270
271                FhirTerser terser = myContext.newTerser();
272
273                // 1. get all contained resources
274                Collection<IBaseResource> containedResources = terser.getAllEmbeddedResources(theResource, false);
275
276                // Extract search parameters
277                IChainedSearchParameterExtractionStrategy strategy = new IChainedSearchParameterExtractionStrategy() {
278                        @Nonnull
279                        @Override
280                        public ISearchParamExtractor.ISearchParamFilter getSearchParamFilter(@Nonnull PathAndRef thePathAndRef) {
281                                // Currently for contained resources we always index all search parameters
282                                // on all contained resources. A potential nice future optimization would
283                                // be to make this configurable, perhaps with an optional extension you could
284                                // add to a SearchParameter?
285                                return ISearchParamExtractor.ALL_PARAMS;
286                        }
287
288                        @Override
289                        public IBaseResource fetchResourceAtPath(@Nonnull PathAndRef thePathAndRef) {
290                                if (thePathAndRef.getRef() == null) {
291                                        return null;
292                                }
293                                return findContainedResource(containedResources, thePathAndRef.getRef());
294                        }
295                };
296                boolean recurse = myStorageSettings.isIndexOnContainedResourcesRecursively();
297                extractSearchIndexParametersForTargetResources(
298                                theRequestDetails,
299                                theParams,
300                                theEntity,
301                                new HashSet<>(),
302                                strategy,
303                                theIndexedReferences,
304                                recurse,
305                                true);
306        }
307
308        /**
309         * Extract search parameter indexes for uplifted refchains. E.g. if we
310         * are storing a Patient with reference to an Organization and the
311         * "Patient:organization" SearchParameter declares an uplifted refchain
312         * on the "name" SearchParameter, we might extract a String index
313         * on the Patient with paramName="organization.name" and value="Org Name"
314         */
315        private void extractSearchIndexParametersForUpliftedRefchains(
316                        RequestDetails theRequestDetails,
317                        ResourceIndexedSearchParams theParams,
318                        ResourceTable theEntity,
319                        RequestPartitionId theRequestPartitionId,
320                        TransactionDetails theTransactionDetails,
321                        ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) {
322                IChainedSearchParameterExtractionStrategy strategy = new IChainedSearchParameterExtractionStrategy() {
323
324                        @Nonnull
325                        @Override
326                        public ISearchParamExtractor.ISearchParamFilter getSearchParamFilter(@Nonnull PathAndRef thePathAndRef) {
327                                String searchParamName = thePathAndRef.getSearchParamName();
328                                RuntimeSearchParam searchParam =
329                                                mySearchParamRegistry.getActiveSearchParam(theEntity.getResourceType(), searchParamName);
330                                Set<String> upliftRefchainCodes = searchParam.getUpliftRefchainCodes();
331                                if (upliftRefchainCodes.isEmpty()) {
332                                        return ISearchParamExtractor.NO_PARAMS;
333                                }
334                                return sp -> sp.stream()
335                                                .filter(t -> upliftRefchainCodes.contains(t.getName()))
336                                                .collect(Collectors.toList());
337                        }
338
339                        @Override
340                        public IBaseResource fetchResourceAtPath(@Nonnull PathAndRef thePathAndRef) {
341                                // The PathAndRef will contain a resource if the SP path was inside a Bundle
342                                // and pointed to a resource (e.g. Bundle.entry.resource) as opposed to
343                                // pointing to a reference (e.g. Observation.subject)
344                                if (thePathAndRef.getResource() != null) {
345                                        return thePathAndRef.getResource();
346                                }
347
348                                // Ok, it's a normal reference
349                                IIdType reference = thePathAndRef.getRef().getReferenceElement();
350
351                                // If we're processing a FHIR transaction, we store the resources
352                                // mapped by their resolved resource IDs in theTransactionDetails
353                                IBaseResource resolvedResource = theTransactionDetails.getResolvedResource(reference);
354
355                                // And the usual case is that the reference points to a resource
356                                // elsewhere in the repository, so we load it
357                                if (resolvedResource == null
358                                                && myResourceLinkResolver != null
359                                                && !reference.getValue().startsWith("urn:uuid:")) {
360                                        RequestPartitionId targetRequestPartitionId = determineResolverPartitionId(theRequestPartitionId);
361                                        resolvedResource = myResourceLinkResolver.loadTargetResource(
362                                                        targetRequestPartitionId,
363                                                        theEntity.getResourceType(),
364                                                        thePathAndRef,
365                                                        theRequestDetails,
366                                                        theTransactionDetails);
367                                        if (resolvedResource != null) {
368                                                ourLog.trace("Found target: {}", resolvedResource.getIdElement());
369                                                theTransactionDetails.addResolvedResource(
370                                                                thePathAndRef.getRef().getReferenceElement(), resolvedResource);
371                                        }
372                                }
373
374                                return resolvedResource;
375                        }
376                };
377                extractSearchIndexParametersForTargetResources(
378                                theRequestDetails, theParams, theEntity, new HashSet<>(), strategy, theIndexedReferences, false, false);
379        }
380
381        /**
382         * Extract indexes for contained references as well as for uplifted refchains.
383         * These two types of indexes are both similar special cases. Normally we handle
384         * chained searches ("Patient?organization.name=Foo") using a join from the
385         * {@link ResourceLink} table (for the "organization" part) to the
386         * {@link ResourceIndexedSearchParamString} table (for the "name" part). But
387         * for both contained resource indexes and uplifted refchains we use only the
388         * {@link ResourceIndexedSearchParamString} table to handle the entire
389         * "organization.name" part, or the other similar tables for token, number, etc.
390         *
391         * @see #extractSearchIndexParametersForContainedResources(RequestDetails, ResourceIndexedSearchParams, IBaseResource, ResourceTable, ISearchParamExtractor.SearchParamSet)
392         * @see #extractSearchIndexParametersForUpliftedRefchains(RequestDetails, ResourceIndexedSearchParams, ResourceTable, RequestPartitionId, TransactionDetails, ISearchParamExtractor.SearchParamSet)
393         */
394        private void extractSearchIndexParametersForTargetResources(
395                        RequestDetails theRequestDetails,
396                        ResourceIndexedSearchParams theParams,
397                        ResourceTable theEntity,
398                        Collection<IBaseResource> theAlreadySeenResources,
399                        IChainedSearchParameterExtractionStrategy theTargetIndexingStrategy,
400                        ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences,
401                        boolean theRecurse,
402                        boolean theIndexOnContainedResources) {
403                // 2. Find referenced search parameters
404
405                String spnamePrefix;
406                // 3. for each referenced search parameter, create an index
407                for (PathAndRef nextPathAndRef : theIndexedReferences) {
408
409                        // 3.1 get the search parameter name as spname prefix
410                        spnamePrefix = nextPathAndRef.getSearchParamName();
411
412                        if (spnamePrefix == null || (nextPathAndRef.getRef() == null && nextPathAndRef.getResource() == null))
413                                continue;
414
415                        // 3.1.2 check if this ref actually applies here
416                        ISearchParamExtractor.ISearchParamFilter searchParamsToIndex =
417                                        theTargetIndexingStrategy.getSearchParamFilter(nextPathAndRef);
418                        if (searchParamsToIndex == ISearchParamExtractor.NO_PARAMS) {
419                                continue;
420                        }
421
422                        // 3.2 find the target resource
423                        IBaseResource targetResource = theTargetIndexingStrategy.fetchResourceAtPath(nextPathAndRef);
424                        if (targetResource == null) continue;
425
426                        // 3.2.1 if we've already processed this resource upstream, do not process it again, to prevent infinite
427                        // loops
428                        if (theAlreadySeenResources.contains(targetResource)) {
429                                continue;
430                        }
431
432                        ResourceIndexedSearchParams currParams = ResourceIndexedSearchParams.withSets();
433
434                        // 3.3 create indexes for the current contained resource
435                        getExtractionUtil()
436                                        .extractSearchIndexParameters(theRequestDetails, currParams, targetResource, searchParamsToIndex);
437
438                        // 3.4 recurse to process any other contained resources referenced by this one
439                        // Recursing is currently only allowed for contained resources and not
440                        // uplifted refchains because the latter could potentially kill performance
441                        // with the number of resource resolutions needed in order to handle
442                        // a single write. Maybe in the future we could add caching to improve
443                        // this
444                        if (theRecurse) {
445                                HashSet<IBaseResource> nextAlreadySeenResources = new HashSet<>(theAlreadySeenResources);
446                                nextAlreadySeenResources.add(targetResource);
447
448                                ISearchParamExtractor.SearchParamSet<PathAndRef> indexedReferences =
449                                                mySearchParamExtractor.extractResourceLinks(targetResource, theIndexOnContainedResources);
450                                SearchParamExtractorService.handleWarnings(
451                                                theRequestDetails, myInterceptorBroadcaster, indexedReferences);
452
453                                extractSearchIndexParametersForTargetResources(
454                                                theRequestDetails,
455                                                currParams,
456                                                theEntity,
457                                                nextAlreadySeenResources,
458                                                theTargetIndexingStrategy,
459                                                indexedReferences,
460                                                true,
461                                                theIndexOnContainedResources);
462                        }
463
464                        // 3.5 added reference name as a prefix for the contained resource if any
465                        // e.g. for Observation.subject contained reference
466                        // the SP_NAME = subject.family
467                        currParams.updateSpnamePrefixForIndexOnUpliftedChain(
468                                        theEntity.getResourceType(), nextPathAndRef.getSearchParamName());
469
470                        // 3.6 merge to the mainParams
471                        // NOTE: the spname prefix is different
472                        mergeParams(currParams, theParams);
473                }
474        }
475
476        private IBaseResource findContainedResource(Collection<IBaseResource> resources, IBaseReference reference) {
477                for (IBaseResource resource : resources) {
478                        if (resource.getIdElement().equals(reference.getReferenceElement())) return resource;
479                }
480                return null;
481        }
482
483        private void mergeParams(ResourceIndexedSearchParams theSrcParams, ResourceIndexedSearchParams theTargetParams) {
484
485                theTargetParams.myNumberParams.addAll(theSrcParams.myNumberParams);
486                theTargetParams.myQuantityParams.addAll(theSrcParams.myQuantityParams);
487                theTargetParams.myQuantityNormalizedParams.addAll(theSrcParams.myQuantityNormalizedParams);
488                theTargetParams.myDateParams.addAll(theSrcParams.myDateParams);
489                theTargetParams.myUriParams.addAll(theSrcParams.myUriParams);
490                theTargetParams.myTokenParams.addAll(theSrcParams.myTokenParams);
491                theTargetParams.myStringParams.addAll(theSrcParams.myStringParams);
492                theTargetParams.myCoordsParams.addAll(theSrcParams.myCoordsParams);
493                theTargetParams.myCompositeParams.addAll(theSrcParams.myCompositeParams);
494        }
495
496        private void populateResourceTables(ResourceIndexedSearchParams theParams, ResourceTable theEntity) {
497
498                populateResourceTable(theParams.myNumberParams, theEntity);
499                populateResourceTable(theParams.myQuantityParams, theEntity);
500                populateResourceTable(theParams.myQuantityNormalizedParams, theEntity);
501                populateResourceTable(theParams.myDateParams, theEntity);
502                populateResourceTable(theParams.myUriParams, theEntity);
503                populateResourceTable(theParams.myTokenParams, theEntity);
504                populateResourceTable(theParams.myStringParams, theEntity);
505                populateResourceTable(theParams.myCoordsParams, theEntity);
506        }
507
508        @VisibleForTesting
509        public void setContext(FhirContext theContext) {
510                myContext = theContext;
511        }
512
513        private void extractResourceLinks(
514                        RequestPartitionId theRequestPartitionId,
515                        ResourceIndexedSearchParams theParams,
516                        ResourceTable theEntity,
517                        IBaseResource theResource,
518                        TransactionDetails theTransactionDetails,
519                        boolean theFailOnInvalidReference,
520                        RequestDetails theRequest,
521                        ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) {
522                extractResourceLinks(
523                                theRequestPartitionId,
524                                ResourceIndexedSearchParams.withSets(),
525                                theParams,
526                                theEntity,
527                                theResource,
528                                theTransactionDetails,
529                                theFailOnInvalidReference,
530                                theRequest,
531                                theIndexedReferences);
532        }
533
534        private void extractResourceLinks(
535                        RequestPartitionId theRequestPartitionId,
536                        ResourceIndexedSearchParams theExistingParams,
537                        ResourceIndexedSearchParams theNewParams,
538                        ResourceTable theEntity,
539                        IBaseResource theResource,
540                        TransactionDetails theTransactionDetails,
541                        boolean theFailOnInvalidReference,
542                        RequestDetails theRequest,
543                        ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) {
544                String sourceResourceName = myContext.getResourceType(theResource);
545
546                for (PathAndRef nextPathAndRef : theIndexedReferences) {
547                        if (nextPathAndRef.getRef() != null) {
548                                if (nextPathAndRef.getRef().getReferenceElement().isLocal()) {
549                                        continue;
550                                }
551
552                                RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam(
553                                                sourceResourceName, nextPathAndRef.getSearchParamName());
554                                extractResourceLinks(
555                                                theRequestPartitionId,
556                                                theExistingParams,
557                                                theNewParams,
558                                                theEntity,
559                                                theTransactionDetails,
560                                                sourceResourceName,
561                                                searchParam,
562                                                nextPathAndRef,
563                                                theFailOnInvalidReference,
564                                                theRequest);
565                        }
566                }
567
568                theEntity.setHasLinks(!theNewParams.myLinks.isEmpty());
569        }
570
571        private void extractResourceLinks(
572                        @Nonnull RequestPartitionId theRequestPartitionId,
573                        ResourceIndexedSearchParams theExistingParams,
574                        ResourceIndexedSearchParams theNewParams,
575                        ResourceTable theEntity,
576                        TransactionDetails theTransactionDetails,
577                        String theSourceResourceName,
578                        RuntimeSearchParam theRuntimeSearchParam,
579                        PathAndRef thePathAndRef,
580                        boolean theFailOnInvalidReference,
581                        RequestDetails theRequest) {
582                IBaseReference nextReference = thePathAndRef.getRef();
583                IIdType nextId = nextReference.getReferenceElement();
584                String path = thePathAndRef.getPath();
585                Date transactionDate = theTransactionDetails.getTransactionDate();
586
587                /*
588                 * This can only really happen if the DAO is being called
589                 * programmatically with a Bundle (not through the FHIR REST API)
590                 * but Smile does this
591                 */
592                if (nextId.isEmpty() && nextReference.getResource() != null) {
593                        nextId = nextReference.getResource().getIdElement();
594                }
595
596                if (myContext.getParserOptions().isStripVersionsFromReferences()
597                                && !myContext
598                                                .getParserOptions()
599                                                .getDontStripVersionsFromReferencesAtPaths()
600                                                .contains(thePathAndRef.getPath())
601                                && nextId.hasVersionIdPart()) {
602                        nextId = nextId.toVersionless();
603                }
604
605                theNewParams.myPopulatedResourceLinkParameters.add(thePathAndRef.getSearchParamName());
606
607                boolean canonical = thePathAndRef.isCanonical();
608                if (LogicalReferenceHelper.isLogicalReference(myStorageSettings, nextId) || canonical) {
609                        String value = nextId.getValue();
610                        ResourceLink resourceLink =
611                                        ResourceLink.forLogicalReference(thePathAndRef.getPath(), theEntity, value, transactionDate);
612                        if (theNewParams.myLinks.add(resourceLink)) {
613                                ourLog.debug("Indexing remote resource reference URL: {}", nextId);
614                        }
615                        return;
616                }
617
618                String baseUrl = nextId.getBaseUrl();
619
620                // If this is a conditional URL, the part after the question mark
621                // can include URLs (e.g. token system URLs) and these really confuse
622                // the IdType parser because a conditional URL isn't actually a valid
623                // FHIR ID. So in order to truly determine whether we're dealing with
624                // an absolute reference, we strip the query part and reparse
625                // the reference.
626                int questionMarkIndex = nextId.getValue().indexOf('?');
627                if (questionMarkIndex != -1) {
628                        IdType preQueryId = new IdType(nextId.getValue().substring(0, questionMarkIndex - 1));
629                        baseUrl = preQueryId.getBaseUrl();
630                }
631
632                String typeString = nextId.getResourceType();
633                if (isBlank(typeString)) {
634                        String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource type - "
635                                        + nextId.getValue();
636                        if (theFailOnInvalidReference) {
637                                throw new InvalidRequestException(Msg.code(505) + msg);
638                        } else {
639                                ourLog.debug(msg);
640                                return;
641                        }
642                }
643                RuntimeResourceDefinition resourceDefinition;
644                try {
645                        resourceDefinition = myContext.getResourceDefinition(typeString);
646                } catch (DataFormatException e) {
647                        String msg = "Invalid resource reference found at path[" + path
648                                        + "] - Resource type is unknown or not supported on this server - " + nextId.getValue();
649                        if (theFailOnInvalidReference) {
650                                throw new InvalidRequestException(Msg.code(506) + msg);
651                        } else {
652                                ourLog.debug(msg);
653                                return;
654                        }
655                }
656
657                if (theRuntimeSearchParam.hasTargets()) {
658                        if (!theRuntimeSearchParam.getTargets().contains(typeString)) {
659                                return;
660                        }
661                }
662
663                if (isNotBlank(baseUrl)) {
664                        if (!myStorageSettings.getTreatBaseUrlsAsLocal().contains(baseUrl)
665                                        && !myStorageSettings.isAllowExternalReferences()) {
666                                String msg = myContext
667                                                .getLocalizer()
668                                                .getMessage(BaseSearchParamExtractor.class, "externalReferenceNotAllowed", nextId.getValue());
669                                throw new InvalidRequestException(Msg.code(507) + msg);
670                        } else {
671                                ResourceLink resourceLink =
672                                                ResourceLink.forAbsoluteReference(thePathAndRef.getPath(), theEntity, nextId, transactionDate);
673                                if (theNewParams.myLinks.add(resourceLink)) {
674                                        ourLog.debug("Indexing remote resource reference URL: {}", nextId);
675                                }
676                                return;
677                        }
678                }
679
680                Class<? extends IBaseResource> type = resourceDefinition.getImplementingClass();
681                String targetId = nextId.getIdPart();
682                if (StringUtils.isBlank(targetId)) {
683                        String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource ID - "
684                                        + nextId.getValue();
685                        if (theFailOnInvalidReference) {
686                                throw new InvalidRequestException(Msg.code(508) + msg);
687                        } else {
688                                ourLog.debug(msg);
689                                return;
690                        }
691                }
692
693                IIdType referenceElement = thePathAndRef.getRef().getReferenceElement();
694                JpaPid resolvedTargetId = (JpaPid) theTransactionDetails.getResolvedResourceId(referenceElement);
695                ResourceLink resourceLink;
696
697                Long targetVersionId = nextId.getVersionIdPartAsLong();
698                if (resolvedTargetId != null) {
699
700                        /*
701                         * If we have already resolved the given reference within this transaction, we don't
702                         * need to resolve it again
703                         */
704                        myResourceLinkResolver.validateTypeOrThrowException(type);
705                        resourceLink = ResourceLink.forLocalReference(
706                                        thePathAndRef.getPath(),
707                                        theEntity,
708                                        typeString,
709                                        resolvedTargetId.getId(),
710                                        targetId,
711                                        transactionDate,
712                                        targetVersionId);
713
714                } else if (theFailOnInvalidReference) {
715
716                        /*
717                         * The reference points to another resource, so let's look it up. We need to do this
718                         * since the target may be a forced ID, but also so that we can throw an exception
719                         * if the reference is invalid
720                         */
721                        myResourceLinkResolver.validateTypeOrThrowException(type);
722
723                        /*
724                         * We need to obtain a resourceLink out of the provided {@literal thePathAndRef}.  In the case
725                         * where we are updating a resource that already has resourceLinks (stored in {@literal theExistingParams.getResourceLinks()}),
726                         * let's try to match thePathAndRef to an already existing resourceLink to avoid the
727                         * very expensive operation of creating a resourceLink that would end up being exactly the same
728                         * one we already have.
729                         */
730                        Optional<ResourceLink> optionalResourceLink =
731                                        findMatchingResourceLink(thePathAndRef, theExistingParams.getResourceLinks());
732                        if (optionalResourceLink.isPresent()) {
733                                resourceLink = optionalResourceLink.get();
734                        } else {
735                                resourceLink = resolveTargetAndCreateResourceLinkOrReturnNull(
736                                                theRequestPartitionId,
737                                                theSourceResourceName,
738                                                thePathAndRef,
739                                                theEntity,
740                                                transactionDate,
741                                                nextId,
742                                                theRequest,
743                                                theTransactionDetails);
744                        }
745
746                        if (resourceLink == null) {
747                                return;
748                        } else {
749                                // Cache the outcome in the current transaction in case there are more references
750                                JpaPid persistentId = JpaPid.fromId(resourceLink.getTargetResourcePid());
751                                theTransactionDetails.addResolvedResourceId(referenceElement, persistentId);
752                        }
753
754                } else {
755
756                        /*
757                         * Just assume the reference is valid. This is used for in-memory matching since there
758                         * is no expectation of a database in this situation
759                         */
760                        ResourceTable target;
761                        target = new ResourceTable();
762                        target.setResourceType(typeString);
763                        resourceLink = ResourceLink.forLocalReference(
764                                        thePathAndRef.getPath(), theEntity, typeString, null, targetId, transactionDate, targetVersionId);
765                }
766
767                theNewParams.myLinks.add(resourceLink);
768        }
769
770        private Optional<ResourceLink> findMatchingResourceLink(
771                        PathAndRef thePathAndRef, Collection<ResourceLink> theResourceLinks) {
772                IIdType referenceElement = thePathAndRef.getRef().getReferenceElement();
773                List<ResourceLink> resourceLinks = new ArrayList<>(theResourceLinks);
774                for (ResourceLink resourceLink : resourceLinks) {
775
776                        // comparing the searchParam path ex: Group.member.entity
777                        boolean hasMatchingSearchParamPath =
778                                        StringUtils.equals(resourceLink.getSourcePath(), thePathAndRef.getPath());
779
780                        boolean hasMatchingResourceType =
781                                        StringUtils.equals(resourceLink.getTargetResourceType(), referenceElement.getResourceType());
782
783                        boolean hasMatchingResourceId =
784                                        StringUtils.equals(resourceLink.getTargetResourceId(), referenceElement.getIdPart());
785
786                        boolean hasMatchingResourceVersion = myContext.getParserOptions().isStripVersionsFromReferences()
787                                        || referenceElement.getVersionIdPartAsLong() == null
788                                        || referenceElement.getVersionIdPartAsLong().equals(resourceLink.getTargetResourceVersion());
789
790                        if (hasMatchingSearchParamPath
791                                        && hasMatchingResourceType
792                                        && hasMatchingResourceId
793                                        && hasMatchingResourceVersion) {
794                                return Optional.of(resourceLink);
795                        }
796                }
797
798                return Optional.empty();
799        }
800
801        private void extractResourceLinksForContainedResources(
802                        RequestPartitionId theRequestPartitionId,
803                        ResourceIndexedSearchParams theParams,
804                        ResourceTable theEntity,
805                        IBaseResource theResource,
806                        TransactionDetails theTransactionDetails,
807                        boolean theFailOnInvalidReference,
808                        RequestDetails theRequest) {
809
810                FhirTerser terser = myContext.newTerser();
811
812                // 1. get all contained resources
813                Collection<IBaseResource> containedResources = terser.getAllEmbeddedResources(theResource, false);
814
815                extractResourceLinksForContainedResources(
816                                theRequestPartitionId,
817                                theParams,
818                                theEntity,
819                                theResource,
820                                theTransactionDetails,
821                                theFailOnInvalidReference,
822                                theRequest,
823                                containedResources,
824                                new HashSet<>());
825        }
826
827        private void extractResourceLinksForContainedResources(
828                        RequestPartitionId theRequestPartitionId,
829                        ResourceIndexedSearchParams theParams,
830                        ResourceTable theEntity,
831                        IBaseResource theResource,
832                        TransactionDetails theTransactionDetails,
833                        boolean theFailOnInvalidReference,
834                        RequestDetails theRequest,
835                        Collection<IBaseResource> theContainedResources,
836                        Collection<IBaseResource> theAlreadySeenResources) {
837
838                // 2. Find referenced search parameters
839                ISearchParamExtractor.SearchParamSet<PathAndRef> referencedSearchParamSet =
840                                mySearchParamExtractor.extractResourceLinks(theResource, true);
841
842                String spNamePrefix;
843                ResourceIndexedSearchParams currParams;
844                // 3. for each referenced search parameter, create an index
845                for (PathAndRef nextPathAndRef : referencedSearchParamSet) {
846
847                        // 3.1 get the search parameter name as spname prefix
848                        spNamePrefix = nextPathAndRef.getSearchParamName();
849
850                        if (spNamePrefix == null || nextPathAndRef.getRef() == null) continue;
851
852                        // 3.2 find the contained resource
853                        IBaseResource containedResource = findContainedResource(theContainedResources, nextPathAndRef.getRef());
854                        if (containedResource == null) continue;
855
856                        // 3.2.1 if we've already processed this resource upstream, do not process it again, to prevent infinite
857                        // loops
858                        if (theAlreadySeenResources.contains(containedResource)) {
859                                continue;
860                        }
861
862                        currParams = ResourceIndexedSearchParams.withSets();
863
864                        // 3.3 create indexes for the current contained resource
865                        ISearchParamExtractor.SearchParamSet<PathAndRef> indexedReferences =
866                                        mySearchParamExtractor.extractResourceLinks(containedResource, true);
867                        extractResourceLinks(
868                                        theRequestPartitionId,
869                                        currParams,
870                                        theEntity,
871                                        containedResource,
872                                        theTransactionDetails,
873                                        theFailOnInvalidReference,
874                                        theRequest,
875                                        indexedReferences);
876
877                        // 3.4 recurse to process any other contained resources referenced by this one
878                        if (myStorageSettings.isIndexOnContainedResourcesRecursively()) {
879                                HashSet<IBaseResource> nextAlreadySeenResources = new HashSet<>(theAlreadySeenResources);
880                                nextAlreadySeenResources.add(containedResource);
881                                extractResourceLinksForContainedResources(
882                                                theRequestPartitionId,
883                                                currParams,
884                                                theEntity,
885                                                containedResource,
886                                                theTransactionDetails,
887                                                theFailOnInvalidReference,
888                                                theRequest,
889                                                theContainedResources,
890                                                nextAlreadySeenResources);
891                        }
892
893                        // 3.4 added reference name as a prefix for the contained resource if any
894                        // e.g. for Observation.subject contained reference
895                        // the SP_NAME = subject.family
896                        currParams.updateSpnamePrefixForLinksOnContainedResource(nextPathAndRef.getPath());
897
898                        // 3.5 merge to the mainParams
899                        // NOTE: the spname prefix is different
900                        theParams.getResourceLinks().addAll(currParams.getResourceLinks());
901                }
902        }
903
904        @SuppressWarnings("unchecked")
905        private ResourceLink resolveTargetAndCreateResourceLinkOrReturnNull(
906                        @Nonnull RequestPartitionId theRequestPartitionId,
907                        String theSourceResourceName,
908                        PathAndRef thePathAndRef,
909                        ResourceTable theEntity,
910                        Date theUpdateTime,
911                        IIdType theNextId,
912                        RequestDetails theRequest,
913                        TransactionDetails theTransactionDetails) {
914                JpaPid resolvedResourceId = (JpaPid) theTransactionDetails.getResolvedResourceId(theNextId);
915                if (resolvedResourceId != null) {
916                        String targetResourceType = theNextId.getResourceType();
917                        Long targetResourcePid = resolvedResourceId.getId();
918                        String targetResourceIdPart = theNextId.getIdPart();
919                        Long targetVersion = theNextId.getVersionIdPartAsLong();
920                        return ResourceLink.forLocalReference(
921                                        thePathAndRef.getPath(),
922                                        theEntity,
923                                        targetResourceType,
924                                        targetResourcePid,
925                                        targetResourceIdPart,
926                                        theUpdateTime,
927                                        targetVersion);
928                }
929
930                /*
931                 * We keep a cache of resolved target resources. This is good since for some resource types, there
932                 * are multiple search parameters that map to the same element path within a resource (e.g.
933                 * Observation:patient and Observation.subject and we don't want to force a resolution of the
934                 * target any more times than we have to.
935                 */
936
937                IResourceLookup<JpaPid> targetResource;
938                if (myPartitionSettings.isPartitioningEnabled()) {
939                        if (myPartitionSettings.getAllowReferencesAcrossPartitions()
940                                        == PartitionSettings.CrossPartitionReferenceMode.ALLOWED_UNQUALIFIED) {
941
942                                // Interceptor: Pointcut.JPA_CROSS_PARTITION_REFERENCE_DETECTED
943                                if (CompositeInterceptorBroadcaster.hasHooks(
944                                                Pointcut.JPA_RESOLVE_CROSS_PARTITION_REFERENCE, myInterceptorBroadcaster, theRequest)) {
945                                        CrossPartitionReferenceDetails referenceDetails = new CrossPartitionReferenceDetails(
946                                                        theRequestPartitionId,
947                                                        theSourceResourceName,
948                                                        thePathAndRef,
949                                                        theRequest,
950                                                        theTransactionDetails);
951                                        HookParams params = new HookParams(referenceDetails);
952                                        targetResource =
953                                                        (IResourceLookup<JpaPid>) CompositeInterceptorBroadcaster.doCallHooksAndReturnObject(
954                                                                        myInterceptorBroadcaster,
955                                                                        theRequest,
956                                                                        Pointcut.JPA_RESOLVE_CROSS_PARTITION_REFERENCE,
957                                                                        params);
958                                } else {
959                                        targetResource = myResourceLinkResolver.findTargetResource(
960                                                        RequestPartitionId.allPartitions(),
961                                                        theSourceResourceName,
962                                                        thePathAndRef,
963                                                        theRequest,
964                                                        theTransactionDetails);
965                                }
966
967                        } else {
968                                targetResource = myResourceLinkResolver.findTargetResource(
969                                                theRequestPartitionId, theSourceResourceName, thePathAndRef, theRequest, theTransactionDetails);
970                        }
971                } else {
972                        targetResource = myResourceLinkResolver.findTargetResource(
973                                        theRequestPartitionId, theSourceResourceName, thePathAndRef, theRequest, theTransactionDetails);
974                }
975
976                if (targetResource == null) {
977                        return null;
978                }
979
980                String targetResourceType = targetResource.getResourceType();
981                Long targetResourcePid = targetResource.getPersistentId().getId();
982                String targetResourceIdPart = theNextId.getIdPart();
983                Long targetVersion = theNextId.getVersionIdPartAsLong();
984                return ResourceLink.forLocalReference(
985                                thePathAndRef.getPath(),
986                                theEntity,
987                                targetResourceType,
988                                targetResourcePid,
989                                targetResourceIdPart,
990                                theUpdateTime,
991                                targetVersion);
992        }
993
994        private RequestPartitionId determineResolverPartitionId(@Nonnull RequestPartitionId theRequestPartitionId) {
995                RequestPartitionId targetRequestPartitionId = theRequestPartitionId;
996                if (myPartitionSettings.isPartitioningEnabled()
997                                && myPartitionSettings.getAllowReferencesAcrossPartitions()
998                                                == PartitionSettings.CrossPartitionReferenceMode.ALLOWED_UNQUALIFIED) {
999                        targetRequestPartitionId = RequestPartitionId.allPartitions();
1000                }
1001                return targetRequestPartitionId;
1002        }
1003
1004        private void populateResourceTable(
1005                        Collection<? extends BaseResourceIndexedSearchParam> theParams, ResourceTable theResourceTable) {
1006                for (BaseResourceIndexedSearchParam next : theParams) {
1007                        if (next.getResourcePid() == null) {
1008                                next.setResource(theResourceTable);
1009                        }
1010                }
1011        }
1012
1013        private void populateResourceTableForComboParams(
1014                        Collection<? extends IResourceIndexComboSearchParameter> theParams, ResourceTable theResourceTable) {
1015                for (IResourceIndexComboSearchParameter next : theParams) {
1016                        if (next.getResource() == null) {
1017                                next.setResource(theResourceTable);
1018                                if (next instanceof BasePartitionable) {
1019                                        ((BasePartitionable) next).setPartitionId(theResourceTable.getPartitionId());
1020                                }
1021                        }
1022                }
1023        }
1024
1025        @VisibleForTesting
1026        void setInterceptorBroadcasterForUnitTest(IInterceptorBroadcaster theInterceptorBroadcaster) {
1027                myInterceptorBroadcaster = theInterceptorBroadcaster;
1028        }
1029
1030        @Nonnull
1031        public List<String> extractParamValuesAsStrings(
1032                        RuntimeSearchParam theActiveSearchParam, IBaseResource theResource) {
1033                return mySearchParamExtractor.extractParamValuesAsStrings(theActiveSearchParam, theResource);
1034        }
1035
1036        public void extractSearchParamComboUnique(ResourceTable theEntity, ResourceIndexedSearchParams theParams) {
1037                String resourceType = theEntity.getResourceType();
1038                Set<ResourceIndexedComboStringUnique> comboUniques =
1039                                mySearchParamExtractor.extractSearchParamComboUnique(resourceType, theParams);
1040                theParams.myComboStringUniques.addAll(comboUniques);
1041                populateResourceTableForComboParams(theParams.myComboStringUniques, theEntity);
1042        }
1043
1044        public void extractSearchParamComboNonUnique(ResourceTable theEntity, ResourceIndexedSearchParams theParams) {
1045                String resourceType = theEntity.getResourceType();
1046                Set<ResourceIndexedComboTokenNonUnique> comboNonUniques =
1047                                mySearchParamExtractor.extractSearchParamComboNonUnique(resourceType, theParams);
1048                theParams.myComboTokenNonUnique.addAll(comboNonUniques);
1049                populateResourceTableForComboParams(theParams.myComboTokenNonUnique, theEntity);
1050        }
1051
1052        /**
1053         * This interface is used by {@link #extractSearchIndexParametersForTargetResources(RequestDetails, ResourceIndexedSearchParams, ResourceTable, Collection, IChainedSearchParameterExtractionStrategy, ISearchParamExtractor.SearchParamSet, boolean, boolean)}
1054         * in order to use that method for extracting chained search parameter indexes both
1055         * from contained resources and from uplifted refchains.
1056         */
1057        private interface IChainedSearchParameterExtractionStrategy {
1058
1059                /**
1060                 * Which search parameters should be indexed for the resource target
1061                 * at the given path. In other words if thePathAndRef contains
1062                 * "Patient/123", then we could return a filter that only lets the
1063                 * "name" and "gender" search params through  if we only want those
1064                 * two parameters to be indexed for the resolved Patient resource
1065                 * with that ID.
1066                 */
1067                @Nonnull
1068                ISearchParamExtractor.ISearchParamFilter getSearchParamFilter(@Nonnull PathAndRef thePathAndRef);
1069
1070                /**
1071                 * Actually fetch the resource at the given path, or return
1072                 * {@literal null} if none can be found.
1073                 */
1074                @Nullable
1075                IBaseResource fetchResourceAtPath(@Nonnull PathAndRef thePathAndRef);
1076        }
1077
1078        static void handleWarnings(
1079                        RequestDetails theRequestDetails,
1080                        IInterceptorBroadcaster theInterceptorBroadcaster,
1081                        ISearchParamExtractor.SearchParamSet<?> theSearchParamSet) {
1082                if (theSearchParamSet.getWarnings().isEmpty()) {
1083                        return;
1084                }
1085
1086                // If extraction generated any warnings, broadcast an error
1087                if (CompositeInterceptorBroadcaster.hasHooks(
1088                                Pointcut.JPA_PERFTRACE_WARNING, theInterceptorBroadcaster, theRequestDetails)) {
1089                        for (String next : theSearchParamSet.getWarnings()) {
1090                                StorageProcessingMessage messageHolder = new StorageProcessingMessage();
1091                                messageHolder.setMessage(next);
1092                                HookParams params = new HookParams()
1093                                                .add(RequestDetails.class, theRequestDetails)
1094                                                .addIfMatchesType(ServletRequestDetails.class, theRequestDetails)
1095                                                .add(StorageProcessingMessage.class, messageHolder);
1096                                CompositeInterceptorBroadcaster.doCallHooks(
1097                                                theInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_WARNING, params);
1098                        }
1099                }
1100        }
1101}