001/*- 002 * #%L 003 * HAPI FHIR JPA - Search Parameters 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.searchparam.extractor; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.context.RuntimeResourceDefinition; 024import ca.uhn.fhir.context.RuntimeSearchParam; 025import ca.uhn.fhir.i18n.Msg; 026import ca.uhn.fhir.interceptor.api.HookParams; 027import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 028import ca.uhn.fhir.interceptor.api.Pointcut; 029import ca.uhn.fhir.interceptor.model.RequestPartitionId; 030import ca.uhn.fhir.jpa.model.config.PartitionSettings; 031import ca.uhn.fhir.jpa.model.cross.IResourceLookup; 032import ca.uhn.fhir.jpa.model.dao.JpaPid; 033import ca.uhn.fhir.jpa.model.entity.BasePartitionable; 034import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; 035import ca.uhn.fhir.jpa.model.entity.IResourceIndexComboSearchParameter; 036import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique; 037import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboTokenNonUnique; 038import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString; 039import ca.uhn.fhir.jpa.model.entity.ResourceLink; 040import ca.uhn.fhir.jpa.model.entity.ResourceLink.ResourceLinkForLocalReferenceParams; 041import ca.uhn.fhir.jpa.model.entity.ResourceTable; 042import ca.uhn.fhir.jpa.model.entity.SearchParamPresentEntity; 043import ca.uhn.fhir.jpa.model.entity.StorageSettings; 044import ca.uhn.fhir.jpa.model.search.StorageProcessingMessage; 045import ca.uhn.fhir.parser.DataFormatException; 046import ca.uhn.fhir.rest.api.server.RequestDetails; 047import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; 048import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 049import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 050import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 051import ca.uhn.fhir.rest.server.util.ISearchParamRegistry; 052import ca.uhn.fhir.rest.server.util.ResourceSearchParams; 053import ca.uhn.fhir.util.FhirTerser; 054import com.google.common.annotations.VisibleForTesting; 055import jakarta.annotation.Nonnull; 056import jakarta.annotation.Nullable; 057import org.apache.commons.lang3.StringUtils; 058import org.hl7.fhir.instance.model.api.IBaseReference; 059import org.hl7.fhir.instance.model.api.IBaseResource; 060import org.hl7.fhir.instance.model.api.IIdType; 061import org.hl7.fhir.r4.model.IdType; 062import org.springframework.beans.factory.annotation.Autowired; 063 064import java.util.ArrayList; 065import java.util.Collection; 066import java.util.Date; 067import java.util.HashMap; 068import java.util.HashSet; 069import java.util.List; 070import java.util.Map; 071import java.util.Optional; 072import java.util.Set; 073import java.util.stream.Collectors; 074 075import static ca.uhn.fhir.jpa.model.config.PartitionSettings.CrossPartitionReferenceMode.ALLOWED_UNQUALIFIED; 076import static ca.uhn.fhir.jpa.model.entity.ResourceLink.forLocalReference; 077import static org.apache.commons.lang3.StringUtils.isBlank; 078import static org.apache.commons.lang3.StringUtils.isNotBlank; 079 080public class SearchParamExtractorService { 081 private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(SearchParamExtractorService.class); 082 083 @Autowired 084 private ISearchParamExtractor mySearchParamExtractor; 085 086 @Autowired 087 private IInterceptorBroadcaster myInterceptorBroadcaster; 088 089 @Autowired 090 private StorageSettings myStorageSettings; 091 092 @Autowired 093 private FhirContext myContext; 094 095 @Autowired 096 private ISearchParamRegistry mySearchParamRegistry; 097 098 @Autowired 099 private PartitionSettings myPartitionSettings; 100 101 @Autowired(required = false) 102 private IResourceLinkResolver myResourceLinkResolver; 103 104 private SearchParamExtractionUtil mySearchParamExtractionUtil; 105 106 @VisibleForTesting 107 public void setSearchParamExtractor(ISearchParamExtractor theSearchParamExtractor) { 108 mySearchParamExtractor = theSearchParamExtractor; 109 } 110 111 /** 112 * This method is responsible for scanning a resource for all of the search parameter instances. 113 * I.e. for all search parameters defined for 114 * a given resource type, it extracts the associated indexes and populates 115 * {@literal theParams}. 116 */ 117 public void extractFromResource( 118 RequestPartitionId theRequestPartitionId, 119 RequestDetails theRequestDetails, 120 ResourceIndexedSearchParams theNewParams, 121 ResourceIndexedSearchParams theExistingParams, 122 ResourceTable theEntity, 123 IBaseResource theResource, 124 TransactionDetails theTransactionDetails, 125 boolean theFailOnInvalidReference, 126 @Nonnull ISearchParamExtractor.ISearchParamFilter theSearchParamFilter) { 127 // All search parameter types except Reference 128 ResourceIndexedSearchParams normalParams = ResourceIndexedSearchParams.withSets(); 129 getExtractionUtil() 130 .extractSearchIndexParameters(theRequestDetails, normalParams, theResource, theSearchParamFilter); 131 mergeParams(normalParams, theNewParams); 132 133 boolean indexOnContainedResources = myStorageSettings.isIndexOnContainedResources(); 134 ISearchParamExtractor.SearchParamSet<PathAndRef> indexedReferences = 135 mySearchParamExtractor.extractResourceLinks(theResource, indexOnContainedResources); 136 SearchParamExtractorService.handleWarnings(theRequestDetails, myInterceptorBroadcaster, indexedReferences); 137 138 if (indexOnContainedResources) { 139 ResourceIndexedSearchParams containedParams = ResourceIndexedSearchParams.withSets(); 140 extractSearchIndexParametersForContainedResources( 141 theRequestDetails, containedParams, theResource, theEntity, indexedReferences); 142 mergeParams(containedParams, theNewParams); 143 } 144 145 if (myStorageSettings.isIndexOnUpliftedRefchains()) { 146 ResourceIndexedSearchParams containedParams = ResourceIndexedSearchParams.withSets(); 147 extractSearchIndexParametersForUpliftedRefchains( 148 theRequestDetails, 149 containedParams, 150 theEntity, 151 theRequestPartitionId, 152 theTransactionDetails, 153 indexedReferences); 154 mergeParams(containedParams, theNewParams); 155 } 156 157 // Do this after, because we add to strings during both string and token processing, and contained resource if 158 // any 159 populateResourceTables(theNewParams, theEntity); 160 161 // Reference search parameters 162 extractResourceLinks( 163 theRequestPartitionId, 164 theExistingParams, 165 theNewParams, 166 theEntity, 167 theResource, 168 theTransactionDetails, 169 theFailOnInvalidReference, 170 theRequestDetails, 171 indexedReferences); 172 173 if (indexOnContainedResources) { 174 extractResourceLinksForContainedResources( 175 theRequestPartitionId, 176 theNewParams, 177 theEntity, 178 theResource, 179 theTransactionDetails, 180 theFailOnInvalidReference, 181 theRequestDetails); 182 } 183 184 // Missing (:missing) Indexes - These are indexes to satisfy the :missing 185 // modifier 186 if (myStorageSettings.getIndexMissingFields() == StorageSettings.IndexEnabledEnum.ENABLED) { 187 188 // References 189 Map<String, Boolean> presenceMap = getReferenceSearchParamPresenceMap(theEntity, theNewParams); 190 presenceMap.forEach((key, value) -> { 191 SearchParamPresentEntity present = new SearchParamPresentEntity(); 192 present.setPartitionSettings(myPartitionSettings); 193 present.setResource(theEntity); 194 present.setParamName(key); 195 present.setPresent(value); 196 present.setPartitionId(theEntity.getPartitionId()); 197 present.calculateHashes(); 198 theNewParams.mySearchParamPresentEntities.add(present); 199 }); 200 201 // Everything else 202 ResourceSearchParams activeSearchParams = 203 mySearchParamRegistry.getActiveSearchParams(theEntity.getResourceType()); 204 theNewParams.findMissingSearchParams(myPartitionSettings, myStorageSettings, theEntity, activeSearchParams); 205 } 206 207 extractSearchParamComboUnique(theEntity, theNewParams); 208 209 extractSearchParamComboNonUnique(theEntity, theNewParams); 210 211 theNewParams.setUpdatedTime(theTransactionDetails.getTransactionDate()); 212 } 213 214 private SearchParamExtractionUtil getExtractionUtil() { 215 if (mySearchParamExtractionUtil == null) { 216 mySearchParamExtractionUtil = new SearchParamExtractionUtil( 217 myContext, myStorageSettings, mySearchParamExtractor, myInterceptorBroadcaster); 218 } 219 return mySearchParamExtractionUtil; 220 } 221 222 @Nonnull 223 private Map<String, Boolean> getReferenceSearchParamPresenceMap( 224 ResourceTable entity, ResourceIndexedSearchParams newParams) { 225 Map<String, Boolean> retval = new HashMap<>(); 226 227 for (String nextKey : newParams.getPopulatedResourceLinkParameters()) { 228 retval.put(nextKey, Boolean.TRUE); 229 } 230 231 ResourceSearchParams activeSearchParams = mySearchParamRegistry.getActiveSearchParams(entity.getResourceType()); 232 activeSearchParams.getReferenceSearchParamNames().forEach(key -> retval.putIfAbsent(key, Boolean.FALSE)); 233 return retval; 234 } 235 236 @VisibleForTesting 237 public void setStorageSettings(StorageSettings theStorageSettings) { 238 myStorageSettings = theStorageSettings; 239 } 240 241 /** 242 * Extract search parameter indexes for contained resources. E.g. if we 243 * are storing a Patient with a contained Organization, we might extract 244 * a String index on the Patient with paramName="organization.name" and 245 * value="Org Name" 246 */ 247 private void extractSearchIndexParametersForContainedResources( 248 RequestDetails theRequestDetails, 249 ResourceIndexedSearchParams theParams, 250 IBaseResource theResource, 251 ResourceTable theEntity, 252 ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) { 253 254 FhirTerser terser = myContext.newTerser(); 255 256 // 1. get all contained resources 257 Collection<IBaseResource> containedResources = terser.getAllEmbeddedResources(theResource, false); 258 259 // Extract search parameters 260 IChainedSearchParameterExtractionStrategy strategy = new IChainedSearchParameterExtractionStrategy() { 261 @Nonnull 262 @Override 263 public ISearchParamExtractor.ISearchParamFilter getSearchParamFilter(@Nonnull PathAndRef thePathAndRef) { 264 // Currently for contained resources we always index all search parameters 265 // on all contained resources. A potential nice future optimization would 266 // be to make this configurable, perhaps with an optional extension you could 267 // add to a SearchParameter? 268 return ISearchParamExtractor.ALL_PARAMS; 269 } 270 271 @Override 272 public IBaseResource fetchResourceAtPath(@Nonnull PathAndRef thePathAndRef) { 273 if (thePathAndRef.getRef() == null) { 274 return null; 275 } 276 return findContainedResource(containedResources, thePathAndRef.getRef()); 277 } 278 }; 279 boolean recurse = myStorageSettings.isIndexOnContainedResourcesRecursively(); 280 extractSearchIndexParametersForTargetResources( 281 theRequestDetails, 282 theParams, 283 theEntity, 284 new HashSet<>(), 285 strategy, 286 theIndexedReferences, 287 recurse, 288 true); 289 } 290 291 /** 292 * Extract search parameter indexes for uplifted refchains. E.g. if we 293 * are storing a Patient with reference to an Organization and the 294 * "Patient:organization" SearchParameter declares an uplifted refchain 295 * on the "name" SearchParameter, we might extract a String index 296 * on the Patient with paramName="organization.name" and value="Org Name" 297 */ 298 private void extractSearchIndexParametersForUpliftedRefchains( 299 RequestDetails theRequestDetails, 300 ResourceIndexedSearchParams theParams, 301 ResourceTable theEntity, 302 RequestPartitionId theRequestPartitionId, 303 TransactionDetails theTransactionDetails, 304 ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) { 305 IChainedSearchParameterExtractionStrategy strategy = new IChainedSearchParameterExtractionStrategy() { 306 307 @Nonnull 308 @Override 309 public ISearchParamExtractor.ISearchParamFilter getSearchParamFilter(@Nonnull PathAndRef thePathAndRef) { 310 String searchParamName = thePathAndRef.getSearchParamName(); 311 RuntimeSearchParam searchParam = 312 mySearchParamRegistry.getActiveSearchParam(theEntity.getResourceType(), searchParamName); 313 Set<String> upliftRefchainCodes = searchParam.getUpliftRefchainCodes(); 314 if (upliftRefchainCodes.isEmpty()) { 315 return ISearchParamExtractor.NO_PARAMS; 316 } 317 return sp -> sp.stream() 318 .filter(t -> upliftRefchainCodes.contains(t.getName())) 319 .collect(Collectors.toList()); 320 } 321 322 @Override 323 public IBaseResource fetchResourceAtPath(@Nonnull PathAndRef thePathAndRef) { 324 // The PathAndRef will contain a resource if the SP path was inside a Bundle 325 // and pointed to a resource (e.g. Bundle.entry.resource) as opposed to 326 // pointing to a reference (e.g. Observation.subject) 327 if (thePathAndRef.getResource() != null) { 328 return thePathAndRef.getResource(); 329 } 330 331 // Ok, it's a normal reference 332 IIdType reference = thePathAndRef.getRef().getReferenceElement(); 333 334 // If we're processing a FHIR transaction, we store the resources 335 // mapped by their resolved resource IDs in theTransactionDetails 336 IBaseResource resolvedResource = theTransactionDetails.getResolvedResource(reference); 337 338 // And the usual case is that the reference points to a resource 339 // elsewhere in the repository, so we load it 340 if (resolvedResource == null 341 && myResourceLinkResolver != null 342 && !reference.getValue().startsWith("urn:uuid:")) { 343 RequestPartitionId targetRequestPartitionId = determineResolverPartitionId(theRequestPartitionId); 344 resolvedResource = myResourceLinkResolver.loadTargetResource( 345 targetRequestPartitionId, 346 theEntity.getResourceType(), 347 thePathAndRef, 348 theRequestDetails, 349 theTransactionDetails); 350 if (resolvedResource != null) { 351 ourLog.trace("Found target: {}", resolvedResource.getIdElement()); 352 theTransactionDetails.addResolvedResource( 353 thePathAndRef.getRef().getReferenceElement(), resolvedResource); 354 } 355 } 356 357 return resolvedResource; 358 } 359 }; 360 extractSearchIndexParametersForTargetResources( 361 theRequestDetails, theParams, theEntity, new HashSet<>(), strategy, theIndexedReferences, false, false); 362 } 363 364 /** 365 * Extract indexes for contained references as well as for uplifted refchains. 366 * These two types of indexes are both similar special cases. Normally we handle 367 * chained searches ("Patient?organization.name=Foo") using a join from the 368 * {@link ResourceLink} table (for the "organization" part) to the 369 * {@link ResourceIndexedSearchParamString} table (for the "name" part). But 370 * for both contained resource indexes and uplifted refchains we use only the 371 * {@link ResourceIndexedSearchParamString} table to handle the entire 372 * "organization.name" part, or the other similar tables for token, number, etc. 373 * 374 * @see #extractSearchIndexParametersForContainedResources(RequestDetails, ResourceIndexedSearchParams, IBaseResource, ResourceTable, ISearchParamExtractor.SearchParamSet) 375 * @see #extractSearchIndexParametersForUpliftedRefchains(RequestDetails, ResourceIndexedSearchParams, ResourceTable, RequestPartitionId, TransactionDetails, ISearchParamExtractor.SearchParamSet) 376 */ 377 private void extractSearchIndexParametersForTargetResources( 378 RequestDetails theRequestDetails, 379 ResourceIndexedSearchParams theParams, 380 ResourceTable theEntity, 381 Collection<IBaseResource> theAlreadySeenResources, 382 IChainedSearchParameterExtractionStrategy theTargetIndexingStrategy, 383 ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences, 384 boolean theRecurse, 385 boolean theIndexOnContainedResources) { 386 // 2. Find referenced search parameters 387 388 String spnamePrefix; 389 // 3. for each referenced search parameter, create an index 390 for (PathAndRef nextPathAndRef : theIndexedReferences) { 391 392 // 3.1 get the search parameter name as spname prefix 393 spnamePrefix = nextPathAndRef.getSearchParamName(); 394 395 if (spnamePrefix == null || (nextPathAndRef.getRef() == null && nextPathAndRef.getResource() == null)) 396 continue; 397 398 // 3.1.2 check if this ref actually applies here 399 ISearchParamExtractor.ISearchParamFilter searchParamsToIndex = 400 theTargetIndexingStrategy.getSearchParamFilter(nextPathAndRef); 401 if (searchParamsToIndex == ISearchParamExtractor.NO_PARAMS) { 402 continue; 403 } 404 405 // 3.2 find the target resource 406 IBaseResource targetResource = theTargetIndexingStrategy.fetchResourceAtPath(nextPathAndRef); 407 if (targetResource == null) continue; 408 409 // 3.2.1 if we've already processed this resource upstream, do not process it again, to prevent infinite 410 // loops 411 if (theAlreadySeenResources.contains(targetResource)) { 412 continue; 413 } 414 415 ResourceIndexedSearchParams currParams = ResourceIndexedSearchParams.withSets(); 416 417 // 3.3 create indexes for the current contained resource 418 getExtractionUtil() 419 .extractSearchIndexParameters(theRequestDetails, currParams, targetResource, searchParamsToIndex); 420 421 // 3.4 recurse to process any other contained resources referenced by this one 422 // Recursing is currently only allowed for contained resources and not 423 // uplifted refchains because the latter could potentially kill performance 424 // with the number of resource resolutions needed in order to handle 425 // a single write. Maybe in the future we could add caching to improve 426 // this 427 if (theRecurse) { 428 HashSet<IBaseResource> nextAlreadySeenResources = new HashSet<>(theAlreadySeenResources); 429 nextAlreadySeenResources.add(targetResource); 430 431 ISearchParamExtractor.SearchParamSet<PathAndRef> indexedReferences = 432 mySearchParamExtractor.extractResourceLinks(targetResource, theIndexOnContainedResources); 433 SearchParamExtractorService.handleWarnings( 434 theRequestDetails, myInterceptorBroadcaster, indexedReferences); 435 436 extractSearchIndexParametersForTargetResources( 437 theRequestDetails, 438 currParams, 439 theEntity, 440 nextAlreadySeenResources, 441 theTargetIndexingStrategy, 442 indexedReferences, 443 true, 444 theIndexOnContainedResources); 445 } 446 447 // 3.5 added reference name as a prefix for the contained resource if any 448 // e.g. for Observation.subject contained reference 449 // the SP_NAME = subject.family 450 currParams.updateSpnamePrefixForIndexOnUpliftedChain( 451 theEntity.getResourceType(), nextPathAndRef.getSearchParamName()); 452 453 // 3.6 merge to the mainParams 454 // NOTE: the spname prefix is different 455 mergeParams(currParams, theParams); 456 } 457 } 458 459 private IBaseResource findContainedResource(Collection<IBaseResource> resources, IBaseReference reference) { 460 for (IBaseResource resource : resources) { 461 if (resource.getIdElement().equals(reference.getReferenceElement())) return resource; 462 } 463 return null; 464 } 465 466 private void mergeParams(ResourceIndexedSearchParams theSrcParams, ResourceIndexedSearchParams theTargetParams) { 467 468 theTargetParams.myNumberParams.addAll(theSrcParams.myNumberParams); 469 theTargetParams.myQuantityParams.addAll(theSrcParams.myQuantityParams); 470 theTargetParams.myQuantityNormalizedParams.addAll(theSrcParams.myQuantityNormalizedParams); 471 theTargetParams.myDateParams.addAll(theSrcParams.myDateParams); 472 theTargetParams.myUriParams.addAll(theSrcParams.myUriParams); 473 theTargetParams.myTokenParams.addAll(theSrcParams.myTokenParams); 474 theTargetParams.myStringParams.addAll(theSrcParams.myStringParams); 475 theTargetParams.myCoordsParams.addAll(theSrcParams.myCoordsParams); 476 theTargetParams.myCompositeParams.addAll(theSrcParams.myCompositeParams); 477 } 478 479 private void populateResourceTables(ResourceIndexedSearchParams theParams, ResourceTable theEntity) { 480 481 populateResourceTable(theParams.myNumberParams, theEntity); 482 populateResourceTable(theParams.myQuantityParams, theEntity); 483 populateResourceTable(theParams.myQuantityNormalizedParams, theEntity); 484 populateResourceTable(theParams.myDateParams, theEntity); 485 populateResourceTable(theParams.myUriParams, theEntity); 486 populateResourceTable(theParams.myTokenParams, theEntity); 487 populateResourceTable(theParams.myStringParams, theEntity); 488 populateResourceTable(theParams.myCoordsParams, theEntity); 489 } 490 491 @VisibleForTesting 492 public void setContext(FhirContext theContext) { 493 myContext = theContext; 494 } 495 496 private void extractResourceLinks( 497 RequestPartitionId theRequestPartitionId, 498 ResourceIndexedSearchParams theParams, 499 ResourceTable theEntity, 500 IBaseResource theResource, 501 TransactionDetails theTransactionDetails, 502 boolean theFailOnInvalidReference, 503 RequestDetails theRequest, 504 ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) { 505 extractResourceLinks( 506 theRequestPartitionId, 507 ResourceIndexedSearchParams.withSets(), 508 theParams, 509 theEntity, 510 theResource, 511 theTransactionDetails, 512 theFailOnInvalidReference, 513 theRequest, 514 theIndexedReferences); 515 } 516 517 private void extractResourceLinks( 518 RequestPartitionId theRequestPartitionId, 519 ResourceIndexedSearchParams theExistingParams, 520 ResourceIndexedSearchParams theNewParams, 521 ResourceTable theEntity, 522 IBaseResource theResource, 523 TransactionDetails theTransactionDetails, 524 boolean theFailOnInvalidReference, 525 RequestDetails theRequest, 526 ISearchParamExtractor.SearchParamSet<PathAndRef> theIndexedReferences) { 527 String sourceResourceName = myContext.getResourceType(theResource); 528 529 for (PathAndRef nextPathAndRef : theIndexedReferences) { 530 if (nextPathAndRef.getRef() != null) { 531 if (nextPathAndRef.getRef().getReferenceElement().isLocal()) { 532 continue; 533 } 534 535 RuntimeSearchParam searchParam = mySearchParamRegistry.getActiveSearchParam( 536 sourceResourceName, nextPathAndRef.getSearchParamName()); 537 extractResourceLinks( 538 theRequestPartitionId, 539 theExistingParams, 540 theNewParams, 541 theEntity, 542 theTransactionDetails, 543 sourceResourceName, 544 searchParam, 545 nextPathAndRef, 546 theFailOnInvalidReference, 547 theRequest); 548 } 549 } 550 551 theEntity.setHasLinks(!theNewParams.myLinks.isEmpty()); 552 } 553 554 private void extractResourceLinks( 555 @Nonnull RequestPartitionId theRequestPartitionId, 556 ResourceIndexedSearchParams theExistingParams, 557 ResourceIndexedSearchParams theNewParams, 558 ResourceTable theEntity, 559 TransactionDetails theTransactionDetails, 560 String theSourceResourceName, 561 RuntimeSearchParam theRuntimeSearchParam, 562 PathAndRef thePathAndRef, 563 boolean theFailOnInvalidReference, 564 RequestDetails theRequest) { 565 IBaseReference nextReference = thePathAndRef.getRef(); 566 IIdType nextId = nextReference.getReferenceElement(); 567 String path = thePathAndRef.getPath(); 568 Date transactionDate = theTransactionDetails.getTransactionDate(); 569 570 /* 571 * This can only really happen if the DAO is being called 572 * programmatically with a Bundle (not through the FHIR REST API) 573 * but Smile does this 574 */ 575 if (nextId.isEmpty() && nextReference.getResource() != null) { 576 nextId = nextReference.getResource().getIdElement(); 577 } 578 579 if (myContext.getParserOptions().isStripVersionsFromReferences() 580 && !myContext 581 .getParserOptions() 582 .getDontStripVersionsFromReferencesAtPaths() 583 .contains(thePathAndRef.getPath()) 584 && nextId.hasVersionIdPart()) { 585 nextId = nextId.toVersionless(); 586 } 587 588 theNewParams.myPopulatedResourceLinkParameters.add(thePathAndRef.getSearchParamName()); 589 590 boolean canonical = thePathAndRef.isCanonical(); 591 if (LogicalReferenceHelper.isLogicalReference(myStorageSettings, nextId) || canonical) { 592 String value = nextId.getValue(); 593 ResourceLink resourceLink = 594 ResourceLink.forLogicalReference(thePathAndRef.getPath(), theEntity, value, transactionDate); 595 if (theNewParams.myLinks.add(resourceLink)) { 596 ourLog.debug("Indexing remote resource reference URL: {}", nextId); 597 } 598 return; 599 } 600 601 String baseUrl = nextId.getBaseUrl(); 602 603 // If this is a conditional URL, the part after the question mark 604 // can include URLs (e.g. token system URLs) and these really confuse 605 // the IdType parser because a conditional URL isn't actually a valid 606 // FHIR ID. So in order to truly determine whether we're dealing with 607 // an absolute reference, we strip the query part and reparse 608 // the reference. 609 int questionMarkIndex = nextId.getValue().indexOf('?'); 610 if (questionMarkIndex != -1) { 611 IdType preQueryId = new IdType(nextId.getValue().substring(0, questionMarkIndex - 1)); 612 baseUrl = preQueryId.getBaseUrl(); 613 } 614 615 String typeString = nextId.getResourceType(); 616 if (isBlank(typeString)) { 617 String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource type - " 618 + nextId.getValue(); 619 if (theFailOnInvalidReference) { 620 throw new InvalidRequestException(Msg.code(505) + msg); 621 } else { 622 ourLog.debug(msg); 623 return; 624 } 625 } 626 RuntimeResourceDefinition resourceDefinition; 627 try { 628 resourceDefinition = myContext.getResourceDefinition(typeString); 629 } catch (DataFormatException e) { 630 String msg = "Invalid resource reference found at path[" + path 631 + "] - Resource type is unknown or not supported on this server - " + nextId.getValue(); 632 if (theFailOnInvalidReference) { 633 throw new InvalidRequestException(Msg.code(506) + msg); 634 } else { 635 ourLog.debug(msg); 636 return; 637 } 638 } 639 640 if (theRuntimeSearchParam.hasTargets()) { 641 if (!theRuntimeSearchParam.getTargets().contains(typeString)) { 642 return; 643 } 644 } 645 646 if (isNotBlank(baseUrl)) { 647 if (!myStorageSettings.getTreatBaseUrlsAsLocal().contains(baseUrl) 648 && !myStorageSettings.isAllowExternalReferences()) { 649 String msg = myContext 650 .getLocalizer() 651 .getMessage(BaseSearchParamExtractor.class, "externalReferenceNotAllowed", nextId.getValue()); 652 throw new InvalidRequestException(Msg.code(507) + msg); 653 } else { 654 ResourceLink resourceLink = 655 ResourceLink.forAbsoluteReference(thePathAndRef.getPath(), theEntity, nextId, transactionDate); 656 if (theNewParams.myLinks.add(resourceLink)) { 657 ourLog.debug("Indexing remote resource reference URL: {}", nextId); 658 } 659 return; 660 } 661 } 662 663 Class<? extends IBaseResource> type = resourceDefinition.getImplementingClass(); 664 String targetId = nextId.getIdPart(); 665 if (StringUtils.isBlank(targetId)) { 666 String msg = "Invalid resource reference found at path[" + path + "] - Does not contain resource ID - " 667 + nextId.getValue(); 668 if (theFailOnInvalidReference) { 669 throw new InvalidRequestException(Msg.code(508) + msg); 670 } else { 671 ourLog.debug(msg); 672 return; 673 } 674 } 675 676 IIdType referenceElement = thePathAndRef.getRef().getReferenceElement(); 677 JpaPid resolvedTargetId = (JpaPid) theTransactionDetails.getResolvedResourceId(referenceElement); 678 ResourceLink resourceLink; 679 680 Long targetVersionId = nextId.getVersionIdPartAsLong(); 681 if (resolvedTargetId != null) { 682 683 /* 684 * If we have already resolved the given reference within this transaction, we don't 685 * need to resolve it again 686 */ 687 myResourceLinkResolver.validateTypeOrThrowException(type); 688 689 ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance() 690 .setSourcePath(thePathAndRef.getPath()) 691 .setSourceResource(theEntity) 692 .setTargetResourceType(typeString) 693 .setTargetResourcePid(resolvedTargetId.getId()) 694 .setTargetResourceId(targetId) 695 .setUpdated(transactionDate) 696 .setTargetResourceVersion(targetVersionId) 697 .setTargetResourcePartitionablePartitionId(resolvedTargetId.getPartitionablePartitionId()); 698 699 resourceLink = forLocalReference(params); 700 701 } else if (theFailOnInvalidReference) { 702 703 /* 704 * The reference points to another resource, so let's look it up. We need to do this 705 * since the target may be a forced ID, but also so that we can throw an exception 706 * if the reference is invalid 707 */ 708 myResourceLinkResolver.validateTypeOrThrowException(type); 709 710 /* 711 * We need to obtain a resourceLink out of the provided {@literal thePathAndRef}. In the case 712 * where we are updating a resource that already has resourceLinks (stored in {@literal theExistingParams.getResourceLinks()}), 713 * let's try to match thePathAndRef to an already existing resourceLink to avoid the 714 * very expensive operation of creating a resourceLink that would end up being exactly the same 715 * one we already have. 716 */ 717 Optional<ResourceLink> optionalResourceLink = 718 findMatchingResourceLink(thePathAndRef, theExistingParams.getResourceLinks()); 719 if (optionalResourceLink.isPresent()) { 720 resourceLink = optionalResourceLink.get(); 721 } else { 722 resourceLink = resolveTargetAndCreateResourceLinkOrReturnNull( 723 theRequestPartitionId, 724 theSourceResourceName, 725 thePathAndRef, 726 theEntity, 727 transactionDate, 728 nextId, 729 theRequest, 730 theTransactionDetails); 731 } 732 733 if (resourceLink == null) { 734 return; 735 } else { 736 // Cache the outcome in the current transaction in case there are more references 737 JpaPid persistentId = JpaPid.fromId(resourceLink.getTargetResourcePid()); 738 persistentId.setPartitionablePartitionId(resourceLink.getTargetResourcePartitionId()); 739 theTransactionDetails.addResolvedResourceId(referenceElement, persistentId); 740 } 741 742 } else { 743 744 /* 745 * Just assume the reference is valid. This is used for in-memory matching since there 746 * is no expectation of a database in this situation 747 */ 748 ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance() 749 .setSourcePath(thePathAndRef.getPath()) 750 .setSourceResource(theEntity) 751 .setTargetResourceType(typeString) 752 .setTargetResourceId(targetId) 753 .setUpdated(transactionDate) 754 .setTargetResourceVersion(targetVersionId); 755 756 resourceLink = forLocalReference(params); 757 } 758 759 theNewParams.myLinks.add(resourceLink); 760 } 761 762 private Optional<ResourceLink> findMatchingResourceLink( 763 PathAndRef thePathAndRef, Collection<ResourceLink> theResourceLinks) { 764 IIdType referenceElement = thePathAndRef.getRef().getReferenceElement(); 765 List<ResourceLink> resourceLinks = new ArrayList<>(theResourceLinks); 766 for (ResourceLink resourceLink : resourceLinks) { 767 768 // comparing the searchParam path ex: Group.member.entity 769 boolean hasMatchingSearchParamPath = 770 StringUtils.equals(resourceLink.getSourcePath(), thePathAndRef.getPath()); 771 772 boolean hasMatchingResourceType = 773 StringUtils.equals(resourceLink.getTargetResourceType(), referenceElement.getResourceType()); 774 775 boolean hasMatchingResourceId = 776 StringUtils.equals(resourceLink.getTargetResourceId(), referenceElement.getIdPart()); 777 778 boolean hasMatchingResourceVersion = myContext.getParserOptions().isStripVersionsFromReferences() 779 || referenceElement.getVersionIdPartAsLong() == null 780 || referenceElement.getVersionIdPartAsLong().equals(resourceLink.getTargetResourceVersion()); 781 782 if (hasMatchingSearchParamPath 783 && hasMatchingResourceType 784 && hasMatchingResourceId 785 && hasMatchingResourceVersion) { 786 return Optional.of(resourceLink); 787 } 788 } 789 790 return Optional.empty(); 791 } 792 793 private void extractResourceLinksForContainedResources( 794 RequestPartitionId theRequestPartitionId, 795 ResourceIndexedSearchParams theParams, 796 ResourceTable theEntity, 797 IBaseResource theResource, 798 TransactionDetails theTransactionDetails, 799 boolean theFailOnInvalidReference, 800 RequestDetails theRequest) { 801 802 FhirTerser terser = myContext.newTerser(); 803 804 // 1. get all contained resources 805 Collection<IBaseResource> containedResources = terser.getAllEmbeddedResources(theResource, false); 806 807 extractResourceLinksForContainedResources( 808 theRequestPartitionId, 809 theParams, 810 theEntity, 811 theResource, 812 theTransactionDetails, 813 theFailOnInvalidReference, 814 theRequest, 815 containedResources, 816 new HashSet<>()); 817 } 818 819 private void extractResourceLinksForContainedResources( 820 RequestPartitionId theRequestPartitionId, 821 ResourceIndexedSearchParams theParams, 822 ResourceTable theEntity, 823 IBaseResource theResource, 824 TransactionDetails theTransactionDetails, 825 boolean theFailOnInvalidReference, 826 RequestDetails theRequest, 827 Collection<IBaseResource> theContainedResources, 828 Collection<IBaseResource> theAlreadySeenResources) { 829 830 // 2. Find referenced search parameters 831 ISearchParamExtractor.SearchParamSet<PathAndRef> referencedSearchParamSet = 832 mySearchParamExtractor.extractResourceLinks(theResource, true); 833 834 String spNamePrefix; 835 ResourceIndexedSearchParams currParams; 836 // 3. for each referenced search parameter, create an index 837 for (PathAndRef nextPathAndRef : referencedSearchParamSet) { 838 839 // 3.1 get the search parameter name as spname prefix 840 spNamePrefix = nextPathAndRef.getSearchParamName(); 841 842 if (spNamePrefix == null || nextPathAndRef.getRef() == null) continue; 843 844 // 3.2 find the contained resource 845 IBaseResource containedResource = findContainedResource(theContainedResources, nextPathAndRef.getRef()); 846 if (containedResource == null) continue; 847 848 // 3.2.1 if we've already processed this resource upstream, do not process it again, to prevent infinite 849 // loops 850 if (theAlreadySeenResources.contains(containedResource)) { 851 continue; 852 } 853 854 currParams = ResourceIndexedSearchParams.withSets(); 855 856 // 3.3 create indexes for the current contained resource 857 ISearchParamExtractor.SearchParamSet<PathAndRef> indexedReferences = 858 mySearchParamExtractor.extractResourceLinks(containedResource, true); 859 extractResourceLinks( 860 theRequestPartitionId, 861 currParams, 862 theEntity, 863 containedResource, 864 theTransactionDetails, 865 theFailOnInvalidReference, 866 theRequest, 867 indexedReferences); 868 869 // 3.4 recurse to process any other contained resources referenced by this one 870 if (myStorageSettings.isIndexOnContainedResourcesRecursively()) { 871 HashSet<IBaseResource> nextAlreadySeenResources = new HashSet<>(theAlreadySeenResources); 872 nextAlreadySeenResources.add(containedResource); 873 extractResourceLinksForContainedResources( 874 theRequestPartitionId, 875 currParams, 876 theEntity, 877 containedResource, 878 theTransactionDetails, 879 theFailOnInvalidReference, 880 theRequest, 881 theContainedResources, 882 nextAlreadySeenResources); 883 } 884 885 // 3.4 added reference name as a prefix for the contained resource if any 886 // e.g. for Observation.subject contained reference 887 // the SP_NAME = subject.family 888 currParams.updateSpnamePrefixForLinksOnContainedResource(nextPathAndRef.getPath()); 889 890 // 3.5 merge to the mainParams 891 // NOTE: the spname prefix is different 892 theParams.getResourceLinks().addAll(currParams.getResourceLinks()); 893 } 894 } 895 896 @SuppressWarnings("unchecked") 897 private ResourceLink resolveTargetAndCreateResourceLinkOrReturnNull( 898 @Nonnull RequestPartitionId theRequestPartitionId, 899 String theSourceResourceName, 900 PathAndRef thePathAndRef, 901 ResourceTable theEntity, 902 Date theUpdateTime, 903 IIdType theNextId, 904 RequestDetails theRequest, 905 TransactionDetails theTransactionDetails) { 906 JpaPid resolvedResourceId = (JpaPid) theTransactionDetails.getResolvedResourceId(theNextId); 907 908 if (resolvedResourceId != null) { 909 String targetResourceType = theNextId.getResourceType(); 910 Long targetResourcePid = resolvedResourceId.getId(); 911 String targetResourceIdPart = theNextId.getIdPart(); 912 Long targetVersion = theNextId.getVersionIdPartAsLong(); 913 914 ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance() 915 .setSourcePath(thePathAndRef.getPath()) 916 .setSourceResource(theEntity) 917 .setTargetResourceType(targetResourceType) 918 .setTargetResourcePid(targetResourcePid) 919 .setTargetResourceId(targetResourceIdPart) 920 .setUpdated(theUpdateTime) 921 .setTargetResourceVersion(targetVersion) 922 .setTargetResourcePartitionablePartitionId(resolvedResourceId.getPartitionablePartitionId()); 923 924 return ResourceLink.forLocalReference(params); 925 } 926 927 /* 928 * We keep a cache of resolved target resources. This is good since for some resource types, there 929 * are multiple search parameters that map to the same element path within a resource (e.g. 930 * Observation:patient and Observation.subject and we don't want to force a resolution of the 931 * target any more times than we have to. 932 */ 933 934 IResourceLookup<JpaPid> targetResource; 935 if (myPartitionSettings.isPartitioningEnabled()) { 936 if (myPartitionSettings.getAllowReferencesAcrossPartitions() == ALLOWED_UNQUALIFIED) { 937 938 // Interceptor: Pointcut.JPA_CROSS_PARTITION_REFERENCE_DETECTED 939 if (CompositeInterceptorBroadcaster.hasHooks( 940 Pointcut.JPA_RESOLVE_CROSS_PARTITION_REFERENCE, myInterceptorBroadcaster, theRequest)) { 941 CrossPartitionReferenceDetails referenceDetails = new CrossPartitionReferenceDetails( 942 theRequestPartitionId, 943 theSourceResourceName, 944 thePathAndRef, 945 theRequest, 946 theTransactionDetails); 947 HookParams params = new HookParams(referenceDetails); 948 targetResource = 949 (IResourceLookup<JpaPid>) CompositeInterceptorBroadcaster.doCallHooksAndReturnObject( 950 myInterceptorBroadcaster, 951 theRequest, 952 Pointcut.JPA_RESOLVE_CROSS_PARTITION_REFERENCE, 953 params); 954 } else { 955 targetResource = myResourceLinkResolver.findTargetResource( 956 RequestPartitionId.allPartitions(), 957 theSourceResourceName, 958 thePathAndRef, 959 theRequest, 960 theTransactionDetails); 961 } 962 963 } else { 964 targetResource = myResourceLinkResolver.findTargetResource( 965 theRequestPartitionId, theSourceResourceName, thePathAndRef, theRequest, theTransactionDetails); 966 } 967 } else { 968 targetResource = myResourceLinkResolver.findTargetResource( 969 theRequestPartitionId, theSourceResourceName, thePathAndRef, theRequest, theTransactionDetails); 970 } 971 972 if (targetResource == null) { 973 return null; 974 } 975 976 String targetResourceType = targetResource.getResourceType(); 977 Long targetResourcePid = targetResource.getPersistentId().getId(); 978 String targetResourceIdPart = theNextId.getIdPart(); 979 Long targetVersion = theNextId.getVersionIdPartAsLong(); 980 981 ResourceLinkForLocalReferenceParams params = ResourceLinkForLocalReferenceParams.instance() 982 .setSourcePath(thePathAndRef.getPath()) 983 .setSourceResource(theEntity) 984 .setTargetResourceType(targetResourceType) 985 .setTargetResourcePid(targetResourcePid) 986 .setTargetResourceId(targetResourceIdPart) 987 .setUpdated(theUpdateTime) 988 .setTargetResourceVersion(targetVersion) 989 .setTargetResourcePartitionablePartitionId( 990 targetResource.getPersistentId().getPartitionablePartitionId()); 991 992 return forLocalReference(params); 993 } 994 995 private RequestPartitionId determineResolverPartitionId(@Nonnull RequestPartitionId theRequestPartitionId) { 996 RequestPartitionId targetRequestPartitionId = theRequestPartitionId; 997 if (myPartitionSettings.isPartitioningEnabled() 998 && myPartitionSettings.getAllowReferencesAcrossPartitions() == ALLOWED_UNQUALIFIED) { 999 targetRequestPartitionId = RequestPartitionId.allPartitions(); 1000 } 1001 return targetRequestPartitionId; 1002 } 1003 1004 private void populateResourceTable( 1005 Collection<? extends BaseResourceIndexedSearchParam> theParams, ResourceTable theResourceTable) { 1006 for (BaseResourceIndexedSearchParam next : theParams) { 1007 if (next.getResourcePid() == null) { 1008 next.setResource(theResourceTable); 1009 } 1010 } 1011 } 1012 1013 private void populateResourceTableForComboParams( 1014 Collection<? extends IResourceIndexComboSearchParameter> theParams, ResourceTable theResourceTable) { 1015 for (IResourceIndexComboSearchParameter next : theParams) { 1016 if (next.getResource() == null) { 1017 next.setResource(theResourceTable); 1018 if (next instanceof BasePartitionable) { 1019 ((BasePartitionable) next).setPartitionId(theResourceTable.getPartitionId()); 1020 } 1021 } 1022 } 1023 } 1024 1025 @VisibleForTesting 1026 void setInterceptorBroadcasterForUnitTest(IInterceptorBroadcaster theInterceptorBroadcaster) { 1027 myInterceptorBroadcaster = theInterceptorBroadcaster; 1028 } 1029 1030 @Nonnull 1031 public List<String> extractParamValuesAsStrings( 1032 RuntimeSearchParam theActiveSearchParam, IBaseResource theResource) { 1033 return mySearchParamExtractor.extractParamValuesAsStrings(theActiveSearchParam, theResource); 1034 } 1035 1036 public void extractSearchParamComboUnique(ResourceTable theEntity, ResourceIndexedSearchParams theParams) { 1037 String resourceType = theEntity.getResourceType(); 1038 Set<ResourceIndexedComboStringUnique> comboUniques = 1039 mySearchParamExtractor.extractSearchParamComboUnique(resourceType, theParams); 1040 theParams.myComboStringUniques.addAll(comboUniques); 1041 populateResourceTableForComboParams(theParams.myComboStringUniques, theEntity); 1042 } 1043 1044 public void extractSearchParamComboNonUnique(ResourceTable theEntity, ResourceIndexedSearchParams theParams) { 1045 String resourceType = theEntity.getResourceType(); 1046 Set<ResourceIndexedComboTokenNonUnique> comboNonUniques = 1047 mySearchParamExtractor.extractSearchParamComboNonUnique(resourceType, theParams); 1048 theParams.myComboTokenNonUnique.addAll(comboNonUniques); 1049 populateResourceTableForComboParams(theParams.myComboTokenNonUnique, theEntity); 1050 } 1051 1052 /** 1053 * This interface is used by {@link #extractSearchIndexParametersForTargetResources(RequestDetails, ResourceIndexedSearchParams, ResourceTable, Collection, IChainedSearchParameterExtractionStrategy, ISearchParamExtractor.SearchParamSet, boolean, boolean)} 1054 * in order to use that method for extracting chained search parameter indexes both 1055 * from contained resources and from uplifted refchains. 1056 */ 1057 private interface IChainedSearchParameterExtractionStrategy { 1058 1059 /** 1060 * Which search parameters should be indexed for the resource target 1061 * at the given path. In other words if thePathAndRef contains 1062 * "Patient/123", then we could return a filter that only lets the 1063 * "name" and "gender" search params through if we only want those 1064 * two parameters to be indexed for the resolved Patient resource 1065 * with that ID. 1066 */ 1067 @Nonnull 1068 ISearchParamExtractor.ISearchParamFilter getSearchParamFilter(@Nonnull PathAndRef thePathAndRef); 1069 1070 /** 1071 * Actually fetch the resource at the given path, or return 1072 * {@literal null} if none can be found. 1073 */ 1074 @Nullable 1075 IBaseResource fetchResourceAtPath(@Nonnull PathAndRef thePathAndRef); 1076 } 1077 1078 static void handleWarnings( 1079 RequestDetails theRequestDetails, 1080 IInterceptorBroadcaster theInterceptorBroadcaster, 1081 ISearchParamExtractor.SearchParamSet<?> theSearchParamSet) { 1082 if (theSearchParamSet.getWarnings().isEmpty()) { 1083 return; 1084 } 1085 1086 // If extraction generated any warnings, broadcast an error 1087 if (CompositeInterceptorBroadcaster.hasHooks( 1088 Pointcut.JPA_PERFTRACE_WARNING, theInterceptorBroadcaster, theRequestDetails)) { 1089 for (String next : theSearchParamSet.getWarnings()) { 1090 StorageProcessingMessage messageHolder = new StorageProcessingMessage(); 1091 messageHolder.setMessage(next); 1092 HookParams params = new HookParams() 1093 .add(RequestDetails.class, theRequestDetails) 1094 .addIfMatchesType(ServletRequestDetails.class, theRequestDetails) 1095 .add(StorageProcessingMessage.class, messageHolder); 1096 CompositeInterceptorBroadcaster.doCallHooks( 1097 theInterceptorBroadcaster, theRequestDetails, Pointcut.JPA_PERFTRACE_WARNING, params); 1098 } 1099 } 1100 } 1101}