001/*- 002 * #%L 003 * HAPI FHIR Storage api 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.binary.interceptor; 021 022import ca.uhn.fhir.context.BaseRuntimeChildDefinition; 023import ca.uhn.fhir.context.BaseRuntimeElementDefinition; 024import ca.uhn.fhir.context.FhirContext; 025import ca.uhn.fhir.i18n.Msg; 026import ca.uhn.fhir.interceptor.api.Hook; 027import ca.uhn.fhir.interceptor.api.HookParams; 028import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; 029import ca.uhn.fhir.interceptor.api.Interceptor; 030import ca.uhn.fhir.interceptor.api.Pointcut; 031import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc; 032import ca.uhn.fhir.jpa.binary.api.IBinaryTarget; 033import ca.uhn.fhir.jpa.binary.api.StoredDetails; 034import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider; 035import ca.uhn.fhir.jpa.binary.svc.BaseBinaryStorageSvcImpl; 036import ca.uhn.fhir.jpa.model.util.JpaConstants; 037import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails; 038import ca.uhn.fhir.rest.api.server.RequestDetails; 039import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; 040import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; 041import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; 042import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; 043import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; 044import ca.uhn.fhir.util.HapiExtensions; 045import ca.uhn.fhir.util.IModelVisitor2; 046import jakarta.annotation.Nonnull; 047import org.apache.commons.io.FileUtils; 048import org.apache.commons.lang3.StringUtils; 049import org.hl7.fhir.instance.model.api.IBase; 050import org.hl7.fhir.instance.model.api.IBaseHasExtensions; 051import org.hl7.fhir.instance.model.api.IBaseResource; 052import org.hl7.fhir.instance.model.api.IIdType; 053import org.hl7.fhir.instance.model.api.IPrimitiveType; 054import org.hl7.fhir.r4.model.IdType; 055import org.slf4j.Logger; 056import org.slf4j.LoggerFactory; 057import org.springframework.beans.factory.annotation.Autowired; 058 059import java.awt.*; 060import java.io.ByteArrayInputStream; 061import java.io.IOException; 062import java.io.InputStream; 063import java.util.ArrayList; 064import java.util.HashSet; 065import java.util.List; 066import java.util.Optional; 067import java.util.Set; 068import java.util.concurrent.atomic.AtomicInteger; 069import java.util.stream.Collectors; 070 071import static ca.uhn.fhir.util.HapiExtensions.EXT_EXTERNALIZED_BINARY_ID; 072import static org.apache.commons.lang3.StringUtils.isNotBlank; 073 074@Interceptor 075public class BinaryStorageInterceptor<T extends IPrimitiveType<byte[]>> { 076 077 private static final Logger ourLog = LoggerFactory.getLogger(BinaryStorageInterceptor.class); 078 079 @Autowired 080 private IBinaryStorageSvc myBinaryStorageSvc; 081 082 private final FhirContext myCtx; 083 084 @Autowired 085 private BinaryAccessProvider myBinaryAccessProvider; 086 087 @Autowired 088 private IInterceptorBroadcaster myInterceptorBroadcaster; 089 090 private Class<T> myBinaryType; 091 private String myDeferredListKey; 092 private long myAutoInflateBinariesMaximumBytes = 10 * FileUtils.ONE_MB; 093 private boolean myAllowAutoInflateBinaries = true; 094 095 public BinaryStorageInterceptor(FhirContext theCtx) { 096 myCtx = theCtx; 097 BaseRuntimeElementDefinition<?> base64Binary = myCtx.getElementDefinition("base64Binary"); 098 assert base64Binary != null; 099 myBinaryType = (Class<T>) base64Binary.getImplementingClass(); 100 myDeferredListKey = getClass().getName() + "_" + hashCode() + "_DEFERRED_LIST"; 101 } 102 103 /** 104 * Any externalized binaries will be rehydrated if their size is below this thhreshold when 105 * reading the resource back. Default is 10MB. 106 */ 107 public long getAutoInflateBinariesMaximumSize() { 108 return myAutoInflateBinariesMaximumBytes; 109 } 110 111 /** 112 * Any externalized binaries will be rehydrated if their size is below this thhreshold when 113 * reading the resource back. Default is 10MB. 114 */ 115 public void setAutoInflateBinariesMaximumSize(long theAutoInflateBinariesMaximumBytes) { 116 myAutoInflateBinariesMaximumBytes = theAutoInflateBinariesMaximumBytes; 117 } 118 119 @Hook(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_RESOURCE) 120 public void expungeResource(AtomicInteger theCounter, IBaseResource theResource) { 121 122 List<? extends IBase> binaryElements = 123 myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType); 124 125 List<String> attachmentIds = binaryElements.stream() 126 .flatMap(t -> ((IBaseHasExtensions) t).getExtension().stream()) 127 .filter(t -> HapiExtensions.EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl())) 128 .map(t -> ((IPrimitiveType<?>) t.getValue()).getValueAsString()) 129 .collect(Collectors.toList()); 130 131 for (String next : attachmentIds) { 132 myBinaryStorageSvc.expungeBlob(theResource.getIdElement(), next); 133 theCounter.incrementAndGet(); 134 135 ourLog.info( 136 "Deleting binary blob {} because resource {} is being expunged", 137 next, 138 theResource.getIdElement().getValue()); 139 } 140 } 141 142 @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED) 143 public void extractLargeBinariesBeforeCreate( 144 RequestDetails theRequestDetails, 145 TransactionDetails theTransactionDetails, 146 IBaseResource theResource, 147 Pointcut thePointcut) 148 throws IOException { 149 extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut); 150 } 151 152 @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED) 153 public void extractLargeBinariesBeforeUpdate( 154 RequestDetails theRequestDetails, 155 TransactionDetails theTransactionDetails, 156 IBaseResource thePreviousResource, 157 IBaseResource theResource, 158 Pointcut thePointcut) 159 throws IOException { 160 blockIllegalExternalBinaryIds(thePreviousResource, theResource); 161 extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut); 162 } 163 164 /** 165 * Don't allow clients to submit resources with binary storage attachments declared unless the ID was already in the 166 * resource. In other words, only HAPI itself may add a binary storage ID extension to a resource unless that 167 * extension was already present. 168 */ 169 private void blockIllegalExternalBinaryIds(IBaseResource thePreviousResource, IBaseResource theResource) { 170 Set<String> existingBinaryIds = new HashSet<>(); 171 if (thePreviousResource != null) { 172 List<T> base64fields = 173 myCtx.newTerser().getAllPopulatedChildElementsOfType(thePreviousResource, myBinaryType); 174 for (IPrimitiveType<byte[]> nextBase64 : base64fields) { 175 if (nextBase64 instanceof IBaseHasExtensions) { 176 ((IBaseHasExtensions) nextBase64) 177 .getExtension().stream() 178 .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null) 179 .filter(t -> EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl())) 180 .map(t -> (IPrimitiveType<?>) t.getValue()) 181 .map(IPrimitiveType::getValueAsString) 182 .filter(StringUtils::isNotBlank) 183 .forEach(existingBinaryIds::add); 184 } 185 } 186 } 187 188 List<T> base64fields = myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType); 189 for (IPrimitiveType<byte[]> nextBase64 : base64fields) { 190 if (nextBase64 instanceof IBaseHasExtensions) { 191 Optional<String> hasExternalizedBinaryReference = ((IBaseHasExtensions) nextBase64) 192 .getExtension().stream() 193 .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null) 194 .filter(t -> t.getUrl().equals(EXT_EXTERNALIZED_BINARY_ID)) 195 .map(t -> (IPrimitiveType<?>) t.getValue()) 196 .map(IPrimitiveType::getValueAsString) 197 .filter(StringUtils::isNotBlank) 198 .filter(t -> !existingBinaryIds.contains(t)) 199 .findFirst(); 200 201 if (hasExternalizedBinaryReference.isPresent()) { 202 String msg = myCtx.getLocalizer() 203 .getMessage( 204 BinaryStorageInterceptor.class, 205 "externalizedBinaryStorageExtensionFoundInRequestBody", 206 EXT_EXTERNALIZED_BINARY_ID, 207 hasExternalizedBinaryReference.get()); 208 throw new InvalidRequestException(Msg.code(1329) + msg); 209 } 210 } 211 } 212 } 213 214 private void extractLargeBinaries( 215 RequestDetails theRequestDetails, 216 TransactionDetails theTransactionDetails, 217 IBaseResource theResource, 218 Pointcut thePointcut) 219 throws IOException { 220 221 IIdType resourceId = theResource.getIdElement(); 222 if (!resourceId.hasResourceType() && resourceId.hasIdPart()) { 223 String resourceType = myCtx.getResourceType(theResource); 224 resourceId = new IdType(resourceType + "/" + resourceId.getIdPart()); 225 } 226 227 List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource); 228 for (IBinaryTarget nextTarget : attachments) { 229 byte[] data = nextTarget.getData(); 230 if (data != null && data.length > 0) { 231 232 long nextPayloadLength = data.length; 233 String nextContentType = nextTarget.getContentType(); 234 boolean shouldStoreBlob = 235 myBinaryStorageSvc.shouldStoreBlob(nextPayloadLength, resourceId, nextContentType); 236 if (shouldStoreBlob) { 237 238 String newBlobId; 239 if (thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED) { 240 ByteArrayInputStream inputStream = new ByteArrayInputStream(data); 241 StoredDetails storedDetails = myBinaryStorageSvc.storeBlob( 242 resourceId, null, nextContentType, inputStream, theRequestDetails); 243 newBlobId = storedDetails.getBlobId(); 244 } else { 245 assert thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED : thePointcut.name(); 246 newBlobId = myBinaryStorageSvc.newBlobId(); 247 248 String prefix = invokeAssignBlobPrefix(theRequestDetails, theResource); 249 if (isNotBlank(prefix)) { 250 newBlobId = prefix + newBlobId; 251 } 252 if (myBinaryStorageSvc.isValidBlobId(newBlobId)) { 253 List<DeferredBinaryTarget> deferredBinaryTargets = 254 getOrCreateDeferredBinaryStorageList(theResource); 255 DeferredBinaryTarget newDeferredBinaryTarget = 256 new DeferredBinaryTarget(newBlobId, nextTarget, data); 257 deferredBinaryTargets.add(newDeferredBinaryTarget); 258 newDeferredBinaryTarget.setBlobIdPrefixHookApplied(true); 259 } else { 260 throw new InternalErrorException(Msg.code(2341) 261 + "Invalid blob ID for backing storage service.[blobId=" + newBlobId + ",service=" 262 + myBinaryStorageSvc.getClass().getName() + "]"); 263 } 264 } 265 266 myBinaryAccessProvider.replaceDataWithExtension(nextTarget, newBlobId); 267 } 268 } 269 } 270 } 271 272 /** 273 * This invokes the {@link Pointcut#STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX} hook and returns the prefix to use for the blob ID, or null if there are no implementers. 274 * @return A string, which will be used to prefix the blob ID. May be null. 275 */ 276 private String invokeAssignBlobPrefix(RequestDetails theRequest, IBaseResource theResource) { 277 if (!CompositeInterceptorBroadcaster.hasHooks( 278 Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, myInterceptorBroadcaster, theRequest)) { 279 return null; 280 } 281 282 HookParams params = 283 new HookParams().add(RequestDetails.class, theRequest).add(IBaseResource.class, theResource); 284 285 BaseBinaryStorageSvcImpl.setBlobIdPrefixApplied(theRequest); 286 287 return (String) CompositeInterceptorBroadcaster.doCallHooksAndReturnObject( 288 myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, params); 289 } 290 291 @Nonnull 292 @SuppressWarnings("unchecked") 293 private List<DeferredBinaryTarget> getOrCreateDeferredBinaryStorageList(IBaseResource theResource) { 294 Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey()); 295 if (deferredBinaryTargetList == null) { 296 deferredBinaryTargetList = new ArrayList<>(); 297 theResource.setUserData(getDeferredListKey(), deferredBinaryTargetList); 298 } 299 return (List<DeferredBinaryTarget>) deferredBinaryTargetList; 300 } 301 302 @SuppressWarnings("unchecked") 303 @Hook(Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED) 304 public void storeLargeBinariesBeforeCreatePersistence( 305 TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut) 306 throws IOException { 307 if (theResource == null) { 308 return; 309 } 310 Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey()); 311 312 if (deferredBinaryTargetList != null) { 313 IIdType resourceId = theResource.getIdElement(); 314 for (DeferredBinaryTarget next : (List<DeferredBinaryTarget>) deferredBinaryTargetList) { 315 String blobId = next.getBlobId(); 316 IBinaryTarget target = next.getBinaryTarget(); 317 InputStream dataStream = next.getDataStream(); 318 String contentType = target.getContentType(); 319 RequestDetails requestDetails = initRequestDetails(next); 320 myBinaryStorageSvc.storeBlob(resourceId, blobId, contentType, dataStream, requestDetails); 321 } 322 } 323 } 324 325 private RequestDetails initRequestDetails(DeferredBinaryTarget theDeferredBinaryTarget) { 326 ServletRequestDetails requestDetails = new ServletRequestDetails(); 327 if (theDeferredBinaryTarget.isBlobIdPrefixHookApplied()) { 328 BaseBinaryStorageSvcImpl.setBlobIdPrefixApplied(requestDetails); 329 } 330 return requestDetails; 331 } 332 333 public String getDeferredListKey() { 334 return myDeferredListKey; 335 } 336 337 @Hook(Pointcut.STORAGE_PRESHOW_RESOURCES) 338 public void preShow(IPreResourceShowDetails theDetails) throws IOException { 339 if (!isAllowAutoInflateBinaries()) { 340 return; 341 } 342 343 long cumulativeInflatedBytes = 0; 344 int inflatedResourceCount = 0; 345 346 for (IBaseResource nextResource : theDetails) { 347 if (nextResource == null) { 348 ourLog.warn( 349 "Received a null resource during STORAGE_PRESHOW_RESOURCES. This is a bug and should be reported. Skipping resource."); 350 continue; 351 } 352 cumulativeInflatedBytes = inflateBinariesInResource(cumulativeInflatedBytes, nextResource); 353 inflatedResourceCount += 1; 354 if (cumulativeInflatedBytes >= myAutoInflateBinariesMaximumBytes) { 355 ourLog.debug( 356 "Exiting binary data inflation early.[byteCount={}, resourcesInflated={}, resourcesSkipped={}]", 357 cumulativeInflatedBytes, 358 inflatedResourceCount, 359 theDetails.size() - inflatedResourceCount); 360 return; 361 } 362 } 363 ourLog.debug( 364 "Exiting binary data inflation having inflated everything.[byteCount={}, resourcesInflated={}, resourcesSkipped=0]", 365 cumulativeInflatedBytes, 366 inflatedResourceCount); 367 } 368 369 private long inflateBinariesInResource(long theCumulativeInflatedBytes, IBaseResource theResource) 370 throws IOException { 371 IIdType resourceId = theResource.getIdElement(); 372 List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource); 373 for (IBinaryTarget nextTarget : attachments) { 374 Optional<String> attachmentId = nextTarget.getAttachmentId(); 375 if (attachmentId.isPresent()) { 376 377 StoredDetails blobDetails = myBinaryStorageSvc.fetchBlobDetails(resourceId, attachmentId.get()); 378 if (blobDetails == null) { 379 String msg = myCtx.getLocalizer().getMessage(BinaryAccessProvider.class, "unknownBlobId"); 380 throw new InvalidRequestException(Msg.code(1330) + msg); 381 } 382 383 if ((theCumulativeInflatedBytes + blobDetails.getBytes()) < myAutoInflateBinariesMaximumBytes) { 384 byte[] bytes = myBinaryStorageSvc.fetchBlob(resourceId, attachmentId.get()); 385 nextTarget.setData(bytes); 386 theCumulativeInflatedBytes += blobDetails.getBytes(); 387 } 388 } 389 } 390 return theCumulativeInflatedBytes; 391 } 392 393 @Nonnull 394 private List<IBinaryTarget> recursivelyScanResourceForBinaryData(IBaseResource theResource) { 395 List<IBinaryTarget> binaryTargets = new ArrayList<>(); 396 myCtx.newTerser().visit(theResource, new IModelVisitor2() { 397 @Override 398 public boolean acceptElement( 399 IBase theElement, 400 List<IBase> theContainingElementPath, 401 List<BaseRuntimeChildDefinition> theChildDefinitionPath, 402 List<BaseRuntimeElementDefinition<?>> theElementDefinitionPath) { 403 404 if (theElement.getClass().equals(myBinaryType)) { 405 IBase parent = theContainingElementPath.get(theContainingElementPath.size() - 2); 406 Optional<IBinaryTarget> binaryTarget = myBinaryAccessProvider.toBinaryTarget(parent); 407 binaryTarget.ifPresent(binaryTargets::add); 408 } 409 return true; 410 } 411 }); 412 return binaryTargets; 413 } 414 415 public void setAllowAutoInflateBinaries(boolean theAllowAutoInflateBinaries) { 416 myAllowAutoInflateBinaries = theAllowAutoInflateBinaries; 417 } 418 419 public boolean isAllowAutoInflateBinaries() { 420 return myAllowAutoInflateBinaries; 421 } 422 423 private static class DeferredBinaryTarget { 424 private final String myBlobId; 425 private final IBinaryTarget myBinaryTarget; 426 private final InputStream myDataStream; 427 private boolean myBlobIdPrefixHookApplied; 428 429 private DeferredBinaryTarget(String theBlobId, IBinaryTarget theBinaryTarget, byte[] theData) { 430 myBlobId = theBlobId; 431 myBinaryTarget = theBinaryTarget; 432 myDataStream = new ByteArrayInputStream(theData); 433 } 434 435 String getBlobId() { 436 return myBlobId; 437 } 438 439 IBinaryTarget getBinaryTarget() { 440 return myBinaryTarget; 441 } 442 443 InputStream getDataStream() { 444 return myDataStream; 445 } 446 447 boolean isBlobIdPrefixHookApplied() { 448 return myBlobIdPrefixHookApplied; 449 } 450 451 void setBlobIdPrefixHookApplied(boolean theBlobIdPrefixHookApplied) { 452 myBlobIdPrefixHookApplied = theBlobIdPrefixHookApplied; 453 } 454 } 455}