Source code

001/*-
002 * #%L
003 * HAPI FHIR Storage api
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.binary.interceptor;
021
022import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
023import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.Hook;
027import ca.uhn.fhir.interceptor.api.HookParams;
028import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
029import ca.uhn.fhir.interceptor.api.Interceptor;
030import ca.uhn.fhir.interceptor.api.Pointcut;
031import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
032import ca.uhn.fhir.jpa.binary.api.IBinaryTarget;
033import ca.uhn.fhir.jpa.binary.api.StoredDetails;
034import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider;
035import ca.uhn.fhir.jpa.binary.svc.BaseBinaryStorageSvcImpl;
036import ca.uhn.fhir.jpa.model.util.JpaConstants;
037import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
038import ca.uhn.fhir.rest.api.server.RequestDetails;
039import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
040import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
041import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
042import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
043import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
044import ca.uhn.fhir.util.HapiExtensions;
045import ca.uhn.fhir.util.IModelVisitor2;
046import jakarta.annotation.Nonnull;
047import org.apache.commons.io.FileUtils;
048import org.apache.commons.lang3.StringUtils;
049import org.hl7.fhir.instance.model.api.IBase;
050import org.hl7.fhir.instance.model.api.IBaseHasExtensions;
051import org.hl7.fhir.instance.model.api.IBaseResource;
052import org.hl7.fhir.instance.model.api.IIdType;
053import org.hl7.fhir.instance.model.api.IPrimitiveType;
054import org.hl7.fhir.r4.model.IdType;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057import org.springframework.beans.factory.annotation.Autowired;
058
059import java.io.ByteArrayInputStream;
060import java.io.IOException;
061import java.io.InputStream;
062import java.util.ArrayList;
063import java.util.HashSet;
064import java.util.List;
065import java.util.Optional;
066import java.util.Set;
067import java.util.concurrent.atomic.AtomicInteger;
068import java.util.stream.Collectors;
069
070import static ca.uhn.fhir.util.HapiExtensions.EXT_EXTERNALIZED_BINARY_ID;
071import static org.apache.commons.lang3.StringUtils.isNotBlank;
072
073@Interceptor
074public class BinaryStorageInterceptor<T extends IPrimitiveType<byte[]>> {
075
076        private static final Logger ourLog = LoggerFactory.getLogger(BinaryStorageInterceptor.class);
077
078        @Autowired
079        private IBinaryStorageSvc myBinaryStorageSvc;
080
081        private final FhirContext myCtx;
082
083        @Autowired
084        private BinaryAccessProvider myBinaryAccessProvider;
085
086        @Autowired
087        private IInterceptorBroadcaster myInterceptorBroadcaster;
088
089        private Class<T> myBinaryType;
090        private String myDeferredListKey;
091        private long myAutoInflateBinariesMaximumBytes = 10 * FileUtils.ONE_MB;
092        private boolean myAllowAutoInflateBinaries = true;
093
094        public BinaryStorageInterceptor(FhirContext theCtx) {
095                myCtx = theCtx;
096                BaseRuntimeElementDefinition<?> base64Binary = myCtx.getElementDefinition("base64Binary");
097                assert base64Binary != null;
098                myBinaryType = (Class<T>) base64Binary.getImplementingClass();
099                myDeferredListKey = getClass().getName() + "_" + hashCode() + "_DEFERRED_LIST";
100        }
101
102        /**
103         * Any externalized binaries will be rehydrated if their size is below this thhreshold when
104         * reading the resource back. Default is 10MB.
105         */
106        public long getAutoInflateBinariesMaximumSize() {
107                return myAutoInflateBinariesMaximumBytes;
108        }
109
110        /**
111         * Any externalized binaries will be rehydrated if their size is below this thhreshold when
112         * reading the resource back. Default is 10MB.
113         */
114        public void setAutoInflateBinariesMaximumSize(long theAutoInflateBinariesMaximumBytes) {
115                myAutoInflateBinariesMaximumBytes = theAutoInflateBinariesMaximumBytes;
116        }
117
118        @Hook(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_RESOURCE)
119        public void expungeResource(AtomicInteger theCounter, IBaseResource theResource) {
120
121                List<? extends IBase> binaryElements =
122                                myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType);
123
124                List<String> attachmentIds = binaryElements.stream()
125                                .flatMap(t -> ((IBaseHasExtensions) t).getExtension().stream())
126                                .filter(t -> HapiExtensions.EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl()))
127                                .map(t -> ((IPrimitiveType<?>) t.getValue()).getValueAsString())
128                                .collect(Collectors.toList());
129
130                for (String next : attachmentIds) {
131                        myBinaryStorageSvc.expungeBinaryContent(theResource.getIdElement(), next);
132                        theCounter.incrementAndGet();
133
134                        ourLog.info(
135                                        "Deleting binary blob {} because resource {} is being expunged",
136                                        next,
137                                        theResource.getIdElement().getValue());
138                }
139        }
140
141        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED)
142        public void extractLargeBinariesBeforeCreate(
143                        RequestDetails theRequestDetails,
144                        TransactionDetails theTransactionDetails,
145                        IBaseResource theResource,
146                        Pointcut thePointcut)
147                        throws IOException {
148                extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut);
149        }
150
151        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED)
152        public void extractLargeBinariesBeforeUpdate(
153                        RequestDetails theRequestDetails,
154                        TransactionDetails theTransactionDetails,
155                        IBaseResource thePreviousResource,
156                        IBaseResource theResource,
157                        Pointcut thePointcut)
158                        throws IOException {
159                blockIllegalExternalBinaryIds(thePreviousResource, theResource);
160                extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut);
161        }
162
163        /**
164         * Don't allow clients to submit resources with binary storage attachments declared unless the ID was already in the
165         * resource. In other words, only HAPI itself may add a binary storage ID extension to a resource unless that
166         * extension was already present.
167         */
168        private void blockIllegalExternalBinaryIds(IBaseResource thePreviousResource, IBaseResource theResource) {
169                Set<String> existingBinaryIds = new HashSet<>();
170                if (thePreviousResource != null) {
171                        List<T> base64fields =
172                                        myCtx.newTerser().getAllPopulatedChildElementsOfType(thePreviousResource, myBinaryType);
173                        for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
174                                if (nextBase64 instanceof IBaseHasExtensions) {
175                                        ((IBaseHasExtensions) nextBase64)
176                                                        .getExtension().stream()
177                                                                        .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
178                                                                        .filter(t -> EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl()))
179                                                                        .map(t -> (IPrimitiveType<?>) t.getValue())
180                                                                        .map(IPrimitiveType::getValueAsString)
181                                                                        .filter(StringUtils::isNotBlank)
182                                                                        .forEach(existingBinaryIds::add);
183                                }
184                        }
185                }
186
187                List<T> base64fields = myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType);
188                for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
189                        if (nextBase64 instanceof IBaseHasExtensions) {
190                                Optional<String> hasExternalizedBinaryReference = ((IBaseHasExtensions) nextBase64)
191                                                .getExtension().stream()
192                                                                .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
193                                                                .filter(t -> t.getUrl().equals(EXT_EXTERNALIZED_BINARY_ID))
194                                                                .map(t -> (IPrimitiveType<?>) t.getValue())
195                                                                .map(IPrimitiveType::getValueAsString)
196                                                                .filter(StringUtils::isNotBlank)
197                                                                .filter(t -> !existingBinaryIds.contains(t))
198                                                                .findFirst();
199
200                                if (hasExternalizedBinaryReference.isPresent()) {
201                                        String msg = myCtx.getLocalizer()
202                                                        .getMessage(
203                                                                        BinaryStorageInterceptor.class,
204                                                                        "externalizedBinaryStorageExtensionFoundInRequestBody",
205                                                                        EXT_EXTERNALIZED_BINARY_ID,
206                                                                        hasExternalizedBinaryReference.get());
207                                        throw new InvalidRequestException(Msg.code(1329) + msg);
208                                }
209                        }
210                }
211        }
212
213        private void extractLargeBinaries(
214                        RequestDetails theRequestDetails,
215                        TransactionDetails theTransactionDetails,
216                        IBaseResource theResource,
217                        Pointcut thePointcut)
218                        throws IOException {
219
220                IIdType resourceId = theResource.getIdElement();
221                if (!resourceId.hasResourceType() && resourceId.hasIdPart()) {
222                        String resourceType = myCtx.getResourceType(theResource);
223                        resourceId = new IdType(resourceType + "/" + resourceId.getIdPart());
224                }
225
226                List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource);
227                for (IBinaryTarget nextTarget : attachments) {
228                        byte[] data = nextTarget.getData();
229                        if (data != null && data.length > 0) {
230
231                                long nextPayloadLength = data.length;
232                                String nextContentType = nextTarget.getContentType();
233                                boolean shouldStoreBlob =
234                                                myBinaryStorageSvc.shouldStoreBinaryContent(nextPayloadLength, resourceId, nextContentType);
235                                if (shouldStoreBlob) {
236
237                                        String newBinaryContentId;
238                                        if (thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED) {
239                                                ByteArrayInputStream inputStream = new ByteArrayInputStream(data);
240                                                StoredDetails storedDetails = myBinaryStorageSvc.storeBinaryContent(
241                                                                resourceId, null, nextContentType, inputStream, theRequestDetails);
242                                                newBinaryContentId = storedDetails.getBinaryContentId();
243                                        } else {
244                                                assert thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED : thePointcut.name();
245                                                newBinaryContentId = myBinaryStorageSvc.newBinaryContentId();
246
247                                                String prefix = invokeAssignBinaryContentPrefix(theRequestDetails, theResource);
248                                                if (isNotBlank(prefix)) {
249                                                        newBinaryContentId = prefix + newBinaryContentId;
250                                                }
251                                                if (myBinaryStorageSvc.isValidBinaryContentId(newBinaryContentId)) {
252                                                        List<DeferredBinaryTarget> deferredBinaryTargets =
253                                                                        getOrCreateDeferredBinaryStorageList(theResource);
254                                                        DeferredBinaryTarget newDeferredBinaryTarget =
255                                                                        new DeferredBinaryTarget(newBinaryContentId, nextTarget, data);
256                                                        deferredBinaryTargets.add(newDeferredBinaryTarget);
257                                                        newDeferredBinaryTarget.setBlobIdPrefixHookApplied(true);
258                                                } else {
259                                                        throw new InternalErrorException(Msg.code(2341)
260                                                                        + "Invalid binaryContent ID for backing storage service.[binaryContentId="
261                                                                        + newBinaryContentId + ",service="
262                                                                        + myBinaryStorageSvc.getClass().getName() + "]");
263                                                }
264                                        }
265
266                                        myBinaryAccessProvider.replaceDataWithExtension(nextTarget, newBinaryContentId);
267                                }
268                        }
269                }
270        }
271
272        /**
273         * This invokes the {@link Pointcut#STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX} hook and returns the prefix to use for the blob ID, or null if there are no implementers.
274         * @return A string, which will be used to prefix the blob ID. May be null.
275         */
276        private String invokeAssignBinaryContentPrefix(RequestDetails theRequest, IBaseResource theResource) {
277                // TODO: to be removed when pointcut STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX has exceeded the grace period
278                boolean hasStorageBinaryAssignBlobIdPrefixHooks = CompositeInterceptorBroadcaster.hasHooks(
279                                Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, myInterceptorBroadcaster, theRequest);
280
281                boolean hasStorageBinaryAssignBinaryContentIdPrefixHooks = CompositeInterceptorBroadcaster.hasHooks(
282                                Pointcut.STORAGE_BINARY_ASSIGN_BINARY_CONTENT_ID_PREFIX, myInterceptorBroadcaster, theRequest);
283
284                if (!(hasStorageBinaryAssignBlobIdPrefixHooks || hasStorageBinaryAssignBinaryContentIdPrefixHooks)) {
285                        return null;
286                }
287
288                HookParams params =
289                                new HookParams().add(RequestDetails.class, theRequest).add(IBaseResource.class, theResource);
290
291                BaseBinaryStorageSvcImpl.setBinaryContentIdPrefixApplied(theRequest);
292
293                Pointcut pointcutToInvoke = Pointcut.STORAGE_BINARY_ASSIGN_BINARY_CONTENT_ID_PREFIX;
294
295                // TODO: to be removed when pointcut STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX has exceeded the grace period
296                if (hasStorageBinaryAssignBlobIdPrefixHooks) {
297                        pointcutToInvoke = Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX;
298                }
299
300                return (String) CompositeInterceptorBroadcaster.doCallHooksAndReturnObject(
301                                myInterceptorBroadcaster, theRequest, pointcutToInvoke, params);
302        }
303
304        @Nonnull
305        @SuppressWarnings("unchecked")
306        private List<DeferredBinaryTarget> getOrCreateDeferredBinaryStorageList(IBaseResource theResource) {
307                Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey());
308                if (deferredBinaryTargetList == null) {
309                        deferredBinaryTargetList = new ArrayList<>();
310                        theResource.setUserData(getDeferredListKey(), deferredBinaryTargetList);
311                }
312                return (List<DeferredBinaryTarget>) deferredBinaryTargetList;
313        }
314
315        @SuppressWarnings("unchecked")
316        @Hook(Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED)
317        public void storeLargeBinariesBeforeCreatePersistence(
318                        TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut)
319                        throws IOException {
320                if (theResource == null) {
321                        return;
322                }
323                Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey());
324
325                if (deferredBinaryTargetList != null) {
326                        IIdType resourceId = theResource.getIdElement();
327                        for (DeferredBinaryTarget next : (List<DeferredBinaryTarget>) deferredBinaryTargetList) {
328                                String blobId = next.getBlobId();
329                                IBinaryTarget target = next.getBinaryTarget();
330                                InputStream dataStream = next.getDataStream();
331                                String contentType = target.getContentType();
332                                RequestDetails requestDetails = initRequestDetails(next);
333                                myBinaryStorageSvc.storeBinaryContent(resourceId, blobId, contentType, dataStream, requestDetails);
334                        }
335                }
336        }
337
338        private RequestDetails initRequestDetails(DeferredBinaryTarget theDeferredBinaryTarget) {
339                ServletRequestDetails requestDetails = new ServletRequestDetails();
340                if (theDeferredBinaryTarget.isBlobIdPrefixHookApplied()) {
341                        BaseBinaryStorageSvcImpl.setBinaryContentIdPrefixApplied(requestDetails);
342                }
343                return requestDetails;
344        }
345
346        public String getDeferredListKey() {
347                return myDeferredListKey;
348        }
349
350        @Hook(Pointcut.STORAGE_PRESHOW_RESOURCES)
351        public void preShow(IPreResourceShowDetails theDetails) throws IOException {
352                if (!isAllowAutoInflateBinaries()) {
353                        return;
354                }
355
356                long cumulativeInflatedBytes = 0;
357                int inflatedResourceCount = 0;
358
359                for (IBaseResource nextResource : theDetails) {
360                        if (nextResource == null) {
361                                ourLog.warn(
362                                                "Received a null resource during STORAGE_PRESHOW_RESOURCES. This is a bug and should be reported. Skipping resource.");
363                                continue;
364                        }
365                        cumulativeInflatedBytes = inflateBinariesInResource(cumulativeInflatedBytes, nextResource);
366                        inflatedResourceCount += 1;
367                        if (cumulativeInflatedBytes >= myAutoInflateBinariesMaximumBytes) {
368                                ourLog.debug(
369                                                "Exiting binary data inflation early.[byteCount={}, resourcesInflated={}, resourcesSkipped={}]",
370                                                cumulativeInflatedBytes,
371                                                inflatedResourceCount,
372                                                theDetails.size() - inflatedResourceCount);
373                                return;
374                        }
375                }
376                ourLog.debug(
377                                "Exiting binary data inflation having inflated everything.[byteCount={}, resourcesInflated={}, resourcesSkipped=0]",
378                                cumulativeInflatedBytes,
379                                inflatedResourceCount);
380        }
381
382        private long inflateBinariesInResource(long theCumulativeInflatedBytes, IBaseResource theResource)
383                        throws IOException {
384                IIdType resourceId = theResource.getIdElement();
385                List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource);
386                for (IBinaryTarget nextTarget : attachments) {
387                        Optional<String> attachmentId = nextTarget.getAttachmentId();
388                        if (attachmentId.isPresent()) {
389
390                                StoredDetails blobDetails =
391                                                myBinaryStorageSvc.fetchBinaryContentDetails(resourceId, attachmentId.get());
392                                if (blobDetails == null) {
393                                        String msg = myCtx.getLocalizer().getMessage(BinaryAccessProvider.class, "unknownBlobId");
394                                        throw new InvalidRequestException(Msg.code(1330) + msg);
395                                }
396
397                                if ((theCumulativeInflatedBytes + blobDetails.getBytes()) < myAutoInflateBinariesMaximumBytes) {
398                                        byte[] bytes = myBinaryStorageSvc.fetchBinaryContent(resourceId, attachmentId.get());
399                                        nextTarget.setData(bytes);
400                                        theCumulativeInflatedBytes += blobDetails.getBytes();
401                                }
402                        }
403                }
404                return theCumulativeInflatedBytes;
405        }
406
407        @Nonnull
408        private List<IBinaryTarget> recursivelyScanResourceForBinaryData(IBaseResource theResource) {
409                List<IBinaryTarget> binaryTargets = new ArrayList<>();
410                myCtx.newTerser().visit(theResource, new IModelVisitor2() {
411                        @Override
412                        public boolean acceptElement(
413                                        IBase theElement,
414                                        List<IBase> theContainingElementPath,
415                                        List<BaseRuntimeChildDefinition> theChildDefinitionPath,
416                                        List<BaseRuntimeElementDefinition<?>> theElementDefinitionPath) {
417
418                                if (theElement.getClass().equals(myBinaryType)) {
419                                        IBase parent = theContainingElementPath.get(theContainingElementPath.size() - 2);
420                                        Optional<IBinaryTarget> binaryTarget = myBinaryAccessProvider.toBinaryTarget(parent);
421                                        binaryTarget.ifPresent(binaryTargets::add);
422                                }
423                                return true;
424                        }
425                });
426                return binaryTargets;
427        }
428
429        public void setAllowAutoInflateBinaries(boolean theAllowAutoInflateBinaries) {
430                myAllowAutoInflateBinaries = theAllowAutoInflateBinaries;
431        }
432
433        public boolean isAllowAutoInflateBinaries() {
434                return myAllowAutoInflateBinaries;
435        }
436
437        private static class DeferredBinaryTarget {
438                private final String myBlobId;
439                private final IBinaryTarget myBinaryTarget;
440                private final InputStream myDataStream;
441                private boolean myBlobIdPrefixHookApplied;
442
443                private DeferredBinaryTarget(String theBlobId, IBinaryTarget theBinaryTarget, byte[] theData) {
444                        myBlobId = theBlobId;
445                        myBinaryTarget = theBinaryTarget;
446                        myDataStream = new ByteArrayInputStream(theData);
447                }
448
449                String getBlobId() {
450                        return myBlobId;
451                }
452
453                IBinaryTarget getBinaryTarget() {
454                        return myBinaryTarget;
455                }
456
457                InputStream getDataStream() {
458                        return myDataStream;
459                }
460
461                boolean isBlobIdPrefixHookApplied() {
462                        return myBlobIdPrefixHookApplied;
463                }
464
465                void setBlobIdPrefixHookApplied(boolean theBlobIdPrefixHookApplied) {
466                        myBlobIdPrefixHookApplied = theBlobIdPrefixHookApplied;
467                }
468        }
469}