001/*-
002 * #%L
003 * HAPI FHIR Storage api
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.binary.interceptor;
021
022import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
023import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.Hook;
027import ca.uhn.fhir.interceptor.api.HookParams;
028import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
029import ca.uhn.fhir.interceptor.api.Interceptor;
030import ca.uhn.fhir.interceptor.api.Pointcut;
031import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
032import ca.uhn.fhir.jpa.binary.api.IBinaryTarget;
033import ca.uhn.fhir.jpa.binary.api.StoredDetails;
034import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider;
035import ca.uhn.fhir.jpa.binary.svc.BaseBinaryStorageSvcImpl;
036import ca.uhn.fhir.jpa.model.util.JpaConstants;
037import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
038import ca.uhn.fhir.rest.api.server.RequestDetails;
039import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
040import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
041import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
042import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
043import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
044import ca.uhn.fhir.util.HapiExtensions;
045import ca.uhn.fhir.util.IModelVisitor2;
046import jakarta.annotation.Nonnull;
047import org.apache.commons.io.FileUtils;
048import org.apache.commons.lang3.StringUtils;
049import org.hl7.fhir.instance.model.api.IBase;
050import org.hl7.fhir.instance.model.api.IBaseHasExtensions;
051import org.hl7.fhir.instance.model.api.IBaseResource;
052import org.hl7.fhir.instance.model.api.IIdType;
053import org.hl7.fhir.instance.model.api.IPrimitiveType;
054import org.hl7.fhir.r4.model.IdType;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057import org.springframework.beans.factory.annotation.Autowired;
058
059import java.awt.*;
060import java.io.ByteArrayInputStream;
061import java.io.IOException;
062import java.io.InputStream;
063import java.util.ArrayList;
064import java.util.HashSet;
065import java.util.List;
066import java.util.Optional;
067import java.util.Set;
068import java.util.concurrent.atomic.AtomicInteger;
069import java.util.stream.Collectors;
070
071import static ca.uhn.fhir.util.HapiExtensions.EXT_EXTERNALIZED_BINARY_ID;
072import static org.apache.commons.lang3.StringUtils.isNotBlank;
073
074@Interceptor
075public class BinaryStorageInterceptor<T extends IPrimitiveType<byte[]>> {
076
077        private static final Logger ourLog = LoggerFactory.getLogger(BinaryStorageInterceptor.class);
078
079        @Autowired
080        private IBinaryStorageSvc myBinaryStorageSvc;
081
082        private final FhirContext myCtx;
083
084        @Autowired
085        private BinaryAccessProvider myBinaryAccessProvider;
086
087        @Autowired
088        private IInterceptorBroadcaster myInterceptorBroadcaster;
089
090        private Class<T> myBinaryType;
091        private String myDeferredListKey;
092        private long myAutoInflateBinariesMaximumBytes = 10 * FileUtils.ONE_MB;
093        private boolean myAllowAutoInflateBinaries = true;
094
095        public BinaryStorageInterceptor(FhirContext theCtx) {
096                myCtx = theCtx;
097                BaseRuntimeElementDefinition<?> base64Binary = myCtx.getElementDefinition("base64Binary");
098                assert base64Binary != null;
099                myBinaryType = (Class<T>) base64Binary.getImplementingClass();
100                myDeferredListKey = getClass().getName() + "_" + hashCode() + "_DEFERRED_LIST";
101        }
102
103        /**
104         * Any externalized binaries will be rehydrated if their size is below this thhreshold when
105         * reading the resource back. Default is 10MB.
106         */
107        public long getAutoInflateBinariesMaximumSize() {
108                return myAutoInflateBinariesMaximumBytes;
109        }
110
111        /**
112         * Any externalized binaries will be rehydrated if their size is below this thhreshold when
113         * reading the resource back. Default is 10MB.
114         */
115        public void setAutoInflateBinariesMaximumSize(long theAutoInflateBinariesMaximumBytes) {
116                myAutoInflateBinariesMaximumBytes = theAutoInflateBinariesMaximumBytes;
117        }
118
119        @Hook(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_RESOURCE)
120        public void expungeResource(AtomicInteger theCounter, IBaseResource theResource) {
121
122                List<? extends IBase> binaryElements =
123                                myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType);
124
125                List<String> attachmentIds = binaryElements.stream()
126                                .flatMap(t -> ((IBaseHasExtensions) t).getExtension().stream())
127                                .filter(t -> HapiExtensions.EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl()))
128                                .map(t -> ((IPrimitiveType<?>) t.getValue()).getValueAsString())
129                                .collect(Collectors.toList());
130
131                for (String next : attachmentIds) {
132                        myBinaryStorageSvc.expungeBlob(theResource.getIdElement(), next);
133                        theCounter.incrementAndGet();
134
135                        ourLog.info(
136                                        "Deleting binary blob {} because resource {} is being expunged",
137                                        next,
138                                        theResource.getIdElement().getValue());
139                }
140        }
141
142        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED)
143        public void extractLargeBinariesBeforeCreate(
144                        RequestDetails theRequestDetails,
145                        TransactionDetails theTransactionDetails,
146                        IBaseResource theResource,
147                        Pointcut thePointcut)
148                        throws IOException {
149                extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut);
150        }
151
152        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED)
153        public void extractLargeBinariesBeforeUpdate(
154                        RequestDetails theRequestDetails,
155                        TransactionDetails theTransactionDetails,
156                        IBaseResource thePreviousResource,
157                        IBaseResource theResource,
158                        Pointcut thePointcut)
159                        throws IOException {
160                blockIllegalExternalBinaryIds(thePreviousResource, theResource);
161                extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut);
162        }
163
164        /**
165         * Don't allow clients to submit resources with binary storage attachments declared unless the ID was already in the
166         * resource. In other words, only HAPI itself may add a binary storage ID extension to a resource unless that
167         * extension was already present.
168         */
169        private void blockIllegalExternalBinaryIds(IBaseResource thePreviousResource, IBaseResource theResource) {
170                Set<String> existingBinaryIds = new HashSet<>();
171                if (thePreviousResource != null) {
172                        List<T> base64fields =
173                                        myCtx.newTerser().getAllPopulatedChildElementsOfType(thePreviousResource, myBinaryType);
174                        for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
175                                if (nextBase64 instanceof IBaseHasExtensions) {
176                                        ((IBaseHasExtensions) nextBase64)
177                                                        .getExtension().stream()
178                                                                        .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
179                                                                        .filter(t -> EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl()))
180                                                                        .map(t -> (IPrimitiveType<?>) t.getValue())
181                                                                        .map(IPrimitiveType::getValueAsString)
182                                                                        .filter(StringUtils::isNotBlank)
183                                                                        .forEach(existingBinaryIds::add);
184                                }
185                        }
186                }
187
188                List<T> base64fields = myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType);
189                for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
190                        if (nextBase64 instanceof IBaseHasExtensions) {
191                                Optional<String> hasExternalizedBinaryReference = ((IBaseHasExtensions) nextBase64)
192                                                .getExtension().stream()
193                                                                .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
194                                                                .filter(t -> t.getUrl().equals(EXT_EXTERNALIZED_BINARY_ID))
195                                                                .map(t -> (IPrimitiveType<?>) t.getValue())
196                                                                .map(IPrimitiveType::getValueAsString)
197                                                                .filter(StringUtils::isNotBlank)
198                                                                .filter(t -> !existingBinaryIds.contains(t))
199                                                                .findFirst();
200
201                                if (hasExternalizedBinaryReference.isPresent()) {
202                                        String msg = myCtx.getLocalizer()
203                                                        .getMessage(
204                                                                        BinaryStorageInterceptor.class,
205                                                                        "externalizedBinaryStorageExtensionFoundInRequestBody",
206                                                                        EXT_EXTERNALIZED_BINARY_ID,
207                                                                        hasExternalizedBinaryReference.get());
208                                        throw new InvalidRequestException(Msg.code(1329) + msg);
209                                }
210                        }
211                }
212        }
213
214        private void extractLargeBinaries(
215                        RequestDetails theRequestDetails,
216                        TransactionDetails theTransactionDetails,
217                        IBaseResource theResource,
218                        Pointcut thePointcut)
219                        throws IOException {
220
221                IIdType resourceId = theResource.getIdElement();
222                if (!resourceId.hasResourceType() && resourceId.hasIdPart()) {
223                        String resourceType = myCtx.getResourceType(theResource);
224                        resourceId = new IdType(resourceType + "/" + resourceId.getIdPart());
225                }
226
227                List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource);
228                for (IBinaryTarget nextTarget : attachments) {
229                        byte[] data = nextTarget.getData();
230                        if (data != null && data.length > 0) {
231
232                                long nextPayloadLength = data.length;
233                                String nextContentType = nextTarget.getContentType();
234                                boolean shouldStoreBlob =
235                                                myBinaryStorageSvc.shouldStoreBlob(nextPayloadLength, resourceId, nextContentType);
236                                if (shouldStoreBlob) {
237
238                                        String newBlobId;
239                                        if (thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED) {
240                                                ByteArrayInputStream inputStream = new ByteArrayInputStream(data);
241                                                StoredDetails storedDetails = myBinaryStorageSvc.storeBlob(
242                                                                resourceId, null, nextContentType, inputStream, theRequestDetails);
243                                                newBlobId = storedDetails.getBlobId();
244                                        } else {
245                                                assert thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED : thePointcut.name();
246                                                newBlobId = myBinaryStorageSvc.newBlobId();
247
248                                                String prefix = invokeAssignBlobPrefix(theRequestDetails, theResource);
249                                                if (isNotBlank(prefix)) {
250                                                        newBlobId = prefix + newBlobId;
251                                                }
252                                                if (myBinaryStorageSvc.isValidBlobId(newBlobId)) {
253                                                        List<DeferredBinaryTarget> deferredBinaryTargets =
254                                                                        getOrCreateDeferredBinaryStorageList(theResource);
255                                                        DeferredBinaryTarget newDeferredBinaryTarget =
256                                                                        new DeferredBinaryTarget(newBlobId, nextTarget, data);
257                                                        deferredBinaryTargets.add(newDeferredBinaryTarget);
258                                                        newDeferredBinaryTarget.setBlobIdPrefixHookApplied(true);
259                                                } else {
260                                                        throw new InternalErrorException(Msg.code(2341)
261                                                                        + "Invalid blob ID for backing storage service.[blobId=" + newBlobId + ",service="
262                                                                        + myBinaryStorageSvc.getClass().getName() + "]");
263                                                }
264                                        }
265
266                                        myBinaryAccessProvider.replaceDataWithExtension(nextTarget, newBlobId);
267                                }
268                        }
269                }
270        }
271
272        /**
273         * This invokes the {@link Pointcut#STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX} hook and returns the prefix to use for the blob ID, or null if there are no implementers.
274         * @return A string, which will be used to prefix the blob ID. May be null.
275         */
276        private String invokeAssignBlobPrefix(RequestDetails theRequest, IBaseResource theResource) {
277                if (!CompositeInterceptorBroadcaster.hasHooks(
278                                Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, myInterceptorBroadcaster, theRequest)) {
279                        return null;
280                }
281
282                HookParams params =
283                                new HookParams().add(RequestDetails.class, theRequest).add(IBaseResource.class, theResource);
284
285                BaseBinaryStorageSvcImpl.setBlobIdPrefixApplied(theRequest);
286
287                return (String) CompositeInterceptorBroadcaster.doCallHooksAndReturnObject(
288                                myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, params);
289        }
290
291        @Nonnull
292        @SuppressWarnings("unchecked")
293        private List<DeferredBinaryTarget> getOrCreateDeferredBinaryStorageList(IBaseResource theResource) {
294                Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey());
295                if (deferredBinaryTargetList == null) {
296                        deferredBinaryTargetList = new ArrayList<>();
297                        theResource.setUserData(getDeferredListKey(), deferredBinaryTargetList);
298                }
299                return (List<DeferredBinaryTarget>) deferredBinaryTargetList;
300        }
301
302        @SuppressWarnings("unchecked")
303        @Hook(Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED)
304        public void storeLargeBinariesBeforeCreatePersistence(
305                        TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut)
306                        throws IOException {
307                if (theResource == null) {
308                        return;
309                }
310                Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey());
311
312                if (deferredBinaryTargetList != null) {
313                        IIdType resourceId = theResource.getIdElement();
314                        for (DeferredBinaryTarget next : (List<DeferredBinaryTarget>) deferredBinaryTargetList) {
315                                String blobId = next.getBlobId();
316                                IBinaryTarget target = next.getBinaryTarget();
317                                InputStream dataStream = next.getDataStream();
318                                String contentType = target.getContentType();
319                                RequestDetails requestDetails = initRequestDetails(next);
320                                myBinaryStorageSvc.storeBlob(resourceId, blobId, contentType, dataStream, requestDetails);
321                        }
322                }
323        }
324
325        private RequestDetails initRequestDetails(DeferredBinaryTarget theDeferredBinaryTarget) {
326                ServletRequestDetails requestDetails = new ServletRequestDetails();
327                if (theDeferredBinaryTarget.isBlobIdPrefixHookApplied()) {
328                        BaseBinaryStorageSvcImpl.setBlobIdPrefixApplied(requestDetails);
329                }
330                return requestDetails;
331        }
332
333        public String getDeferredListKey() {
334                return myDeferredListKey;
335        }
336
337        @Hook(Pointcut.STORAGE_PRESHOW_RESOURCES)
338        public void preShow(IPreResourceShowDetails theDetails) throws IOException {
339                if (!isAllowAutoInflateBinaries()) {
340                        return;
341                }
342
343                long cumulativeInflatedBytes = 0;
344                int inflatedResourceCount = 0;
345
346                for (IBaseResource nextResource : theDetails) {
347                        if (nextResource == null) {
348                                ourLog.warn(
349                                                "Received a null resource during STORAGE_PRESHOW_RESOURCES. This is a bug and should be reported. Skipping resource.");
350                                continue;
351                        }
352                        cumulativeInflatedBytes = inflateBinariesInResource(cumulativeInflatedBytes, nextResource);
353                        inflatedResourceCount += 1;
354                        if (cumulativeInflatedBytes >= myAutoInflateBinariesMaximumBytes) {
355                                ourLog.debug(
356                                                "Exiting binary data inflation early.[byteCount={}, resourcesInflated={}, resourcesSkipped={}]",
357                                                cumulativeInflatedBytes,
358                                                inflatedResourceCount,
359                                                theDetails.size() - inflatedResourceCount);
360                                return;
361                        }
362                }
363                ourLog.debug(
364                                "Exiting binary data inflation having inflated everything.[byteCount={}, resourcesInflated={}, resourcesSkipped=0]",
365                                cumulativeInflatedBytes,
366                                inflatedResourceCount);
367        }
368
369        private long inflateBinariesInResource(long theCumulativeInflatedBytes, IBaseResource theResource)
370                        throws IOException {
371                IIdType resourceId = theResource.getIdElement();
372                List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource);
373                for (IBinaryTarget nextTarget : attachments) {
374                        Optional<String> attachmentId = nextTarget.getAttachmentId();
375                        if (attachmentId.isPresent()) {
376
377                                StoredDetails blobDetails = myBinaryStorageSvc.fetchBlobDetails(resourceId, attachmentId.get());
378                                if (blobDetails == null) {
379                                        String msg = myCtx.getLocalizer().getMessage(BinaryAccessProvider.class, "unknownBlobId");
380                                        throw new InvalidRequestException(Msg.code(1330) + msg);
381                                }
382
383                                if ((theCumulativeInflatedBytes + blobDetails.getBytes()) < myAutoInflateBinariesMaximumBytes) {
384                                        byte[] bytes = myBinaryStorageSvc.fetchBlob(resourceId, attachmentId.get());
385                                        nextTarget.setData(bytes);
386                                        theCumulativeInflatedBytes += blobDetails.getBytes();
387                                }
388                        }
389                }
390                return theCumulativeInflatedBytes;
391        }
392
393        @Nonnull
394        private List<IBinaryTarget> recursivelyScanResourceForBinaryData(IBaseResource theResource) {
395                List<IBinaryTarget> binaryTargets = new ArrayList<>();
396                myCtx.newTerser().visit(theResource, new IModelVisitor2() {
397                        @Override
398                        public boolean acceptElement(
399                                        IBase theElement,
400                                        List<IBase> theContainingElementPath,
401                                        List<BaseRuntimeChildDefinition> theChildDefinitionPath,
402                                        List<BaseRuntimeElementDefinition<?>> theElementDefinitionPath) {
403
404                                if (theElement.getClass().equals(myBinaryType)) {
405                                        IBase parent = theContainingElementPath.get(theContainingElementPath.size() - 2);
406                                        Optional<IBinaryTarget> binaryTarget = myBinaryAccessProvider.toBinaryTarget(parent);
407                                        binaryTarget.ifPresent(binaryTargets::add);
408                                }
409                                return true;
410                        }
411                });
412                return binaryTargets;
413        }
414
415        public void setAllowAutoInflateBinaries(boolean theAllowAutoInflateBinaries) {
416                myAllowAutoInflateBinaries = theAllowAutoInflateBinaries;
417        }
418
419        public boolean isAllowAutoInflateBinaries() {
420                return myAllowAutoInflateBinaries;
421        }
422
423        private static class DeferredBinaryTarget {
424                private final String myBlobId;
425                private final IBinaryTarget myBinaryTarget;
426                private final InputStream myDataStream;
427                private boolean myBlobIdPrefixHookApplied;
428
429                private DeferredBinaryTarget(String theBlobId, IBinaryTarget theBinaryTarget, byte[] theData) {
430                        myBlobId = theBlobId;
431                        myBinaryTarget = theBinaryTarget;
432                        myDataStream = new ByteArrayInputStream(theData);
433                }
434
435                String getBlobId() {
436                        return myBlobId;
437                }
438
439                IBinaryTarget getBinaryTarget() {
440                        return myBinaryTarget;
441                }
442
443                InputStream getDataStream() {
444                        return myDataStream;
445                }
446
447                boolean isBlobIdPrefixHookApplied() {
448                        return myBlobIdPrefixHookApplied;
449                }
450
451                void setBlobIdPrefixHookApplied(boolean theBlobIdPrefixHookApplied) {
452                        myBlobIdPrefixHookApplied = theBlobIdPrefixHookApplied;
453                }
454        }
455}