Source code

001/*-
002 * #%L
003 * HAPI FHIR Storage api
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.binary.interceptor;
021
022import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
023import ca.uhn.fhir.context.BaseRuntimeElementDefinition;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.i18n.Msg;
026import ca.uhn.fhir.interceptor.api.Hook;
027import ca.uhn.fhir.interceptor.api.HookParams;
028import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
029import ca.uhn.fhir.interceptor.api.Interceptor;
030import ca.uhn.fhir.interceptor.api.Pointcut;
031import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
032import ca.uhn.fhir.jpa.binary.api.IBinaryTarget;
033import ca.uhn.fhir.jpa.binary.api.StoredDetails;
034import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider;
035import ca.uhn.fhir.jpa.binary.svc.BaseBinaryStorageSvcImpl;
036import ca.uhn.fhir.jpa.model.util.JpaConstants;
037import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
038import ca.uhn.fhir.rest.api.server.RequestDetails;
039import ca.uhn.fhir.rest.api.server.storage.TransactionDetails;
040import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
041import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
042import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
043import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
044import ca.uhn.fhir.util.HapiExtensions;
045import ca.uhn.fhir.util.IModelVisitor2;
046import jakarta.annotation.Nonnull;
047import org.apache.commons.io.FileUtils;
048import org.apache.commons.lang3.StringUtils;
049import org.hl7.fhir.instance.model.api.IBase;
050import org.hl7.fhir.instance.model.api.IBaseHasExtensions;
051import org.hl7.fhir.instance.model.api.IBaseResource;
052import org.hl7.fhir.instance.model.api.IIdType;
053import org.hl7.fhir.instance.model.api.IPrimitiveType;
054import org.hl7.fhir.r4.model.IdType;
055import org.slf4j.Logger;
056import org.slf4j.LoggerFactory;
057import org.springframework.beans.factory.annotation.Autowired;
058
059import java.io.ByteArrayInputStream;
060import java.io.IOException;
061import java.io.InputStream;
062import java.util.ArrayList;
063import java.util.HashSet;
064import java.util.List;
065import java.util.Optional;
066import java.util.Set;
067import java.util.concurrent.atomic.AtomicInteger;
068import java.util.stream.Collectors;
069
070import static ca.uhn.fhir.util.HapiExtensions.EXT_EXTERNALIZED_BINARY_ID;
071import static org.apache.commons.lang3.StringUtils.isNotBlank;
072
073@Interceptor
074public class BinaryStorageInterceptor<T extends IPrimitiveType<byte[]>> {
075
076        private static final Logger ourLog = LoggerFactory.getLogger(BinaryStorageInterceptor.class);
077
078        @Autowired
079        private IBinaryStorageSvc myBinaryStorageSvc;
080
081        private final FhirContext myCtx;
082
083        @Autowired
084        private BinaryAccessProvider myBinaryAccessProvider;
085
086        @Autowired
087        private IInterceptorBroadcaster myInterceptorBroadcaster;
088
089        private Class<T> myBinaryType;
090        private String myDeferredListKey;
091        private long myAutoInflateBinariesMaximumBytes = 10 * FileUtils.ONE_MB;
092        private boolean myAllowAutoInflateBinaries = true;
093
094        public BinaryStorageInterceptor(FhirContext theCtx) {
095                myCtx = theCtx;
096                BaseRuntimeElementDefinition<?> base64Binary = myCtx.getElementDefinition("base64Binary");
097                assert base64Binary != null;
098                myBinaryType = (Class<T>) base64Binary.getImplementingClass();
099                myDeferredListKey = getClass().getName() + "_" + hashCode() + "_DEFERRED_LIST";
100        }
101
102        /**
103         * Any externalized binaries will be rehydrated if their size is below this thhreshold when
104         * reading the resource back. Default is 10MB.
105         */
106        public long getAutoInflateBinariesMaximumSize() {
107                return myAutoInflateBinariesMaximumBytes;
108        }
109
110        /**
111         * Any externalized binaries will be rehydrated if their size is below this thhreshold when
112         * reading the resource back. Default is 10MB.
113         */
114        public void setAutoInflateBinariesMaximumSize(long theAutoInflateBinariesMaximumBytes) {
115                myAutoInflateBinariesMaximumBytes = theAutoInflateBinariesMaximumBytes;
116        }
117
118        @Hook(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_RESOURCE)
119        public void expungeResource(AtomicInteger theCounter, IBaseResource theResource) {
120
121                List<? extends IBase> binaryElements =
122                                myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType);
123
124                List<String> attachmentIds = binaryElements.stream()
125                                .flatMap(t -> ((IBaseHasExtensions) t).getExtension().stream())
126                                .filter(t -> HapiExtensions.EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl()))
127                                .map(t -> ((IPrimitiveType<?>) t.getValue()).getValueAsString())
128                                .collect(Collectors.toList());
129
130                for (String next : attachmentIds) {
131                        myBinaryStorageSvc.expungeBinaryContent(theResource.getIdElement(), next);
132                        theCounter.incrementAndGet();
133
134                        ourLog.info(
135                                        "Deleting binary blob {} because resource {} is being expunged",
136                                        next,
137                                        theResource.getIdElement().getValue());
138                }
139        }
140
141        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED)
142        public void extractLargeBinariesBeforeCreate(
143                        RequestDetails theRequestDetails,
144                        TransactionDetails theTransactionDetails,
145                        IBaseResource theResource,
146                        Pointcut thePointcut)
147                        throws IOException {
148                extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut);
149        }
150
151        @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED)
152        public void extractLargeBinariesBeforeUpdate(
153                        RequestDetails theRequestDetails,
154                        TransactionDetails theTransactionDetails,
155                        IBaseResource thePreviousResource,
156                        IBaseResource theResource,
157                        Pointcut thePointcut)
158                        throws IOException {
159                blockIllegalExternalBinaryIds(thePreviousResource, theResource);
160                extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut);
161        }
162
163        /**
164         * Don't allow clients to submit resources with binary storage attachments declared unless the ID was already in the
165         * resource. In other words, only HAPI itself may add a binary storage ID extension to a resource unless that
166         * extension was already present.
167         */
168        private void blockIllegalExternalBinaryIds(IBaseResource thePreviousResource, IBaseResource theResource) {
169                Set<String> existingBinaryIds = new HashSet<>();
170                if (thePreviousResource != null) {
171                        List<T> base64fields =
172                                        myCtx.newTerser().getAllPopulatedChildElementsOfType(thePreviousResource, myBinaryType);
173                        for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
174                                if (nextBase64 instanceof IBaseHasExtensions) {
175                                        ((IBaseHasExtensions) nextBase64)
176                                                        .getExtension().stream()
177                                                                        .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
178                                                                        .filter(t -> EXT_EXTERNALIZED_BINARY_ID.equals(t.getUrl()))
179                                                                        .map(t -> (IPrimitiveType<?>) t.getValue())
180                                                                        .map(IPrimitiveType::getValueAsString)
181                                                                        .filter(StringUtils::isNotBlank)
182                                                                        .forEach(existingBinaryIds::add);
183                                }
184                        }
185                }
186
187                List<T> base64fields = myCtx.newTerser().getAllPopulatedChildElementsOfType(theResource, myBinaryType);
188                for (IPrimitiveType<byte[]> nextBase64 : base64fields) {
189                        if (nextBase64 instanceof IBaseHasExtensions) {
190                                Optional<String> hasExternalizedBinaryReference = ((IBaseHasExtensions) nextBase64)
191                                                .getExtension().stream()
192                                                                .filter(t -> t.getUserData(JpaConstants.EXTENSION_EXT_SYSTEMDEFINED) == null)
193                                                                .filter(t -> t.getUrl().equals(EXT_EXTERNALIZED_BINARY_ID))
194                                                                .map(t -> (IPrimitiveType<?>) t.getValue())
195                                                                .map(IPrimitiveType::getValueAsString)
196                                                                .filter(StringUtils::isNotBlank)
197                                                                .filter(t -> !existingBinaryIds.contains(t))
198                                                                .findFirst();
199
200                                if (hasExternalizedBinaryReference.isPresent()) {
201                                        String msg = myCtx.getLocalizer()
202                                                        .getMessage(
203                                                                        BinaryStorageInterceptor.class,
204                                                                        "externalizedBinaryStorageExtensionFoundInRequestBody",
205                                                                        EXT_EXTERNALIZED_BINARY_ID,
206                                                                        hasExternalizedBinaryReference.get());
207                                        throw new InvalidRequestException(Msg.code(1329) + msg);
208                                }
209                        }
210                }
211        }
212
213        private void extractLargeBinaries(
214                        RequestDetails theRequestDetails,
215                        TransactionDetails theTransactionDetails,
216                        IBaseResource theResource,
217                        Pointcut thePointcut)
218                        throws IOException {
219
220                IIdType resourceId = theResource.getIdElement();
221                if (!resourceId.hasResourceType() && resourceId.hasIdPart()) {
222                        String resourceType = myCtx.getResourceType(theResource);
223                        resourceId = new IdType(resourceType + "/" + resourceId.getIdPart());
224                }
225
226                List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource);
227                for (IBinaryTarget nextTarget : attachments) {
228                        byte[] data = nextTarget.getData();
229                        if (data != null && data.length > 0) {
230
231                                long nextPayloadLength = data.length;
232                                String nextContentType = nextTarget.getContentType();
233                                boolean shouldStoreBlob =
234                                                myBinaryStorageSvc.shouldStoreBinaryContent(nextPayloadLength, resourceId, nextContentType);
235                                if (shouldStoreBlob) {
236
237                                        String newBinaryContentId;
238                                        if (thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED) {
239                                                ByteArrayInputStream inputStream = new ByteArrayInputStream(data);
240                                                StoredDetails storedDetails = myBinaryStorageSvc.storeBinaryContent(
241                                                                resourceId, null, nextContentType, inputStream, theRequestDetails);
242                                                newBinaryContentId = storedDetails.getBinaryContentId();
243                                        } else {
244                                                assert thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED : thePointcut.name();
245                                                newBinaryContentId = myBinaryStorageSvc.newBinaryContentId();
246
247                                                String prefix = invokeAssignBinaryContentPrefix(theRequestDetails, theResource);
248                                                if (isNotBlank(prefix)) {
249                                                        newBinaryContentId = prefix + newBinaryContentId;
250                                                }
251                                                if (myBinaryStorageSvc.isValidBinaryContentId(newBinaryContentId)) {
252                                                        List<DeferredBinaryTarget> deferredBinaryTargets =
253                                                                        getOrCreateDeferredBinaryStorageList(theResource);
254                                                        DeferredBinaryTarget newDeferredBinaryTarget =
255                                                                        new DeferredBinaryTarget(newBinaryContentId, nextTarget, data);
256                                                        deferredBinaryTargets.add(newDeferredBinaryTarget);
257                                                        newDeferredBinaryTarget.setBlobIdPrefixHookApplied(true);
258                                                } else {
259                                                        throw new InternalErrorException(Msg.code(2341)
260                                                                        + "Invalid binaryContent ID for backing storage service.[binaryContentId="
261                                                                        + newBinaryContentId + ",service="
262                                                                        + myBinaryStorageSvc.getClass().getName() + "]");
263                                                }
264                                        }
265
266                                        myBinaryAccessProvider.replaceDataWithExtension(nextTarget, newBinaryContentId);
267                                }
268                        }
269                }
270        }
271
272        /**
273         * This invokes the {@link Pointcut#STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX} hook and returns the prefix to use for the blob ID, or null if there are no implementers.
274         * @return A string, which will be used to prefix the blob ID. May be null.
275         */
276        private String invokeAssignBinaryContentPrefix(RequestDetails theRequest, IBaseResource theResource) {
277                IInterceptorBroadcaster compositeBroadcaster =
278                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
279
280                // TODO: to be removed when pointcut STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX has exceeded the grace period
281                boolean hasStorageBinaryAssignBlobIdPrefixHooks =
282                                compositeBroadcaster.hasHooks(Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX);
283
284                boolean hasStorageBinaryAssignBinaryContentIdPrefixHooks =
285                                compositeBroadcaster.hasHooks(Pointcut.STORAGE_BINARY_ASSIGN_BINARY_CONTENT_ID_PREFIX);
286
287                if (!(hasStorageBinaryAssignBlobIdPrefixHooks || hasStorageBinaryAssignBinaryContentIdPrefixHooks)) {
288                        return null;
289                }
290
291                HookParams params =
292                                new HookParams().add(RequestDetails.class, theRequest).add(IBaseResource.class, theResource);
293
294                BaseBinaryStorageSvcImpl.setBinaryContentIdPrefixApplied(theRequest);
295
296                Pointcut pointcutToInvoke = Pointcut.STORAGE_BINARY_ASSIGN_BINARY_CONTENT_ID_PREFIX;
297
298                // TODO: to be removed when pointcut STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX has exceeded the grace period
299                if (hasStorageBinaryAssignBlobIdPrefixHooks) {
300                        pointcutToInvoke = Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX;
301                }
302
303                return (String) compositeBroadcaster.callHooksAndReturnObject(pointcutToInvoke, params);
304        }
305
306        @Nonnull
307        @SuppressWarnings("unchecked")
308        private List<DeferredBinaryTarget> getOrCreateDeferredBinaryStorageList(IBaseResource theResource) {
309                Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey());
310                if (deferredBinaryTargetList == null) {
311                        deferredBinaryTargetList = new ArrayList<>();
312                        theResource.setUserData(getDeferredListKey(), deferredBinaryTargetList);
313                }
314                return (List<DeferredBinaryTarget>) deferredBinaryTargetList;
315        }
316
317        @SuppressWarnings("unchecked")
318        @Hook(Pointcut.STORAGE_PRECOMMIT_RESOURCE_CREATED)
319        public void storeLargeBinariesBeforeCreatePersistence(
320                        TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut)
321                        throws IOException {
322                if (theResource == null) {
323                        return;
324                }
325                Object deferredBinaryTargetList = theResource.getUserData(getDeferredListKey());
326
327                if (deferredBinaryTargetList != null) {
328                        IIdType resourceId = theResource.getIdElement();
329                        for (DeferredBinaryTarget next : (List<DeferredBinaryTarget>) deferredBinaryTargetList) {
330                                String blobId = next.getBlobId();
331                                IBinaryTarget target = next.getBinaryTarget();
332                                InputStream dataStream = next.getDataStream();
333                                String contentType = target.getContentType();
334                                RequestDetails requestDetails = initRequestDetails(next);
335                                myBinaryStorageSvc.storeBinaryContent(resourceId, blobId, contentType, dataStream, requestDetails);
336                        }
337                }
338        }
339
340        private RequestDetails initRequestDetails(DeferredBinaryTarget theDeferredBinaryTarget) {
341                ServletRequestDetails requestDetails = new ServletRequestDetails();
342                if (theDeferredBinaryTarget.isBlobIdPrefixHookApplied()) {
343                        BaseBinaryStorageSvcImpl.setBinaryContentIdPrefixApplied(requestDetails);
344                }
345                return requestDetails;
346        }
347
348        public String getDeferredListKey() {
349                return myDeferredListKey;
350        }
351
352        @Hook(Pointcut.STORAGE_PRESHOW_RESOURCES)
353        public void preShow(IPreResourceShowDetails theDetails) throws IOException {
354                if (!isAllowAutoInflateBinaries()) {
355                        return;
356                }
357
358                long cumulativeInflatedBytes = 0;
359                int inflatedResourceCount = 0;
360
361                for (IBaseResource nextResource : theDetails) {
362                        if (nextResource == null) {
363                                ourLog.warn(
364                                                "Received a null resource during STORAGE_PRESHOW_RESOURCES. This is a bug and should be reported. Skipping resource.");
365                                continue;
366                        }
367                        cumulativeInflatedBytes = inflateBinariesInResource(cumulativeInflatedBytes, nextResource);
368                        inflatedResourceCount += 1;
369                        if (cumulativeInflatedBytes >= myAutoInflateBinariesMaximumBytes) {
370                                ourLog.debug(
371                                                "Exiting binary data inflation early.[byteCount={}, resourcesInflated={}, resourcesSkipped={}]",
372                                                cumulativeInflatedBytes,
373                                                inflatedResourceCount,
374                                                theDetails.size() - inflatedResourceCount);
375                                return;
376                        }
377                }
378                ourLog.debug(
379                                "Exiting binary data inflation having inflated everything.[byteCount={}, resourcesInflated={}, resourcesSkipped=0]",
380                                cumulativeInflatedBytes,
381                                inflatedResourceCount);
382        }
383
384        private long inflateBinariesInResource(long theCumulativeInflatedBytes, IBaseResource theResource)
385                        throws IOException {
386                IIdType resourceId = theResource.getIdElement();
387                List<IBinaryTarget> attachments = recursivelyScanResourceForBinaryData(theResource);
388                for (IBinaryTarget nextTarget : attachments) {
389                        Optional<String> attachmentId = nextTarget.getAttachmentId();
390                        if (attachmentId.isPresent()) {
391
392                                StoredDetails blobDetails =
393                                                myBinaryStorageSvc.fetchBinaryContentDetails(resourceId, attachmentId.get());
394                                if (blobDetails == null) {
395                                        String msg = myCtx.getLocalizer().getMessage(BinaryAccessProvider.class, "unknownBlobId");
396                                        throw new InvalidRequestException(Msg.code(1330) + msg);
397                                }
398
399                                if ((theCumulativeInflatedBytes + blobDetails.getBytes()) < myAutoInflateBinariesMaximumBytes) {
400                                        byte[] bytes = myBinaryStorageSvc.fetchBinaryContent(resourceId, attachmentId.get());
401                                        nextTarget.setData(bytes);
402                                        theCumulativeInflatedBytes += blobDetails.getBytes();
403                                }
404                        }
405                }
406                return theCumulativeInflatedBytes;
407        }
408
409        @Nonnull
410        private List<IBinaryTarget> recursivelyScanResourceForBinaryData(IBaseResource theResource) {
411                List<IBinaryTarget> binaryTargets = new ArrayList<>();
412                myCtx.newTerser().visit(theResource, new IModelVisitor2() {
413                        @Override
414                        public boolean acceptElement(
415                                        IBase theElement,
416                                        List<IBase> theContainingElementPath,
417                                        List<BaseRuntimeChildDefinition> theChildDefinitionPath,
418                                        List<BaseRuntimeElementDefinition<?>> theElementDefinitionPath) {
419
420                                if (theElement.getClass().equals(myBinaryType)) {
421                                        IBase parent = theContainingElementPath.get(theContainingElementPath.size() - 2);
422                                        Optional<IBinaryTarget> binaryTarget = myBinaryAccessProvider.toBinaryTarget(parent);
423                                        binaryTarget.ifPresent(binaryTargets::add);
424                                }
425                                return true;
426                        }
427                });
428                return binaryTargets;
429        }
430
431        public void setAllowAutoInflateBinaries(boolean theAllowAutoInflateBinaries) {
432                myAllowAutoInflateBinaries = theAllowAutoInflateBinaries;
433        }
434
435        public boolean isAllowAutoInflateBinaries() {
436                return myAllowAutoInflateBinaries;
437        }
438
439        private static class DeferredBinaryTarget {
440                private final String myBlobId;
441                private final IBinaryTarget myBinaryTarget;
442                private final InputStream myDataStream;
443                private boolean myBlobIdPrefixHookApplied;
444
445                private DeferredBinaryTarget(String theBlobId, IBinaryTarget theBinaryTarget, byte[] theData) {
446                        myBlobId = theBlobId;
447                        myBinaryTarget = theBinaryTarget;
448                        myDataStream = new ByteArrayInputStream(theData);
449                }
450
451                String getBlobId() {
452                        return myBlobId;
453                }
454
455                IBinaryTarget getBinaryTarget() {
456                        return myBinaryTarget;
457                }
458
459                InputStream getDataStream() {
460                        return myDataStream;
461                }
462
463                boolean isBlobIdPrefixHookApplied() {
464                        return myBlobIdPrefixHookApplied;
465                }
466
467                void setBlobIdPrefixHookApplied(boolean theBlobIdPrefixHookApplied) {
468                        myBlobIdPrefixHookApplied = theBlobIdPrefixHookApplied;
469                }
470        }
471}