Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao.expunge;
021
022import ca.uhn.fhir.interceptor.api.HookParams;
023import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
024import ca.uhn.fhir.interceptor.api.Pointcut;
025import ca.uhn.fhir.interceptor.model.RequestPartitionId;
026import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
027import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity;
028import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity;
029import ca.uhn.fhir.jpa.entity.BulkImportJobEntity;
030import ca.uhn.fhir.jpa.entity.BulkImportJobFileEntity;
031import ca.uhn.fhir.jpa.entity.MdmLink;
032import ca.uhn.fhir.jpa.entity.PartitionEntity;
033import ca.uhn.fhir.jpa.entity.Search;
034import ca.uhn.fhir.jpa.entity.SearchInclude;
035import ca.uhn.fhir.jpa.entity.SearchResult;
036import ca.uhn.fhir.jpa.entity.SubscriptionTable;
037import ca.uhn.fhir.jpa.entity.TermCodeSystem;
038import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
039import ca.uhn.fhir.jpa.entity.TermConcept;
040import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
041import ca.uhn.fhir.jpa.entity.TermConceptMap;
042import ca.uhn.fhir.jpa.entity.TermConceptMapGroup;
043import ca.uhn.fhir.jpa.entity.TermConceptMapGroupElement;
044import ca.uhn.fhir.jpa.entity.TermConceptMapGroupElementTarget;
045import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
046import ca.uhn.fhir.jpa.entity.TermConceptProperty;
047import ca.uhn.fhir.jpa.entity.TermValueSet;
048import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
049import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
050import ca.uhn.fhir.jpa.model.entity.IndexedSearchParamIdentity;
051import ca.uhn.fhir.jpa.model.entity.NpmPackageEntity;
052import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionEntity;
053import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionResourceEntity;
054import ca.uhn.fhir.jpa.model.entity.ResourceHistoryProvenanceEntity;
055import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
056import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTag;
057import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique;
058import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboTokenNonUnique;
059import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamCoords;
060import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
061import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamNumber;
062import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
063import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantityNormalized;
064import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
065import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
066import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
067import ca.uhn.fhir.jpa.model.entity.ResourceLink;
068import ca.uhn.fhir.jpa.model.entity.ResourceSearchUrlEntity;
069import ca.uhn.fhir.jpa.model.entity.ResourceTable;
070import ca.uhn.fhir.jpa.model.entity.ResourceTag;
071import ca.uhn.fhir.jpa.model.entity.SearchParamPresentEntity;
072import ca.uhn.fhir.jpa.model.entity.TagDefinition;
073import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
074import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
075import ca.uhn.fhir.jpa.util.MemoryCacheService;
076import ca.uhn.fhir.rest.api.server.RequestDetails;
077import ca.uhn.fhir.rest.server.provider.ProviderConstants;
078import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
079import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
080import ca.uhn.fhir.util.StopWatch;
081import jakarta.annotation.Nullable;
082import jakarta.persistence.EntityManager;
083import jakarta.persistence.PersistenceContext;
084import jakarta.persistence.PersistenceContextType;
085import jakarta.persistence.Query;
086import jakarta.persistence.TypedQuery;
087import jakarta.persistence.criteria.CriteriaBuilder;
088import jakarta.persistence.criteria.CriteriaQuery;
089import jakarta.persistence.metamodel.EntityType;
090import jakarta.persistence.metamodel.Metamodel;
091import jakarta.persistence.metamodel.SingularAttribute;
092import org.slf4j.Logger;
093import org.slf4j.LoggerFactory;
094import org.springframework.beans.factory.annotation.Autowired;
095import org.springframework.stereotype.Service;
096import org.springframework.transaction.annotation.Propagation;
097import org.springframework.util.comparator.Comparators;
098
099import java.util.ArrayList;
100import java.util.Iterator;
101import java.util.List;
102import java.util.Set;
103import java.util.concurrent.atomic.AtomicInteger;
104
105@Service
106public class ExpungeEverythingService implements IExpungeEverythingService {
107        private static final Logger ourLog = LoggerFactory.getLogger(ExpungeEverythingService.class);
108
109        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
110        protected EntityManager myEntityManager;
111
112        @Autowired
113        protected IInterceptorBroadcaster myInterceptorBroadcaster;
114
115        @Autowired
116        private HapiTransactionService myTxService;
117
118        @Autowired
119        private MemoryCacheService myMemoryCacheService;
120
121        @Autowired
122        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
123
124        private int deletedResourceEntityCount;
125
126        @Override
127        public void expungeEverything(@Nullable RequestDetails theRequest) {
128
129                final AtomicInteger counter = new AtomicInteger();
130
131                // Notify Interceptors about pre-action call
132                IInterceptorBroadcaster compositeBroadcaster =
133                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
134                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_EVERYTHING)) {
135                        HookParams hooks = new HookParams()
136                                        .add(AtomicInteger.class, counter)
137                                        .add(RequestDetails.class, theRequest)
138                                        .addIfMatchesType(ServletRequestDetails.class, theRequest);
139                        compositeBroadcaster.callHooks(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_EVERYTHING, hooks);
140                }
141
142                ourLog.info("BEGINNING GLOBAL $expunge");
143                Propagation propagation = Propagation.REQUIRES_NEW;
144                RequestPartitionId requestPartitionId =
145                                myRequestPartitionHelperSvc.determineReadPartitionForRequestForServerOperation(
146                                                theRequest, ProviderConstants.OPERATION_EXPUNGE);
147
148                deleteAll(theRequest, propagation, requestPartitionId, counter);
149
150                purgeAllCaches();
151
152                ourLog.info("COMPLETED GLOBAL $expunge - Deleted {} rows", counter.get());
153        }
154
155        protected void deleteAll(
156                        @Nullable RequestDetails theRequest,
157                        Propagation propagation,
158                        RequestPartitionId requestPartitionId,
159                        AtomicInteger counter) {
160                myTxService
161                                .withRequest(theRequest)
162                                .withPropagation(propagation)
163                                .withRequestPartitionId(requestPartitionId)
164                                .execute(() -> {
165                                        counter.addAndGet(doExpungeEverythingQuery(
166                                                        "UPDATE " + TermCodeSystem.class.getSimpleName() + " d SET d.myCurrentVersion = null"));
167                                });
168                counter.addAndGet(
169                                expungeEverythingByTypeWithoutPurging(theRequest, Batch2WorkChunkEntity.class, requestPartitionId));
170                counter.addAndGet(
171                                expungeEverythingByTypeWithoutPurging(theRequest, Batch2JobInstanceEntity.class, requestPartitionId));
172                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
173                                theRequest, NpmPackageVersionResourceEntity.class, requestPartitionId));
174                counter.addAndGet(
175                                expungeEverythingByTypeWithoutPurging(theRequest, NpmPackageVersionEntity.class, requestPartitionId));
176                counter.addAndGet(
177                                expungeEverythingByTypeWithoutPurging(theRequest, NpmPackageEntity.class, requestPartitionId));
178                counter.addAndGet(
179                                expungeEverythingByTypeWithoutPurging(theRequest, SearchParamPresentEntity.class, requestPartitionId));
180                counter.addAndGet(
181                                expungeEverythingByTypeWithoutPurging(theRequest, BulkImportJobFileEntity.class, requestPartitionId));
182                counter.addAndGet(
183                                expungeEverythingByTypeWithoutPurging(theRequest, BulkImportJobEntity.class, requestPartitionId));
184                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
185                                theRequest, ResourceIndexedSearchParamDate.class, requestPartitionId));
186                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
187                                theRequest, ResourceIndexedSearchParamNumber.class, requestPartitionId));
188                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
189                                theRequest, ResourceIndexedSearchParamQuantity.class, requestPartitionId));
190                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
191                                theRequest, ResourceIndexedSearchParamQuantityNormalized.class, requestPartitionId));
192                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
193                                theRequest, ResourceIndexedSearchParamString.class, requestPartitionId));
194                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
195                                theRequest, ResourceIndexedSearchParamToken.class, requestPartitionId));
196                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
197                                theRequest, ResourceIndexedSearchParamUri.class, requestPartitionId));
198                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
199                                theRequest, ResourceIndexedSearchParamCoords.class, requestPartitionId));
200                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
201                                theRequest, ResourceIndexedComboStringUnique.class, requestPartitionId));
202                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
203                                theRequest, ResourceIndexedComboTokenNonUnique.class, requestPartitionId));
204                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
205                                theRequest, IndexedSearchParamIdentity.class, requestPartitionId));
206                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceLink.class, requestPartitionId));
207                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, SearchResult.class, requestPartitionId));
208                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, SearchInclude.class, requestPartitionId));
209                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
210                                theRequest, TermValueSetConceptDesignation.class, requestPartitionId));
211                counter.addAndGet(
212                                expungeEverythingByTypeWithoutPurging(theRequest, TermValueSetConcept.class, requestPartitionId));
213                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermValueSet.class, requestPartitionId));
214                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
215                                theRequest, TermConceptParentChildLink.class, requestPartitionId));
216                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
217                                theRequest, TermConceptMapGroupElementTarget.class, requestPartitionId));
218                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
219                                theRequest, TermConceptMapGroupElement.class, requestPartitionId));
220                counter.addAndGet(
221                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptMapGroup.class, requestPartitionId));
222                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermConceptMap.class, requestPartitionId));
223                counter.addAndGet(
224                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptProperty.class, requestPartitionId));
225                counter.addAndGet(
226                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptDesignation.class, requestPartitionId));
227                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermConcept.class, requestPartitionId));
228                myTxService
229                                .withRequest(theRequest)
230                                .withPropagation(propagation)
231                                .withRequestPartitionId(requestPartitionId)
232                                .execute(() -> {
233                                        for (TermCodeSystem next : myEntityManager
234                                                        .createQuery("SELECT c FROM " + TermCodeSystem.class.getName() + " c", TermCodeSystem.class)
235                                                        .getResultList()) {
236                                                next.setCurrentVersion(null);
237                                                myEntityManager.merge(next);
238                                        }
239                                });
240                counter.addAndGet(
241                                expungeEverythingByTypeWithoutPurging(theRequest, TermCodeSystemVersion.class, requestPartitionId));
242                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermCodeSystem.class, requestPartitionId));
243                counter.addAndGet(
244                                expungeEverythingByTypeWithoutPurging(theRequest, SubscriptionTable.class, requestPartitionId));
245                counter.addAndGet(
246                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceHistoryTag.class, requestPartitionId));
247                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceTag.class, requestPartitionId));
248                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TagDefinition.class, requestPartitionId));
249                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
250                                theRequest, ResourceHistoryProvenanceEntity.class, requestPartitionId));
251                counter.addAndGet(
252                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceHistoryTable.class, requestPartitionId));
253                counter.addAndGet(
254                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceSearchUrlEntity.class, requestPartitionId));
255
256                int counterBefore = counter.get();
257                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceTable.class, requestPartitionId));
258                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, PartitionEntity.class, requestPartitionId));
259
260                deletedResourceEntityCount = counter.get() - counterBefore;
261
262                myTxService
263                                .withRequest(theRequest)
264                                .withPropagation(propagation)
265                                .withRequestPartitionId(requestPartitionId)
266                                .execute(() -> {
267                                        counter.addAndGet(doExpungeEverythingQuery("DELETE from " + Search.class.getSimpleName() + " d"));
268                                });
269        }
270
271        @Override
272        public int getExpungeDeletedEntityCount() {
273                return deletedResourceEntityCount;
274        }
275
276        private void purgeAllCaches() {
277                myMemoryCacheService.invalidateAllCaches();
278        }
279
280        protected <T> int expungeEverythingByTypeWithoutPurging(
281                        RequestDetails theRequest, Class<T> theEntityType, RequestPartitionId theRequestPartitionId) {
282                HapiTransactionService.noTransactionAllowed();
283
284                int outcome = 0;
285                while (true) {
286                        StopWatch sw = new StopWatch();
287
288                        int count = myTxService
289                                        .withRequest(theRequest)
290                                        .withPropagation(Propagation.REQUIRES_NEW)
291                                        .withRequestPartitionId(theRequestPartitionId)
292                                        .execute(() -> {
293
294                                                /*
295                                                 * This method uses a nice efficient mechanism where we figure out the PID datatype
296                                                 * and load only the PIDs and delete by PID for all resource types except ResourceTable.
297                                                 * We delete ResourceTable using the entitymanager so that Hibernate Search knows to
298                                                 * delete the corresponding records it manages in ElasticSearch. See
299                                                 * FhirResourceDaoR4SearchWithElasticSearchIT for a test that fails without the
300                                                 * block below.
301                                                 */
302                                                if (ResourceTable.class.equals(theEntityType)) {
303                                                        CriteriaBuilder cb = myEntityManager.getCriteriaBuilder();
304                                                        CriteriaQuery<?> cq = cb.createQuery(theEntityType);
305                                                        cq.from(theEntityType);
306                                                        TypedQuery<?> query = myEntityManager.createQuery(cq);
307                                                        query.setMaxResults(800);
308                                                        List<?> results = query.getResultList();
309                                                        for (Object result : results) {
310                                                                myEntityManager.remove(result);
311                                                        }
312                                                        return results.size();
313                                                }
314
315                                                Metamodel metamodel = myEntityManager.getMetamodel();
316                                                EntityType<T> entity = metamodel.entity(theEntityType);
317                                                Set<SingularAttribute<? super T, ?>> singularAttributes = entity.getSingularAttributes();
318                                                List<String> idProperty = new ArrayList<>();
319                                                for (SingularAttribute<? super T, ?> singularAttribute : singularAttributes) {
320                                                        if (singularAttribute.isId()) {
321                                                                idProperty.add(singularAttribute.getName());
322                                                        }
323                                                }
324                                                idProperty.sort(Comparators.comparable());
325                                                String idPropertyNames = String.join(",", idProperty);
326
327                                                Query nativeQuery = myEntityManager.createQuery(
328                                                                "SELECT (" + idPropertyNames + ") FROM " + theEntityType.getSimpleName());
329
330                                                // Each ID is 2 parameters in DB partition mode, so this
331                                                // is the maximum we should allow
332                                                nativeQuery.setMaxResults(SearchBuilder.getMaximumPageSize() / 2);
333
334                                                List pids = nativeQuery.getResultList();
335                                                if (pids.isEmpty()) {
336                                                        return 0;
337                                                }
338
339                                                StringBuilder deleteBuilder = new StringBuilder();
340                                                deleteBuilder.append("DELETE FROM ");
341                                                deleteBuilder.append(theEntityType.getSimpleName());
342                                                deleteBuilder.append(" WHERE (");
343                                                deleteBuilder.append(idPropertyNames);
344                                                deleteBuilder.append(") IN ");
345                                                if (idProperty.size() > 1) {
346                                                        deleteBuilder.append('(');
347                                                        for (Iterator<Object> iter = pids.iterator(); iter.hasNext(); ) {
348                                                                Object[] pid = (Object[]) iter.next();
349                                                                deleteBuilder.append('(');
350                                                                for (int i = 0; i < pid.length; i++) {
351                                                                        if (i > 0) {
352                                                                                deleteBuilder.append(',');
353                                                                        }
354                                                                        deleteBuilder.append(pid[i]);
355                                                                }
356                                                                deleteBuilder.append(')');
357                                                                if (iter.hasNext()) {
358                                                                        deleteBuilder.append(',');
359                                                                }
360                                                        }
361                                                        deleteBuilder.append(')');
362                                                } else {
363                                                        deleteBuilder.append("(:pids)");
364                                                }
365                                                String deleteSql = deleteBuilder.toString();
366                                                nativeQuery = myEntityManager.createQuery(deleteSql);
367                                                if (idProperty.size() == 1) {
368                                                        nativeQuery.setParameter("pids", pids);
369                                                }
370                                                nativeQuery.executeUpdate();
371                                                return pids.size();
372                                        });
373
374                        outcome += count;
375                        if (count == 0) {
376                                break;
377                        }
378
379                        ourLog.info("Have deleted {} entities of type {} in {}", outcome, theEntityType.getSimpleName(), sw);
380                }
381                return outcome;
382        }
383
384        @Override
385        public int expungeEverythingByType(Class<?> theEntityType) {
386                int result = expungeEverythingByTypeWithoutPurging(null, theEntityType, RequestPartitionId.allPartitions());
387                purgeAllCaches();
388                return result;
389        }
390
391        @Override
392        public int expungeEverythingMdmLinks() {
393                return expungeEverythingByType(MdmLink.class);
394        }
395
396        private int doExpungeEverythingQuery(String theQuery) {
397                StopWatch sw = new StopWatch();
398                int outcome = myEntityManager.createQuery(theQuery).executeUpdate();
399                ourLog.debug("SqlQuery affected {} rows in {}: {}", outcome, sw, theQuery);
400                return outcome;
401        }
402}