Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2025 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao.expunge;
021
022import ca.uhn.fhir.interceptor.api.HookParams;
023import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
024import ca.uhn.fhir.interceptor.api.Pointcut;
025import ca.uhn.fhir.interceptor.model.RequestPartitionId;
026import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
027import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity;
028import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity;
029import ca.uhn.fhir.jpa.entity.BulkImportJobEntity;
030import ca.uhn.fhir.jpa.entity.BulkImportJobFileEntity;
031import ca.uhn.fhir.jpa.entity.MdmLink;
032import ca.uhn.fhir.jpa.entity.PartitionEntity;
033import ca.uhn.fhir.jpa.entity.Search;
034import ca.uhn.fhir.jpa.entity.SearchInclude;
035import ca.uhn.fhir.jpa.entity.SearchResult;
036import ca.uhn.fhir.jpa.entity.SubscriptionTable;
037import ca.uhn.fhir.jpa.entity.TermCodeSystem;
038import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
039import ca.uhn.fhir.jpa.entity.TermConcept;
040import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
041import ca.uhn.fhir.jpa.entity.TermConceptMap;
042import ca.uhn.fhir.jpa.entity.TermConceptMapGroup;
043import ca.uhn.fhir.jpa.entity.TermConceptMapGroupElement;
044import ca.uhn.fhir.jpa.entity.TermConceptMapGroupElementTarget;
045import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
046import ca.uhn.fhir.jpa.entity.TermConceptProperty;
047import ca.uhn.fhir.jpa.entity.TermValueSet;
048import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
049import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
050import ca.uhn.fhir.jpa.model.entity.NpmPackageEntity;
051import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionEntity;
052import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionResourceEntity;
053import ca.uhn.fhir.jpa.model.entity.ResourceHistoryProvenanceEntity;
054import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
055import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTag;
056import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique;
057import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboTokenNonUnique;
058import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamCoords;
059import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
060import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamNumber;
061import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
062import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantityNormalized;
063import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
064import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
065import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
066import ca.uhn.fhir.jpa.model.entity.ResourceLink;
067import ca.uhn.fhir.jpa.model.entity.ResourceSearchUrlEntity;
068import ca.uhn.fhir.jpa.model.entity.ResourceTable;
069import ca.uhn.fhir.jpa.model.entity.ResourceTag;
070import ca.uhn.fhir.jpa.model.entity.SearchParamPresentEntity;
071import ca.uhn.fhir.jpa.model.entity.TagDefinition;
072import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
073import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
074import ca.uhn.fhir.jpa.util.MemoryCacheService;
075import ca.uhn.fhir.rest.api.server.RequestDetails;
076import ca.uhn.fhir.rest.server.provider.ProviderConstants;
077import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
078import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
079import ca.uhn.fhir.util.StopWatch;
080import jakarta.annotation.Nullable;
081import jakarta.persistence.EntityManager;
082import jakarta.persistence.PersistenceContext;
083import jakarta.persistence.PersistenceContextType;
084import jakarta.persistence.Query;
085import jakarta.persistence.TypedQuery;
086import jakarta.persistence.criteria.CriteriaBuilder;
087import jakarta.persistence.criteria.CriteriaQuery;
088import jakarta.persistence.metamodel.EntityType;
089import jakarta.persistence.metamodel.Metamodel;
090import jakarta.persistence.metamodel.SingularAttribute;
091import org.slf4j.Logger;
092import org.slf4j.LoggerFactory;
093import org.springframework.beans.factory.annotation.Autowired;
094import org.springframework.stereotype.Service;
095import org.springframework.transaction.annotation.Propagation;
096import org.springframework.util.comparator.Comparators;
097
098import java.util.ArrayList;
099import java.util.Iterator;
100import java.util.List;
101import java.util.Set;
102import java.util.concurrent.atomic.AtomicInteger;
103
104@Service
105public class ExpungeEverythingService implements IExpungeEverythingService {
106        private static final Logger ourLog = LoggerFactory.getLogger(ExpungeEverythingService.class);
107
108        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
109        protected EntityManager myEntityManager;
110
111        @Autowired
112        protected IInterceptorBroadcaster myInterceptorBroadcaster;
113
114        @Autowired
115        private HapiTransactionService myTxService;
116
117        @Autowired
118        private MemoryCacheService myMemoryCacheService;
119
120        @Autowired
121        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
122
123        private int deletedResourceEntityCount;
124
125        @Override
126        public void expungeEverything(@Nullable RequestDetails theRequest) {
127
128                final AtomicInteger counter = new AtomicInteger();
129
130                // Notify Interceptors about pre-action call
131                IInterceptorBroadcaster compositeBroadcaster =
132                                CompositeInterceptorBroadcaster.newCompositeBroadcaster(myInterceptorBroadcaster, theRequest);
133                if (compositeBroadcaster.hasHooks(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_EVERYTHING)) {
134                        HookParams hooks = new HookParams()
135                                        .add(AtomicInteger.class, counter)
136                                        .add(RequestDetails.class, theRequest)
137                                        .addIfMatchesType(ServletRequestDetails.class, theRequest);
138                        compositeBroadcaster.callHooks(Pointcut.STORAGE_PRESTORAGE_EXPUNGE_EVERYTHING, hooks);
139                }
140
141                ourLog.info("BEGINNING GLOBAL $expunge");
142                Propagation propagation = Propagation.REQUIRES_NEW;
143                RequestPartitionId requestPartitionId =
144                                myRequestPartitionHelperSvc.determineReadPartitionForRequestForServerOperation(
145                                                theRequest, ProviderConstants.OPERATION_EXPUNGE);
146
147                deleteAll(theRequest, propagation, requestPartitionId, counter);
148
149                purgeAllCaches();
150
151                ourLog.info("COMPLETED GLOBAL $expunge - Deleted {} rows", counter.get());
152        }
153
154        protected void deleteAll(
155                        @Nullable RequestDetails theRequest,
156                        Propagation propagation,
157                        RequestPartitionId requestPartitionId,
158                        AtomicInteger counter) {
159                myTxService
160                                .withRequest(theRequest)
161                                .withPropagation(propagation)
162                                .withRequestPartitionId(requestPartitionId)
163                                .execute(() -> {
164                                        counter.addAndGet(doExpungeEverythingQuery(
165                                                        "UPDATE " + TermCodeSystem.class.getSimpleName() + " d SET d.myCurrentVersion = null"));
166                                });
167                counter.addAndGet(
168                                expungeEverythingByTypeWithoutPurging(theRequest, Batch2WorkChunkEntity.class, requestPartitionId));
169                counter.addAndGet(
170                                expungeEverythingByTypeWithoutPurging(theRequest, Batch2JobInstanceEntity.class, requestPartitionId));
171                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
172                                theRequest, NpmPackageVersionResourceEntity.class, requestPartitionId));
173                counter.addAndGet(
174                                expungeEverythingByTypeWithoutPurging(theRequest, NpmPackageVersionEntity.class, requestPartitionId));
175                counter.addAndGet(
176                                expungeEverythingByTypeWithoutPurging(theRequest, NpmPackageEntity.class, requestPartitionId));
177                counter.addAndGet(
178                                expungeEverythingByTypeWithoutPurging(theRequest, SearchParamPresentEntity.class, requestPartitionId));
179                counter.addAndGet(
180                                expungeEverythingByTypeWithoutPurging(theRequest, BulkImportJobFileEntity.class, requestPartitionId));
181                counter.addAndGet(
182                                expungeEverythingByTypeWithoutPurging(theRequest, BulkImportJobEntity.class, requestPartitionId));
183                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
184                                theRequest, ResourceIndexedSearchParamDate.class, requestPartitionId));
185                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
186                                theRequest, ResourceIndexedSearchParamNumber.class, requestPartitionId));
187                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
188                                theRequest, ResourceIndexedSearchParamQuantity.class, requestPartitionId));
189                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
190                                theRequest, ResourceIndexedSearchParamQuantityNormalized.class, requestPartitionId));
191                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
192                                theRequest, ResourceIndexedSearchParamString.class, requestPartitionId));
193                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
194                                theRequest, ResourceIndexedSearchParamToken.class, requestPartitionId));
195                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
196                                theRequest, ResourceIndexedSearchParamUri.class, requestPartitionId));
197                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
198                                theRequest, ResourceIndexedSearchParamCoords.class, requestPartitionId));
199                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
200                                theRequest, ResourceIndexedComboStringUnique.class, requestPartitionId));
201                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
202                                theRequest, ResourceIndexedComboTokenNonUnique.class, requestPartitionId));
203                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceLink.class, requestPartitionId));
204                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, SearchResult.class, requestPartitionId));
205                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, SearchInclude.class, requestPartitionId));
206                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
207                                theRequest, TermValueSetConceptDesignation.class, requestPartitionId));
208                counter.addAndGet(
209                                expungeEverythingByTypeWithoutPurging(theRequest, TermValueSetConcept.class, requestPartitionId));
210                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermValueSet.class, requestPartitionId));
211                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
212                                theRequest, TermConceptParentChildLink.class, requestPartitionId));
213                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
214                                theRequest, TermConceptMapGroupElementTarget.class, requestPartitionId));
215                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
216                                theRequest, TermConceptMapGroupElement.class, requestPartitionId));
217                counter.addAndGet(
218                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptMapGroup.class, requestPartitionId));
219                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermConceptMap.class, requestPartitionId));
220                counter.addAndGet(
221                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptProperty.class, requestPartitionId));
222                counter.addAndGet(
223                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptDesignation.class, requestPartitionId));
224                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermConcept.class, requestPartitionId));
225                myTxService
226                                .withRequest(theRequest)
227                                .withPropagation(propagation)
228                                .withRequestPartitionId(requestPartitionId)
229                                .execute(() -> {
230                                        for (TermCodeSystem next : myEntityManager
231                                                        .createQuery("SELECT c FROM " + TermCodeSystem.class.getName() + " c", TermCodeSystem.class)
232                                                        .getResultList()) {
233                                                next.setCurrentVersion(null);
234                                                myEntityManager.merge(next);
235                                        }
236                                });
237                counter.addAndGet(
238                                expungeEverythingByTypeWithoutPurging(theRequest, TermCodeSystemVersion.class, requestPartitionId));
239                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermCodeSystem.class, requestPartitionId));
240                counter.addAndGet(
241                                expungeEverythingByTypeWithoutPurging(theRequest, SubscriptionTable.class, requestPartitionId));
242                counter.addAndGet(
243                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceHistoryTag.class, requestPartitionId));
244                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceTag.class, requestPartitionId));
245                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TagDefinition.class, requestPartitionId));
246                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
247                                theRequest, ResourceHistoryProvenanceEntity.class, requestPartitionId));
248                counter.addAndGet(
249                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceHistoryTable.class, requestPartitionId));
250                counter.addAndGet(
251                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceSearchUrlEntity.class, requestPartitionId));
252
253                int counterBefore = counter.get();
254                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceTable.class, requestPartitionId));
255                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, PartitionEntity.class, requestPartitionId));
256
257                deletedResourceEntityCount = counter.get() - counterBefore;
258
259                myTxService
260                                .withRequest(theRequest)
261                                .withPropagation(propagation)
262                                .withRequestPartitionId(requestPartitionId)
263                                .execute(() -> {
264                                        counter.addAndGet(doExpungeEverythingQuery("DELETE from " + Search.class.getSimpleName() + " d"));
265                                });
266        }
267
268        @Override
269        public int getExpungeDeletedEntityCount() {
270                return deletedResourceEntityCount;
271        }
272
273        private void purgeAllCaches() {
274                myMemoryCacheService.invalidateAllCaches();
275        }
276
277        protected <T> int expungeEverythingByTypeWithoutPurging(
278                        RequestDetails theRequest, Class<T> theEntityType, RequestPartitionId theRequestPartitionId) {
279                HapiTransactionService.noTransactionAllowed();
280
281                int outcome = 0;
282                while (true) {
283                        StopWatch sw = new StopWatch();
284
285                        int count = myTxService
286                                        .withRequest(theRequest)
287                                        .withPropagation(Propagation.REQUIRES_NEW)
288                                        .withRequestPartitionId(theRequestPartitionId)
289                                        .execute(() -> {
290
291                                                /*
292                                                 * This method uses a nice efficient mechanism where we figure out the PID datatype
293                                                 * and load only the PIDs and delete by PID for all resource types except ResourceTable.
294                                                 * We delete ResourceTable using the entitymanager so that Hibernate Search knows to
295                                                 * delete the corresponding records it manages in ElasticSearch. See
296                                                 * FhirResourceDaoR4SearchWithElasticSearchIT for a test that fails without the
297                                                 * block below.
298                                                 */
299                                                if (ResourceTable.class.equals(theEntityType)) {
300                                                        CriteriaBuilder cb = myEntityManager.getCriteriaBuilder();
301                                                        CriteriaQuery<?> cq = cb.createQuery(theEntityType);
302                                                        cq.from(theEntityType);
303                                                        TypedQuery<?> query = myEntityManager.createQuery(cq);
304                                                        query.setMaxResults(800);
305                                                        List<?> results = query.getResultList();
306                                                        for (Object result : results) {
307                                                                myEntityManager.remove(result);
308                                                        }
309                                                        return results.size();
310                                                }
311
312                                                Metamodel metamodel = myEntityManager.getMetamodel();
313                                                EntityType<T> entity = metamodel.entity(theEntityType);
314                                                Set<SingularAttribute<? super T, ?>> singularAttributes = entity.getSingularAttributes();
315                                                List<String> idProperty = new ArrayList<>();
316                                                for (SingularAttribute<? super T, ?> singularAttribute : singularAttributes) {
317                                                        if (singularAttribute.isId()) {
318                                                                idProperty.add(singularAttribute.getName());
319                                                        }
320                                                }
321                                                idProperty.sort(Comparators.comparable());
322                                                String idPropertyNames = String.join(",", idProperty);
323
324                                                Query nativeQuery = myEntityManager.createQuery(
325                                                                "SELECT (" + idPropertyNames + ") FROM " + theEntityType.getSimpleName());
326
327                                                // Each ID is 2 parameters in DB partition mode, so this
328                                                // is the maximum we should allow
329                                                nativeQuery.setMaxResults(SearchBuilder.getMaximumPageSize() / 2);
330
331                                                List pids = nativeQuery.getResultList();
332                                                if (pids.isEmpty()) {
333                                                        return 0;
334                                                }
335
336                                                StringBuilder deleteBuilder = new StringBuilder();
337                                                deleteBuilder.append("DELETE FROM ");
338                                                deleteBuilder.append(theEntityType.getSimpleName());
339                                                deleteBuilder.append(" WHERE (");
340                                                deleteBuilder.append(idPropertyNames);
341                                                deleteBuilder.append(") IN ");
342                                                if (idProperty.size() > 1) {
343                                                        deleteBuilder.append('(');
344                                                        for (Iterator<Object> iter = pids.iterator(); iter.hasNext(); ) {
345                                                                Object[] pid = (Object[]) iter.next();
346                                                                deleteBuilder.append('(');
347                                                                for (int i = 0; i < pid.length; i++) {
348                                                                        if (i > 0) {
349                                                                                deleteBuilder.append(',');
350                                                                        }
351                                                                        deleteBuilder.append(pid[i]);
352                                                                }
353                                                                deleteBuilder.append(')');
354                                                                if (iter.hasNext()) {
355                                                                        deleteBuilder.append(',');
356                                                                }
357                                                        }
358                                                        deleteBuilder.append(')');
359                                                } else {
360                                                        deleteBuilder.append("(:pids)");
361                                                }
362                                                String deleteSql = deleteBuilder.toString();
363                                                nativeQuery = myEntityManager.createQuery(deleteSql);
364                                                if (idProperty.size() == 1) {
365                                                        nativeQuery.setParameter("pids", pids);
366                                                }
367                                                nativeQuery.executeUpdate();
368                                                return pids.size();
369                                        });
370
371                        outcome += count;
372                        if (count == 0) {
373                                break;
374                        }
375
376                        ourLog.info("Have deleted {} entities of type {} in {}", outcome, theEntityType.getSimpleName(), sw);
377                }
378                return outcome;
379        }
380
381        @Override
382        public int expungeEverythingByType(Class<?> theEntityType) {
383                int result = expungeEverythingByTypeWithoutPurging(null, theEntityType, RequestPartitionId.allPartitions());
384                purgeAllCaches();
385                return result;
386        }
387
388        @Override
389        public int expungeEverythingMdmLinks() {
390                return expungeEverythingByType(MdmLink.class);
391        }
392
393        private int doExpungeEverythingQuery(String theQuery) {
394                StopWatch sw = new StopWatch();
395                int outcome = myEntityManager.createQuery(theQuery).executeUpdate();
396                ourLog.debug("SqlQuery affected {} rows in {}: {}", outcome, sw, theQuery);
397                return outcome;
398        }
399}