View Javadoc
1   package ca.uhn.fhir.jpa.search;
2   
3   /*-
4    * #%L
5    * HAPI FHIR JPA Server
6    * %%
7    * Copyright (C) 2014 - 2018 University Health Network
8    * %%
9    * Licensed under the Apache License, Version 2.0 (the "License");
10   * you may not use this file except in compliance with the License.
11   * You may obtain a copy of the License at
12   * 
13   *      http://www.apache.org/licenses/LICENSE-2.0
14   * 
15   * Unless required by applicable law or agreed to in writing, software
16   * distributed under the License is distributed on an "AS IS" BASIS,
17   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18   * See the License for the specific language governing permissions and
19   * limitations under the License.
20   * #L%
21   */
22  
23  import ca.uhn.fhir.jpa.dao.DaoConfig;
24  import ca.uhn.fhir.jpa.dao.data.ISearchDao;
25  import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao;
26  import ca.uhn.fhir.jpa.dao.data.ISearchResultDao;
27  import com.google.common.annotations.VisibleForTesting;
28  import org.apache.commons.lang3.time.DateUtils;
29  import org.hl7.fhir.dstu3.model.InstantType;
30  import org.springframework.beans.factory.annotation.Autowired;
31  import org.springframework.data.domain.PageRequest;
32  import org.springframework.data.domain.Slice;
33  import org.springframework.scheduling.annotation.Scheduled;
34  import org.springframework.transaction.PlatformTransactionManager;
35  import org.springframework.transaction.annotation.Propagation;
36  import org.springframework.transaction.annotation.Transactional;
37  import org.springframework.transaction.support.TransactionTemplate;
38  
39  import javax.persistence.EntityManager;
40  import javax.persistence.PersistenceContext;
41  import java.util.Date;
42  
43  /**
44   * Deletes old searches
45   */
46  public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc {
47  	public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND;
48  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(StaleSearchDeletingSvcImpl.class);
49  	/*
50  	 * Be careful increasing this number! We use the number of params here in a
51  	 * DELETE FROM foo WHERE params IN (aaaa)
52  	 * type query and this can fail if we have 1000s of params
53  	 */
54  	public static int ourMaximumResultsToDelete = 500;
55  	private static Long ourNowForUnitTests;
56  	/*
57  	 * We give a bit of extra leeway just to avoid race conditions where a query result
58  	 * is being reused (because a new client request came in with the same params) right before
59  	 * the result is to be deleted
60  	 */
61  	private long myCutoffSlack = DEFAULT_CUTOFF_SLACK;
62  	@Autowired
63  	private DaoConfig myDaoConfig;
64  	@Autowired
65  	private ISearchDao mySearchDao;
66  	@Autowired
67  	private ISearchIncludeDao mySearchIncludeDao;
68  	@Autowired
69  	private ISearchResultDao mySearchResultDao;
70  	@Autowired
71  	private PlatformTransactionManager myTransactionManager;
72  	@PersistenceContext()
73  	private EntityManager myEntityManager;
74  
75  	private void deleteSearch(final Long theSearchPid) {
76  		mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> {
77  			mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
78  
79  			/*
80  			 * Note, we're only deleting up to 500 results in an individual search here. This
81  			 * is to prevent really long running transactions in cases where there are
82  			 * huge searches with tons of results in them. By the time we've gotten here
83  			 * we have marked the parent Search entity as deleted, so it's not such a
84  			 * huge deal to be only partially deleting search results. They'll get deleted
85  			 * eventually
86  			 */
87  			int max = ourMaximumResultsToDelete;
88  			Slice<Long> resultPids = mySearchResultDao.findForSearch(PageRequest.of(0, max), searchToDelete.getId());
89  			if (resultPids.hasContent()) {
90  				mySearchResultDao.deleteByIds(resultPids.getContent());
91  			}
92  
93  			// Only delete if we don't have results left in this search
94  			if (resultPids.getNumberOfElements() < max) {
95  				ourLog.info("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned()));
96  				mySearchDao.deleteByPid(searchToDelete.getId());
97  			} else {
98  				ourLog.info("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid());
99  			}
100 		});
101 	}
102 
103 	@Override
104 	@Transactional(propagation = Propagation.NEVER)
105 	public void pollForStaleSearchesAndDeleteThem() {
106 		if (!myDaoConfig.isExpireSearchResults()) {
107 			return;
108 		}
109 
110 		long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis();
111 		if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) {
112 			cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis());
113 		}
114 		final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack);
115 
116 		if (ourNowForUnitTests != null) {
117 			ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now())));
118 		}
119 
120 		ourLog.debug("Searching for searches which are before {}", cutoff);
121 
122 		TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
123 		final Slice<Long> toDelete = tt.execute(theStatus ->
124 			mySearchDao.findWhereLastReturnedBefore(cutoff, PageRequest.of(0, 1000))
125 		);
126 		for (final Long nextSearchToDelete : toDelete) {
127 			ourLog.debug("Deleting search with PID {}", nextSearchToDelete);
128 			tt.execute(t -> {
129 				mySearchDao.updateDeleted(nextSearchToDelete, true);
130 				return null;
131 			});
132 
133 			tt.execute(t -> {
134 				deleteSearch(nextSearchToDelete);
135 				return null;
136 			});
137 		}
138 
139 		int count = toDelete.getContent().size();
140 		if (count > 0) {
141 			long total = tt.execute(t -> mySearchDao.count());
142 			ourLog.info("Deleted {} searches, {} remaining", count, total);
143 		}
144 
145 	}
146 
147 	@Scheduled(fixedDelay = DEFAULT_CUTOFF_SLACK)
148 	@Transactional(propagation = Propagation.NEVER)
149 	@Override
150 	public synchronized void schedulePollForStaleSearches() {
151 		if (!myDaoConfig.isSchedulingDisabled()) {
152 			pollForStaleSearchesAndDeleteThem();
153 		}
154 	}
155 
156 	@VisibleForTesting
157 	public void setCutoffSlackForUnitTest(long theCutoffSlack) {
158 		myCutoffSlack = theCutoffSlack;
159 	}
160 
161 	@VisibleForTesting
162 	public static void setMaximumResultsToDeleteForUnitTest(int theMaximumResultsToDelete) {
163 		ourMaximumResultsToDelete = theMaximumResultsToDelete;
164 	}
165 
166 	private static long now() {
167 		if (ourNowForUnitTests != null) {
168 			return ourNowForUnitTests;
169 		}
170 		return System.currentTimeMillis();
171 	}
172 
173 	/**
174 	 * This is for unit tests only, do not call otherwise
175 	 */
176 	@VisibleForTesting
177 	public static void setNowForUnitTests(Long theNowForUnitTests) {
178 		ourNowForUnitTests = theNowForUnitTests;
179 	}
180 
181 }