View Javadoc
1   package ca.uhn.fhir.jpa.search;
2   
3   /*-
4    * #%L
5    * HAPI FHIR JPA Server
6    * %%
7    * Copyright (C) 2014 - 2019 University Health Network
8    * %%
9    * Licensed under the Apache License, Version 2.0 (the "License");
10   * you may not use this file except in compliance with the License.
11   * You may obtain a copy of the License at
12   * 
13   *      http://www.apache.org/licenses/LICENSE-2.0
14   * 
15   * Unless required by applicable law or agreed to in writing, software
16   * distributed under the License is distributed on an "AS IS" BASIS,
17   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18   * See the License for the specific language governing permissions and
19   * limitations under the License.
20   * #L%
21   */
22  
23  import ca.uhn.fhir.jpa.dao.DaoConfig;
24  import ca.uhn.fhir.jpa.dao.data.ISearchDao;
25  import ca.uhn.fhir.jpa.dao.data.ISearchIncludeDao;
26  import ca.uhn.fhir.jpa.dao.data.ISearchResultDao;
27  import com.google.common.annotations.VisibleForTesting;
28  import com.google.common.collect.Lists;
29  import org.apache.commons.lang3.time.DateUtils;
30  import org.hl7.fhir.dstu3.model.InstantType;
31  import org.springframework.beans.factory.annotation.Autowired;
32  import org.springframework.data.domain.PageRequest;
33  import org.springframework.data.domain.Slice;
34  import org.springframework.scheduling.annotation.Scheduled;
35  import org.springframework.transaction.PlatformTransactionManager;
36  import org.springframework.transaction.annotation.Propagation;
37  import org.springframework.transaction.annotation.Transactional;
38  import org.springframework.transaction.support.TransactionTemplate;
39  
40  import java.util.Date;
41  import java.util.List;
42  
43  /**
44   * Deletes old searches
45   */
46  //
47  // NOTE: This is not a @Service because we manually instantiate
48  // it in BaseConfig. This is so that we can override the definition
49  // in Smile.
50  //
51  public class StaleSearchDeletingSvcImpl implements IStaleSearchDeletingSvc {
52  	public static final long DEFAULT_CUTOFF_SLACK = 10 * DateUtils.MILLIS_PER_SECOND;
53  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(StaleSearchDeletingSvcImpl.class);
54  	/*
55  	 * Be careful increasing this number! We use the number of params here in a
56  	 * DELETE FROM foo WHERE params IN (aaaa)
57  	 * type query and this can fail if we have 1000s of params
58  	 */
59  	public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT = 500;
60  	public static final int DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS = 20000;
61  	private static int ourMaximumResultsToDeleteInOneStatement = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_STMT;
62  	private static int ourMaximumResultsToDeleteInOnePass = DEFAULT_MAX_RESULTS_TO_DELETE_IN_ONE_PAS;
63  	private static Long ourNowForUnitTests;
64  	/*
65  	 * We give a bit of extra leeway just to avoid race conditions where a query result
66  	 * is being reused (because a new client request came in with the same params) right before
67  	 * the result is to be deleted
68  	 */
69  	private long myCutoffSlack = DEFAULT_CUTOFF_SLACK;
70  	@Autowired
71  	private DaoConfig myDaoConfig;
72  	@Autowired
73  	private ISearchDao mySearchDao;
74  	@Autowired
75  	private ISearchIncludeDao mySearchIncludeDao;
76  	@Autowired
77  	private ISearchResultDao mySearchResultDao;
78  	@Autowired
79  	private PlatformTransactionManager myTransactionManager;
80  
81  	private void deleteSearch(final Long theSearchPid) {
82  		mySearchDao.findById(theSearchPid).ifPresent(searchToDelete -> {
83  			mySearchIncludeDao.deleteForSearch(searchToDelete.getId());
84  
85  			/*
86  			 * Note, we're only deleting up to 500 results in an individual search here. This
87  			 * is to prevent really long running transactions in cases where there are
88  			 * huge searches with tons of results in them. By the time we've gotten here
89  			 * we have marked the parent Search entity as deleted, so it's not such a
90  			 * huge deal to be only partially deleting search results. They'll get deleted
91  			 * eventually
92  			 */
93  			int max = ourMaximumResultsToDeleteInOnePass;
94  			Slice<Long> resultPids = mySearchResultDao.findForSearch(PageRequest.of(0, max), searchToDelete.getId());
95  			if (resultPids.hasContent()) {
96  				List<List<Long>> partitions = Lists.partition(resultPids.getContent(), ourMaximumResultsToDeleteInOneStatement);
97  				for (List<Long> nextPartition : partitions) {
98  					mySearchResultDao.deleteByIds(nextPartition);
99  				}
100 
101 			}
102 
103 			// Only delete if we don't have results left in this search
104 			if (resultPids.getNumberOfElements() < max) {
105 				ourLog.debug("Deleting search {}/{} - Created[{}] -- Last returned[{}]", searchToDelete.getId(), searchToDelete.getUuid(), new InstantType(searchToDelete.getCreated()), new InstantType(searchToDelete.getSearchLastReturned()));
106 				mySearchDao.deleteByPid(searchToDelete.getId());
107 			} else {
108 				ourLog.debug("Purged {} search results for deleted search {}/{}", resultPids.getSize(), searchToDelete.getId(), searchToDelete.getUuid());
109 			}
110 		});
111 	}
112 
113 	@Override
114 	@Transactional(propagation = Propagation.NEVER)
115 	public void pollForStaleSearchesAndDeleteThem() {
116 		if (!myDaoConfig.isExpireSearchResults()) {
117 			return;
118 		}
119 
120 		long cutoffMillis = myDaoConfig.getExpireSearchResultsAfterMillis();
121 		if (myDaoConfig.getReuseCachedSearchResultsForMillis() != null) {
122 			cutoffMillis = Math.max(cutoffMillis, myDaoConfig.getReuseCachedSearchResultsForMillis());
123 		}
124 		final Date cutoff = new Date((now() - cutoffMillis) - myCutoffSlack);
125 
126 		if (ourNowForUnitTests != null) {
127 			ourLog.info("Searching for searches which are before {} - now is {}", new InstantType(cutoff), new InstantType(new Date(now())));
128 		}
129 
130 		ourLog.debug("Searching for searches which are before {}", cutoff);
131 
132 		TransactionTemplate tt = new TransactionTemplate(myTransactionManager);
133 		final Slice<Long> toDelete = tt.execute(theStatus ->
134 			mySearchDao.findWhereLastReturnedBefore(cutoff, PageRequest.of(0, 2000))
135 		);
136 		for (final Long nextSearchToDelete : toDelete) {
137 			ourLog.debug("Deleting search with PID {}", nextSearchToDelete);
138 			tt.execute(t -> {
139 				mySearchDao.updateDeleted(nextSearchToDelete, true);
140 				return null;
141 			});
142 
143 			tt.execute(t -> {
144 				deleteSearch(nextSearchToDelete);
145 				return null;
146 			});
147 		}
148 
149 		int count = toDelete.getContent().size();
150 		if (count > 0) {
151 			if (ourLog.isDebugEnabled()) {
152 				long total = tt.execute(t -> mySearchDao.count());
153 				ourLog.debug("Deleted {} searches, {} remaining", count, total);
154 			}
155 		}
156 
157 	}
158 
159 	@Scheduled(fixedDelay = DEFAULT_CUTOFF_SLACK)
160 	@Transactional(propagation = Propagation.NEVER)
161 	@Override
162 	public synchronized void schedulePollForStaleSearches() {
163 		if (!myDaoConfig.isSchedulingDisabled()) {
164 			pollForStaleSearchesAndDeleteThem();
165 		}
166 	}
167 
168 	@VisibleForTesting
169 	public void setCutoffSlackForUnitTest(long theCutoffSlack) {
170 		myCutoffSlack = theCutoffSlack;
171 	}
172 
173 	@VisibleForTesting
174 	public static void setMaximumResultsToDeleteInOnePassForUnitTest(int theMaximumResultsToDeleteInOnePass) {
175 		ourMaximumResultsToDeleteInOnePass = theMaximumResultsToDeleteInOnePass;
176 	}
177 
178 	@VisibleForTesting
179 	public static void setMaximumResultsToDeleteForUnitTest(int theMaximumResultsToDelete) {
180 		ourMaximumResultsToDeleteInOneStatement = theMaximumResultsToDelete;
181 	}
182 
183 	private static long now() {
184 		if (ourNowForUnitTests != null) {
185 			return ourNowForUnitTests;
186 		}
187 		return System.currentTimeMillis();
188 	}
189 
190 	/**
191 	 * This is for unit tests only, do not call otherwise
192 	 */
193 	@VisibleForTesting
194 	public static void setNowForUnitTests(Long theNowForUnitTests) {
195 		ourNowForUnitTests = theNowForUnitTests;
196 	}
197 
198 }