001/*- 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2024 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.dao.index; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 025import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; 026import ca.uhn.fhir.jpa.dao.data.IResourceIndexedComboStringUniqueDao; 027import ca.uhn.fhir.jpa.model.entity.BaseResourceIndex; 028import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; 029import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique; 030import ca.uhn.fhir.jpa.model.entity.ResourceTable; 031import ca.uhn.fhir.jpa.model.entity.StorageSettings; 032import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; 033import ca.uhn.fhir.jpa.util.AddRemoveCount; 034import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException; 035import com.google.common.annotations.VisibleForTesting; 036import jakarta.annotation.Nullable; 037import jakarta.persistence.EntityManager; 038import jakarta.persistence.PersistenceContext; 039import jakarta.persistence.PersistenceContextType; 040import org.springframework.beans.factory.annotation.Autowired; 041import org.springframework.stereotype.Service; 042 043import java.util.ArrayList; 044import java.util.Collection; 045import java.util.Date; 046import java.util.HashSet; 047import java.util.Iterator; 048import java.util.List; 049import java.util.Set; 050 051@Service 052public class DaoSearchParamSynchronizer { 053 054 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 055 protected EntityManager myEntityManager; 056 057 @Autowired 058 private JpaStorageSettings myStorageSettings; 059 060 @Autowired 061 private IResourceIndexedComboStringUniqueDao myResourceIndexedCompositeStringUniqueDao; 062 063 @Autowired 064 private FhirContext myFhirContext; 065 066 public AddRemoveCount synchronizeSearchParamsToDatabase( 067 ResourceIndexedSearchParams theParams, 068 ResourceTable theEntity, 069 ResourceIndexedSearchParams existingParams) { 070 AddRemoveCount retVal = new AddRemoveCount(); 071 072 synchronize(theEntity, retVal, theParams.myStringParams, existingParams.myStringParams, null); 073 synchronize(theEntity, retVal, theParams.myTokenParams, existingParams.myTokenParams, null); 074 synchronize(theEntity, retVal, theParams.myNumberParams, existingParams.myNumberParams, null); 075 synchronize(theEntity, retVal, theParams.myQuantityParams, existingParams.myQuantityParams, null); 076 synchronize( 077 theEntity, 078 retVal, 079 theParams.myQuantityNormalizedParams, 080 existingParams.myQuantityNormalizedParams, 081 null); 082 synchronize(theEntity, retVal, theParams.myDateParams, existingParams.myDateParams, null); 083 synchronize(theEntity, retVal, theParams.myUriParams, existingParams.myUriParams, null); 084 synchronize(theEntity, retVal, theParams.myCoordsParams, existingParams.myCoordsParams, null); 085 synchronize(theEntity, retVal, theParams.myLinks, existingParams.myLinks, null); 086 synchronize(theEntity, retVal, theParams.myComboTokenNonUnique, existingParams.myComboTokenNonUnique, null); 087 synchronize( 088 theEntity, 089 retVal, 090 theParams.myComboStringUniques, 091 existingParams.myComboStringUniques, 092 new UniqueIndexPreExistenceChecker()); 093 094 // make sure links are indexed 095 theEntity.setResourceLinks(theParams.myLinks); 096 097 return retVal; 098 } 099 100 @VisibleForTesting 101 public void setEntityManager(EntityManager theEntityManager) { 102 myEntityManager = theEntityManager; 103 } 104 105 @VisibleForTesting 106 public void setStorageSettings(JpaStorageSettings theStorageSettings) { 107 myStorageSettings = theStorageSettings; 108 } 109 110 private <T extends BaseResourceIndex> void synchronize( 111 ResourceTable theEntity, 112 AddRemoveCount theAddRemoveCount, 113 Collection<T> theNewParams, 114 Collection<T> theExistingParams, 115 @Nullable IPreSaveHook<T> theAddParamPreSaveHook) { 116 Collection<T> newParams = theNewParams; 117 for (T next : newParams) { 118 next.setPartitionId(theEntity.getPartitionId()); 119 next.calculateHashes(); 120 } 121 122 /* 123 * It's technically possible that the existing index collection 124 * contains duplicates. Duplicates don't actually cause any 125 * issues for searching since we always deduplicate the PIDs we 126 * get back from the search, but they are wasteful. We don't 127 * enforce uniqueness in the DB for the index tables for 128 * performance reasons (no sense adding a constraint that slows 129 * down writes when dupes don't actually hurt anything other than 130 * a bit of wasted space). 131 * 132 * So we check if there are any dupes, and if we find any we 133 * remove them. 134 */ 135 Set<T> existingParamsAsSet = new HashSet<>(theExistingParams.size()); 136 for (Iterator<T> iterator = theExistingParams.iterator(); iterator.hasNext(); ) { 137 T next = iterator.next(); 138 next.setPlaceholderHashesIfMissing(); 139 if (!existingParamsAsSet.add(next)) { 140 iterator.remove(); 141 myEntityManager.remove(next); 142 } 143 } 144 145 /* 146 * HashCodes may have changed as a result of setting the partition ID, so 147 * create a new set that will reflect the new hashcodes 148 */ 149 newParams = new HashSet<>(newParams); 150 151 List<T> paramsToRemove = subtract(theExistingParams, newParams); 152 List<T> paramsToAdd = subtract(newParams, theExistingParams); 153 154 if (theAddParamPreSaveHook != null) { 155 theAddParamPreSaveHook.preSave(paramsToRemove, paramsToAdd); 156 } 157 158 tryToReuseIndexEntities(paramsToRemove, paramsToAdd); 159 updateExistingParamsIfRequired(theExistingParams, paramsToAdd, newParams, paramsToRemove); 160 161 for (T next : paramsToRemove) { 162 if (!myEntityManager.contains(next)) { 163 // If a resource is created and deleted in the same transaction, we can end up 164 // in a state where we're deleting entities that don't actually exist. Hibernate 165 // 6 is stricter about this, so we skip here. 166 continue; 167 } 168 myEntityManager.remove(next); 169 } 170 171 for (T next : paramsToAdd) { 172 myEntityManager.merge(next); 173 } 174 175 // TODO: are there any unintended consequences to fixing this bug? 176 theAddRemoveCount.addToAddCount(paramsToAdd.size()); 177 theAddRemoveCount.addToRemoveCount(paramsToRemove.size()); 178 } 179 180 /** 181 * <p> 182 * This method performs an update of Search Parameter's fields in the case of 183 * <code>$reindex</code> or update operation by: 184 * 1. Marking existing entities for updating to apply index storage optimization, 185 * if it is enabled (disabled by default). 186 * 2. Recovering <code>SP_NAME</code>, <code>RES_TYPE</code> values of Search Parameter's fields 187 * for existing entities in case if index storage optimization is disabled (but was enabled previously). 188 * </p> 189 * For details, see: {@link StorageSettings#isIndexStorageOptimized()} 190 */ 191 private <T extends BaseResourceIndex> void updateExistingParamsIfRequired( 192 Collection<T> theExistingParams, 193 List<T> theParamsToAdd, 194 Collection<T> theNewParams, 195 List<T> theParamsToRemove) { 196 197 theExistingParams.stream() 198 .filter(BaseResourceIndexedSearchParam.class::isInstance) 199 .map(BaseResourceIndexedSearchParam.class::cast) 200 .filter(this::isSearchParameterUpdateRequired) 201 .filter(sp -> !theParamsToAdd.contains(sp)) 202 .filter(sp -> !theParamsToRemove.contains(sp)) 203 .forEach(sp -> { 204 // force hibernate to update Search Parameter entity by resetting SP_UPDATED value 205 sp.setUpdated(new Date()); 206 recoverExistingSearchParameterIfRequired(sp, theNewParams); 207 theParamsToAdd.add((T) sp); 208 }); 209 } 210 211 /** 212 * Search parameters should be updated after changing IndexStorageOptimized setting. 213 * If IndexStorageOptimized is disabled (and was enabled previously), this method copies paramName 214 * and Resource Type from extracted to existing search parameter. 215 */ 216 private <T extends BaseResourceIndex> void recoverExistingSearchParameterIfRequired( 217 BaseResourceIndexedSearchParam theSearchParamToRecover, Collection<T> theNewParams) { 218 if (!myStorageSettings.isIndexStorageOptimized()) { 219 theNewParams.stream() 220 .filter(BaseResourceIndexedSearchParam.class::isInstance) 221 .map(BaseResourceIndexedSearchParam.class::cast) 222 .filter(paramToAdd -> paramToAdd.equals(theSearchParamToRecover)) 223 .findFirst() 224 .ifPresent(newParam -> { 225 theSearchParamToRecover.restoreParamName(newParam.getParamName()); 226 theSearchParamToRecover.setResourceType(newParam.getResourceType()); 227 }); 228 } 229 } 230 231 private boolean isSearchParameterUpdateRequired(BaseResourceIndexedSearchParam theSearchParameter) { 232 return (myStorageSettings.isIndexStorageOptimized() && !theSearchParameter.isIndexStorageOptimized()) 233 || (!myStorageSettings.isIndexStorageOptimized() && theSearchParameter.isIndexStorageOptimized()); 234 } 235 236 /** 237 * The logic here is that often times when we update a resource we are dropping 238 * one index row and adding another. This method tries to reuse rows that would otherwise 239 * have been deleted by updating them with the contents of rows that would have 240 * otherwise been added. In other words, we're trying to replace 241 * "one delete + one insert" with "one update" 242 * 243 * @param theIndexesToRemove The rows that would be removed 244 * @param theIndexesToAdd The rows that would be added 245 */ 246 private <T extends BaseResourceIndex> void tryToReuseIndexEntities( 247 List<T> theIndexesToRemove, List<T> theIndexesToAdd) { 248 for (int addIndex = 0; addIndex < theIndexesToAdd.size(); addIndex++) { 249 250 // If there are no more rows to remove, there's nothing we can reuse 251 if (theIndexesToRemove.isEmpty()) { 252 break; 253 } 254 255 T targetEntity = theIndexesToAdd.get(addIndex); 256 if (targetEntity.getId() != null) { 257 continue; 258 } 259 260 // Take a row we were going to remove, and repurpose its ID 261 T entityToReuse = theIndexesToRemove.remove(theIndexesToRemove.size() - 1); 262 entityToReuse.copyMutableValuesFrom(targetEntity); 263 theIndexesToAdd.set(addIndex, entityToReuse); 264 } 265 } 266 267 public static <T> List<T> subtract(Collection<T> theSubtractFrom, Collection<T> theToSubtract) { 268 assert theSubtractFrom != theToSubtract || (theSubtractFrom.isEmpty()); 269 270 if (theSubtractFrom.isEmpty()) { 271 return new ArrayList<>(); 272 } 273 274 ArrayList<T> retVal = new ArrayList<>(); 275 for (T next : theSubtractFrom) { 276 if (!theToSubtract.contains(next)) { 277 retVal.add(next); 278 } 279 } 280 return retVal; 281 } 282 283 private interface IPreSaveHook<T> { 284 285 void preSave(Collection<T> theParamsToRemove, Collection<T> theParamsToAdd); 286 } 287 288 private class UniqueIndexPreExistenceChecker implements IPreSaveHook<ResourceIndexedComboStringUnique> { 289 290 @Override 291 public void preSave( 292 Collection<ResourceIndexedComboStringUnique> theParamsToRemove, 293 Collection<ResourceIndexedComboStringUnique> theParamsToAdd) { 294 if (myStorageSettings.isUniqueIndexesCheckedBeforeSave()) { 295 for (ResourceIndexedComboStringUnique theIndex : theParamsToAdd) { 296 ResourceIndexedComboStringUnique existing = 297 myResourceIndexedCompositeStringUniqueDao.findByQueryString(theIndex.getIndexString()); 298 if (existing != null) { 299 300 /* 301 * If we're reindexing, and the previous index row is being updated 302 * to add previously missing hashes, we may falsely detect that the index 303 * creation is going to fail. 304 */ 305 boolean existingIndexIsScheduledForRemoval = false; 306 for (var next : theParamsToRemove) { 307 if (existing == next) { 308 existingIndexIsScheduledForRemoval = true; 309 break; 310 } 311 } 312 if (existingIndexIsScheduledForRemoval) { 313 continue; 314 } 315 316 String searchParameterId = "(unknown)"; 317 if (theIndex.getSearchParameterId() != null) { 318 searchParameterId = theIndex.getSearchParameterId().getValue(); 319 } 320 321 String msg = myFhirContext 322 .getLocalizer() 323 .getMessage( 324 BaseHapiFhirDao.class, 325 "uniqueIndexConflictFailure", 326 existing.getResource().getResourceType(), 327 theIndex.getIndexString(), 328 existing.getResource() 329 .getIdDt() 330 .toUnqualifiedVersionless() 331 .getValue(), 332 searchParameterId); 333 334 // Use ResourceVersionConflictException here because the HapiTransactionService 335 // catches this and can retry it if needed 336 throw new ResourceVersionConflictException(Msg.code(1093) + msg); 337 } 338 } 339 } 340 } 341 } 342}