001/*- 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2025 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.dao.index; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 025import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; 026import ca.uhn.fhir.jpa.dao.data.IResourceIndexedComboStringUniqueDao; 027import ca.uhn.fhir.jpa.model.entity.BaseResourceIndex; 028import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; 029import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique; 030import ca.uhn.fhir.jpa.model.entity.ResourceTable; 031import ca.uhn.fhir.jpa.model.entity.StorageSettings; 032import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; 033import ca.uhn.fhir.jpa.util.AddRemoveCount; 034import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException; 035import com.google.common.annotations.VisibleForTesting; 036import jakarta.annotation.Nullable; 037import jakarta.persistence.EntityManager; 038import jakarta.persistence.PersistenceContext; 039import jakarta.persistence.PersistenceContextType; 040import org.springframework.beans.factory.annotation.Autowired; 041import org.springframework.stereotype.Service; 042 043import java.util.ArrayList; 044import java.util.Collection; 045import java.util.Date; 046import java.util.HashSet; 047import java.util.Iterator; 048import java.util.List; 049import java.util.Set; 050 051@Service 052public class DaoSearchParamSynchronizer { 053 054 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 055 protected EntityManager myEntityManager; 056 057 @Autowired 058 private JpaStorageSettings myStorageSettings; 059 060 @Autowired 061 private IResourceIndexedComboStringUniqueDao myResourceIndexedCompositeStringUniqueDao; 062 063 @Autowired 064 private FhirContext myFhirContext; 065 066 public AddRemoveCount synchronizeSearchParamsToDatabase( 067 ResourceIndexedSearchParams theParams, 068 ResourceTable theEntity, 069 ResourceIndexedSearchParams existingParams) { 070 AddRemoveCount retVal = new AddRemoveCount(); 071 072 synchronize(theEntity, retVal, theParams.myStringParams, existingParams.myStringParams, null); 073 synchronize(theEntity, retVal, theParams.myTokenParams, existingParams.myTokenParams, null); 074 synchronize(theEntity, retVal, theParams.myNumberParams, existingParams.myNumberParams, null); 075 synchronize(theEntity, retVal, theParams.myQuantityParams, existingParams.myQuantityParams, null); 076 synchronize( 077 theEntity, 078 retVal, 079 theParams.myQuantityNormalizedParams, 080 existingParams.myQuantityNormalizedParams, 081 null); 082 synchronize(theEntity, retVal, theParams.myDateParams, existingParams.myDateParams, null); 083 synchronize(theEntity, retVal, theParams.myUriParams, existingParams.myUriParams, null); 084 synchronize(theEntity, retVal, theParams.myCoordsParams, existingParams.myCoordsParams, null); 085 synchronize(theEntity, retVal, theParams.myLinks, existingParams.myLinks, null); 086 synchronize(theEntity, retVal, theParams.myComboTokenNonUnique, existingParams.myComboTokenNonUnique, null); 087 synchronize( 088 theEntity, 089 retVal, 090 theParams.myComboStringUniques, 091 existingParams.myComboStringUniques, 092 new UniqueIndexPreExistenceChecker()); 093 094 // make sure links are indexed 095 theEntity.setResourceLinks(theParams.myLinks); 096 097 return retVal; 098 } 099 100 @VisibleForTesting 101 public void setEntityManager(EntityManager theEntityManager) { 102 myEntityManager = theEntityManager; 103 } 104 105 @VisibleForTesting 106 public void setStorageSettings(JpaStorageSettings theStorageSettings) { 107 myStorageSettings = theStorageSettings; 108 } 109 110 private <T extends BaseResourceIndex> void synchronize( 111 ResourceTable theEntity, 112 AddRemoveCount theAddRemoveCount, 113 Collection<T> theNewParams, 114 Collection<T> theExistingParams, 115 @Nullable IPreSaveHook<T> theAddParamPreSaveHook) { 116 Collection<T> newParams = theNewParams; 117 for (T next : newParams) { 118 next.setResourceId(theEntity.getId().getId()); 119 next.setPartitionId(theEntity.getPartitionId()); 120 next.calculateHashes(); 121 } 122 123 /* 124 * It's technically possible that the existing index collection 125 * contains duplicates. Duplicates don't actually cause any 126 * issues for searching since we always deduplicate the PIDs we 127 * get back from the search, but they are wasteful. We don't 128 * enforce uniqueness in the DB for the index tables for 129 * performance reasons (no sense adding a constraint that slows 130 * down writes when dupes don't actually hurt anything other than 131 * a bit of wasted space). 132 * 133 * So we check if there are any dupes, and if we find any we 134 * remove them. 135 */ 136 Set<T> existingParamsAsSet = new HashSet<>(theExistingParams.size()); 137 for (Iterator<T> iterator = theExistingParams.iterator(); iterator.hasNext(); ) { 138 T next = iterator.next(); 139 next.setPlaceholderHashesIfMissing(); 140 if (!existingParamsAsSet.add(next)) { 141 iterator.remove(); 142 myEntityManager.remove(next); 143 } 144 } 145 146 /* 147 * HashCodes may have changed as a result of setting the partition ID, so 148 * create a new set that will reflect the new hashcodes 149 */ 150 newParams = new HashSet<>(newParams); 151 152 List<T> paramsToRemove = subtract(theExistingParams, newParams); 153 List<T> paramsToAdd = subtract(newParams, theExistingParams); 154 155 if (theAddParamPreSaveHook != null) { 156 theAddParamPreSaveHook.preSave(paramsToRemove, paramsToAdd); 157 } 158 159 tryToReuseIndexEntities(paramsToRemove, paramsToAdd); 160 updateExistingParamsIfRequired(theExistingParams, paramsToAdd, newParams, paramsToRemove); 161 162 for (T next : paramsToRemove) { 163 if (!myEntityManager.contains(next)) { 164 // If a resource is created and deleted in the same transaction, we can end up 165 // in a state where we're deleting entities that don't actually exist. Hibernate 166 // 6 is stricter about this, so we skip here. 167 continue; 168 } 169 myEntityManager.remove(next); 170 } 171 172 for (T next : paramsToAdd) { 173 if (next.getId() == null) { 174 myEntityManager.persist(next); 175 } else { 176 myEntityManager.merge(next); 177 } 178 } 179 180 // TODO: are there any unintended consequences to fixing this bug? 181 theAddRemoveCount.addToAddCount(paramsToAdd.size()); 182 theAddRemoveCount.addToRemoveCount(paramsToRemove.size()); 183 184 // Replace the existing "new set" with the set of params we should be adding. 185 // We're going to add them back into the entity just in case it gets updated 186 // a second time within the same transaction 187 theNewParams.clear(); 188 theNewParams.addAll(theExistingParams); 189 theNewParams.addAll(paramsToAdd); 190 theNewParams.removeAll(paramsToRemove); 191 } 192 193 /** 194 * <p> 195 * This method performs an update of Search Parameter's fields in the case of 196 * <code>$reindex</code> or update operation by: 197 * 1. Marking existing entities for updating to apply index storage optimization, 198 * if it is enabled (disabled by default). 199 * 2. Recovering <code>SP_NAME</code>, <code>RES_TYPE</code> values of Search Parameter's fields 200 * for existing entities in case if index storage optimization is disabled (but was enabled previously). 201 * </p> 202 * For details, see: {@link StorageSettings#isIndexStorageOptimized()} 203 */ 204 private <T extends BaseResourceIndex> void updateExistingParamsIfRequired( 205 Collection<T> theExistingParams, 206 List<T> theParamsToAdd, 207 Collection<T> theNewParams, 208 List<T> theParamsToRemove) { 209 210 theExistingParams.stream() 211 .filter(BaseResourceIndexedSearchParam.class::isInstance) 212 .map(BaseResourceIndexedSearchParam.class::cast) 213 .filter(this::isSearchParameterUpdateRequired) 214 .filter(sp -> !theParamsToAdd.contains(sp)) 215 .filter(sp -> !theParamsToRemove.contains(sp)) 216 .forEach(sp -> { 217 // force hibernate to update Search Parameter entity by resetting SP_UPDATED value 218 sp.setUpdated(new Date()); 219 recoverExistingSearchParameterIfRequired(sp, theNewParams); 220 theParamsToAdd.add((T) sp); 221 }); 222 } 223 224 /** 225 * Search parameters should be updated after changing IndexStorageOptimized setting. 226 * If IndexStorageOptimized is disabled (and was enabled previously), this method copies paramName 227 * and Resource Type from extracted to existing search parameter. 228 */ 229 private <T extends BaseResourceIndex> void recoverExistingSearchParameterIfRequired( 230 BaseResourceIndexedSearchParam theSearchParamToRecover, Collection<T> theNewParams) { 231 if (!myStorageSettings.isIndexStorageOptimized()) { 232 theNewParams.stream() 233 .filter(BaseResourceIndexedSearchParam.class::isInstance) 234 .map(BaseResourceIndexedSearchParam.class::cast) 235 .filter(paramToAdd -> paramToAdd.equals(theSearchParamToRecover)) 236 .findFirst() 237 .ifPresent(newParam -> { 238 theSearchParamToRecover.restoreParamName(newParam.getParamName()); 239 theSearchParamToRecover.setResourceType(newParam.getResourceType()); 240 }); 241 } 242 } 243 244 private boolean isSearchParameterUpdateRequired(BaseResourceIndexedSearchParam theSearchParameter) { 245 return (myStorageSettings.isIndexStorageOptimized() && !theSearchParameter.isIndexStorageOptimized()) 246 || (!myStorageSettings.isIndexStorageOptimized() && theSearchParameter.isIndexStorageOptimized()); 247 } 248 249 /** 250 * The logic here is that often times when we update a resource we are dropping 251 * one index row and adding another. This method tries to reuse rows that would otherwise 252 * have been deleted by updating them with the contents of rows that would have 253 * otherwise been added. In other words, we're trying to replace 254 * "one delete + one insert" with "one update" 255 * 256 * @param theIndexesToRemove The rows that would be removed 257 * @param theIndexesToAdd The rows that would be added 258 */ 259 private <T extends BaseResourceIndex> void tryToReuseIndexEntities( 260 List<T> theIndexesToRemove, List<T> theIndexesToAdd) { 261 for (int addIndex = 0; addIndex < theIndexesToAdd.size(); addIndex++) { 262 263 // If there are no more rows to remove, there's nothing we can reuse 264 if (theIndexesToRemove.isEmpty()) { 265 break; 266 } 267 268 T targetEntity = theIndexesToAdd.get(addIndex); 269 if (targetEntity.getId() != null) { 270 continue; 271 } 272 273 // Take a row we were going to remove, and repurpose its ID 274 T entityToReuse = theIndexesToRemove.remove(theIndexesToRemove.size() - 1); 275 entityToReuse.copyMutableValuesFrom(targetEntity); 276 theIndexesToAdd.set(addIndex, entityToReuse); 277 } 278 } 279 280 public static <T> List<T> subtract(Collection<T> theSubtractFrom, Collection<T> theToSubtract) { 281 assert theSubtractFrom != theToSubtract || (theSubtractFrom.isEmpty()); 282 283 if (theSubtractFrom.isEmpty()) { 284 return new ArrayList<>(); 285 } 286 287 ArrayList<T> retVal = new ArrayList<>(); 288 for (T next : theSubtractFrom) { 289 if (!theToSubtract.contains(next)) { 290 retVal.add(next); 291 } 292 } 293 return retVal; 294 } 295 296 private interface IPreSaveHook<T> { 297 298 void preSave(Collection<T> theParamsToRemove, Collection<T> theParamsToAdd); 299 } 300 301 private class UniqueIndexPreExistenceChecker implements IPreSaveHook<ResourceIndexedComboStringUnique> { 302 303 @Override 304 public void preSave( 305 Collection<ResourceIndexedComboStringUnique> theParamsToRemove, 306 Collection<ResourceIndexedComboStringUnique> theParamsToAdd) { 307 if (myStorageSettings.isUniqueIndexesCheckedBeforeSave()) { 308 for (ResourceIndexedComboStringUnique theIndex : theParamsToAdd) { 309 ResourceIndexedComboStringUnique existing = 310 myResourceIndexedCompositeStringUniqueDao.findByQueryString(theIndex.getIndexString()); 311 if (existing != null) { 312 313 /* 314 * If we're reindexing, and the previous index row is being updated 315 * to add previously missing hashes, we may falsely detect that the index 316 * creation is going to fail. 317 */ 318 boolean existingIndexIsScheduledForRemoval = false; 319 for (var next : theParamsToRemove) { 320 if (existing == next) { 321 existingIndexIsScheduledForRemoval = true; 322 break; 323 } 324 } 325 if (existingIndexIsScheduledForRemoval) { 326 continue; 327 } 328 329 String searchParameterId = "(unknown)"; 330 if (theIndex.getSearchParameterId() != null) { 331 searchParameterId = theIndex.getSearchParameterId().getValue(); 332 } 333 334 String msg = myFhirContext 335 .getLocalizer() 336 .getMessage( 337 BaseHapiFhirDao.class, 338 "uniqueIndexConflictFailure", 339 existing.getResource().getResourceType(), 340 theIndex.getIndexString(), 341 existing.getResource() 342 .getIdDt() 343 .toUnqualifiedVersionless() 344 .getValue(), 345 searchParameterId); 346 347 // Use ResourceVersionConflictException here because the HapiTransactionService 348 // catches this and can retry it if needed 349 throw new ResourceVersionConflictException(Msg.code(1093) + msg); 350 } 351 } 352 } 353 } 354 } 355}