001/*- 002 * #%L 003 * HAPI FHIR JPA Server 004 * %% 005 * Copyright (C) 2014 - 2025 Smile CDR, Inc. 006 * %% 007 * Licensed under the Apache License, Version 2.0 (the "License"); 008 * you may not use this file except in compliance with the License. 009 * You may obtain a copy of the License at 010 * 011 * http://www.apache.org/licenses/LICENSE-2.0 012 * 013 * Unless required by applicable law or agreed to in writing, software 014 * distributed under the License is distributed on an "AS IS" BASIS, 015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 016 * See the License for the specific language governing permissions and 017 * limitations under the License. 018 * #L% 019 */ 020package ca.uhn.fhir.jpa.dao.index; 021 022import ca.uhn.fhir.context.FhirContext; 023import ca.uhn.fhir.i18n.Msg; 024import ca.uhn.fhir.jpa.api.config.JpaStorageSettings; 025import ca.uhn.fhir.jpa.cache.ISearchParamIdentityCacheSvc; 026import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao; 027import ca.uhn.fhir.jpa.dao.data.IResourceIndexedComboStringUniqueDao; 028import ca.uhn.fhir.jpa.model.entity.BaseResourceIndex; 029import ca.uhn.fhir.jpa.model.entity.BaseResourceIndexedSearchParam; 030import ca.uhn.fhir.jpa.model.entity.IndexedSearchParamIdentity; 031import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique; 032import ca.uhn.fhir.jpa.model.entity.ResourceTable; 033import ca.uhn.fhir.jpa.model.entity.StorageSettings; 034import ca.uhn.fhir.jpa.searchparam.extractor.ResourceIndexedSearchParams; 035import ca.uhn.fhir.jpa.sp.SearchParamIdentityCacheSvcImpl; 036import ca.uhn.fhir.jpa.util.AddRemoveCount; 037import ca.uhn.fhir.rest.server.exceptions.ResourceVersionConflictException; 038import com.google.common.annotations.VisibleForTesting; 039import jakarta.annotation.Nullable; 040import jakarta.persistence.EntityManager; 041import jakarta.persistence.PersistenceContext; 042import jakarta.persistence.PersistenceContextType; 043import org.springframework.beans.factory.annotation.Autowired; 044import org.springframework.stereotype.Service; 045 046import java.util.ArrayList; 047import java.util.Collection; 048import java.util.Date; 049import java.util.HashSet; 050import java.util.Iterator; 051import java.util.List; 052import java.util.Set; 053 054@Service 055public class DaoSearchParamSynchronizer { 056 057 @PersistenceContext(type = PersistenceContextType.TRANSACTION) 058 protected EntityManager myEntityManager; 059 060 @Autowired 061 private JpaStorageSettings myStorageSettings; 062 063 @Autowired 064 private IResourceIndexedComboStringUniqueDao myResourceIndexedCompositeStringUniqueDao; 065 066 @Autowired 067 private ISearchParamIdentityCacheSvc mySearchParamIdentityCacheSvc; 068 069 @Autowired 070 private FhirContext myFhirContext; 071 072 public AddRemoveCount synchronizeSearchParamsToDatabase( 073 ResourceIndexedSearchParams theParams, 074 ResourceTable theEntity, 075 ResourceIndexedSearchParams existingParams) { 076 AddRemoveCount retVal = new AddRemoveCount(); 077 078 synchronize(theEntity, retVal, theParams.myStringParams, existingParams.myStringParams, null); 079 synchronize(theEntity, retVal, theParams.myTokenParams, existingParams.myTokenParams, null); 080 synchronize(theEntity, retVal, theParams.myNumberParams, existingParams.myNumberParams, null); 081 synchronize(theEntity, retVal, theParams.myQuantityParams, existingParams.myQuantityParams, null); 082 synchronize( 083 theEntity, 084 retVal, 085 theParams.myQuantityNormalizedParams, 086 existingParams.myQuantityNormalizedParams, 087 null); 088 synchronize(theEntity, retVal, theParams.myDateParams, existingParams.myDateParams, null); 089 synchronize(theEntity, retVal, theParams.myUriParams, existingParams.myUriParams, null); 090 synchronize(theEntity, retVal, theParams.myCoordsParams, existingParams.myCoordsParams, null); 091 synchronize(theEntity, retVal, theParams.myLinks, existingParams.myLinks, null); 092 synchronize(theEntity, retVal, theParams.myComboTokenNonUnique, existingParams.myComboTokenNonUnique, null); 093 synchronize( 094 theEntity, 095 retVal, 096 theParams.myComboStringUniques, 097 existingParams.myComboStringUniques, 098 new UniqueIndexPreExistenceChecker()); 099 100 // make sure links are indexed 101 theEntity.setResourceLinks(theParams.myLinks); 102 103 return retVal; 104 } 105 106 @VisibleForTesting 107 public void setEntityManager(EntityManager theEntityManager) { 108 myEntityManager = theEntityManager; 109 } 110 111 @VisibleForTesting 112 public void setSearchParamIdentityCacheSvc(ISearchParamIdentityCacheSvc theSearchParamIdentityCacheSvc) { 113 mySearchParamIdentityCacheSvc = theSearchParamIdentityCacheSvc; 114 } 115 116 @VisibleForTesting 117 public void setStorageSettings(JpaStorageSettings theStorageSettings) { 118 myStorageSettings = theStorageSettings; 119 } 120 121 private <T extends BaseResourceIndex> void synchronize( 122 ResourceTable theEntity, 123 AddRemoveCount theAddRemoveCount, 124 Collection<T> theNewParams, 125 Collection<T> theExistingParams, 126 @Nullable IPreSaveHook<T> theAddParamPreSaveHook) { 127 Collection<T> newParams = theNewParams; 128 for (T next : newParams) { 129 next.setResourceId(theEntity.getId().getId()); 130 next.setPartitionId(theEntity.getPartitionId()); 131 next.calculateHashes(); 132 } 133 134 /* 135 * It's technically possible that the existing index collection 136 * contains duplicates. Duplicates don't actually cause any 137 * issues for searching since we always deduplicate the PIDs we 138 * get back from the search, but they are wasteful. We don't 139 * enforce uniqueness in the DB for the index tables for 140 * performance reasons (no sense adding a constraint that slows 141 * down writes when dupes don't actually hurt anything other than 142 * a bit of wasted space). 143 * 144 * So we check if there are any dupes, and if we find any we 145 * remove them. 146 */ 147 Set<T> existingParamsAsSet = new HashSet<>(theExistingParams.size()); 148 for (Iterator<T> iterator = theExistingParams.iterator(); iterator.hasNext(); ) { 149 T next = iterator.next(); 150 next.setPlaceholderHashesIfMissing(); 151 if (!existingParamsAsSet.add(next)) { 152 iterator.remove(); 153 myEntityManager.remove(next); 154 } 155 } 156 157 /* 158 * HashCodes may have changed as a result of setting the partition ID, so 159 * create a new set that will reflect the new hashcodes 160 */ 161 newParams = new HashSet<>(newParams); 162 163 List<T> paramsToRemove = subtract(theExistingParams, newParams); 164 List<T> paramsToAdd = subtract(newParams, theExistingParams); 165 166 if (theAddParamPreSaveHook != null) { 167 theAddParamPreSaveHook.preSave(paramsToRemove, paramsToAdd); 168 } 169 170 tryToReuseIndexEntities(paramsToRemove, paramsToAdd); 171 updateExistingParamsIfRequired(theExistingParams, paramsToAdd, newParams, paramsToRemove); 172 173 for (T next : paramsToRemove) { 174 if (!myEntityManager.contains(next)) { 175 // If a resource is created and deleted in the same transaction, we can end up 176 // in a state where we're deleting entities that don't actually exist. Hibernate 177 // 6 is stricter about this, so we skip here. 178 continue; 179 } 180 myEntityManager.remove(next); 181 } 182 183 for (T next : paramsToAdd) { 184 findOrCreateSearchParamIdentity(next); 185 if (next.getId() == null) { 186 myEntityManager.persist(next); 187 } else { 188 myEntityManager.merge(next); 189 } 190 } 191 192 // TODO: are there any unintended consequences to fixing this bug? 193 theAddRemoveCount.addToAddCount(paramsToAdd.size()); 194 theAddRemoveCount.addToRemoveCount(paramsToRemove.size()); 195 196 // Replace the existing "new set" with the set of params we should be adding. 197 // We're going to add them back into the entity just in case it gets updated 198 // a second time within the same transaction 199 theNewParams.clear(); 200 theNewParams.addAll(theExistingParams); 201 theNewParams.addAll(paramsToAdd); 202 theNewParams.removeAll(paramsToRemove); 203 } 204 205 /** 206 * Checks whether the Indexed Search Parameter hash identity exists in the cache. 207 * If the identity is missing, a new {@link IndexedSearchParamIdentity} will be 208 * created asynchronously in a separate thread. 209 * 210 * <p>For details, see: 211 * {@link SearchParamIdentityCacheSvcImpl#findOrCreateSearchParamIdentity(Long, String, String)}</p> 212 */ 213 private <T extends BaseResourceIndex> void findOrCreateSearchParamIdentity(T theNewParam) { 214 if (theNewParam instanceof BaseResourceIndexedSearchParam) { 215 BaseResourceIndexedSearchParam indexedSearchParam = ((BaseResourceIndexedSearchParam) theNewParam); 216 mySearchParamIdentityCacheSvc.findOrCreateSearchParamIdentity( 217 indexedSearchParam.getHashIdentity(), 218 indexedSearchParam.getResourceType(), 219 indexedSearchParam.getParamName()); 220 } 221 } 222 223 /** 224 * <p> 225 * This method performs an update of Search Parameter's fields in the case of 226 * <code>$reindex</code> or update operation by: 227 * 1. Marking existing entities for updating to apply index storage optimization, 228 * if it is enabled (disabled by default). 229 * 2. Recovering <code>SP_NAME</code>, <code>RES_TYPE</code> values of Search Parameter's fields 230 * for existing entities in case if index storage optimization is disabled (but was enabled previously). 231 * </p> 232 * For details, see: {@link StorageSettings#isIndexStorageOptimized()} 233 */ 234 private <T extends BaseResourceIndex> void updateExistingParamsIfRequired( 235 Collection<T> theExistingParams, 236 List<T> theParamsToAdd, 237 Collection<T> theNewParams, 238 List<T> theParamsToRemove) { 239 240 theExistingParams.stream() 241 .filter(BaseResourceIndexedSearchParam.class::isInstance) 242 .map(BaseResourceIndexedSearchParam.class::cast) 243 .filter(this::isSearchParameterUpdateRequired) 244 .filter(sp -> !theParamsToAdd.contains(sp)) 245 .filter(sp -> !theParamsToRemove.contains(sp)) 246 .forEach(sp -> { 247 // force hibernate to update Search Parameter entity by resetting SP_UPDATED value 248 sp.setUpdated(new Date()); 249 recoverExistingSearchParameterIfRequired(sp, theNewParams); 250 theParamsToAdd.add((T) sp); 251 }); 252 } 253 254 /** 255 * Search parameters should be updated after changing IndexStorageOptimized setting. 256 * If IndexStorageOptimized is disabled (and was enabled previously), this method copies paramName 257 * and Resource Type from extracted to existing search parameter. 258 */ 259 private <T extends BaseResourceIndex> void recoverExistingSearchParameterIfRequired( 260 BaseResourceIndexedSearchParam theSearchParamToRecover, Collection<T> theNewParams) { 261 if (!myStorageSettings.isIndexStorageOptimized()) { 262 theNewParams.stream() 263 .filter(BaseResourceIndexedSearchParam.class::isInstance) 264 .map(BaseResourceIndexedSearchParam.class::cast) 265 .filter(paramToAdd -> paramToAdd.equals(theSearchParamToRecover)) 266 .findFirst() 267 .ifPresent(newParam -> { 268 theSearchParamToRecover.restoreParamName(newParam.getParamName()); 269 theSearchParamToRecover.setResourceType(newParam.getResourceType()); 270 }); 271 } 272 } 273 274 private boolean isSearchParameterUpdateRequired(BaseResourceIndexedSearchParam theSearchParameter) { 275 return (myStorageSettings.isIndexStorageOptimized() && !theSearchParameter.isIndexStorageOptimized()) 276 || (!myStorageSettings.isIndexStorageOptimized() && theSearchParameter.isIndexStorageOptimized()); 277 } 278 279 /** 280 * The logic here is that often times when we update a resource we are dropping 281 * one index row and adding another. This method tries to reuse rows that would otherwise 282 * have been deleted by updating them with the contents of rows that would have 283 * otherwise been added. In other words, we're trying to replace 284 * "one delete + one insert" with "one update" 285 * 286 * @param theIndexesToRemove The rows that would be removed 287 * @param theIndexesToAdd The rows that would be added 288 */ 289 private <T extends BaseResourceIndex> void tryToReuseIndexEntities( 290 List<T> theIndexesToRemove, List<T> theIndexesToAdd) { 291 for (int addIndex = 0; addIndex < theIndexesToAdd.size(); addIndex++) { 292 293 // If there are no more rows to remove, there's nothing we can reuse 294 if (theIndexesToRemove.isEmpty()) { 295 break; 296 } 297 298 T targetEntity = theIndexesToAdd.get(addIndex); 299 if (targetEntity.getId() != null) { 300 continue; 301 } 302 303 // Take a row we were going to remove, and repurpose its ID 304 T entityToReuse = theIndexesToRemove.remove(theIndexesToRemove.size() - 1); 305 entityToReuse.copyMutableValuesFrom(targetEntity); 306 theIndexesToAdd.set(addIndex, entityToReuse); 307 } 308 } 309 310 public static <T> List<T> subtract(Collection<T> theSubtractFrom, Collection<T> theToSubtract) { 311 assert theSubtractFrom != theToSubtract || (theSubtractFrom.isEmpty()); 312 313 if (theSubtractFrom.isEmpty()) { 314 return new ArrayList<>(); 315 } 316 317 ArrayList<T> retVal = new ArrayList<>(); 318 for (T next : theSubtractFrom) { 319 if (!theToSubtract.contains(next)) { 320 retVal.add(next); 321 } 322 } 323 return retVal; 324 } 325 326 private interface IPreSaveHook<T> { 327 328 void preSave(Collection<T> theParamsToRemove, Collection<T> theParamsToAdd); 329 } 330 331 private class UniqueIndexPreExistenceChecker implements IPreSaveHook<ResourceIndexedComboStringUnique> { 332 333 @Override 334 public void preSave( 335 Collection<ResourceIndexedComboStringUnique> theParamsToRemove, 336 Collection<ResourceIndexedComboStringUnique> theParamsToAdd) { 337 if (myStorageSettings.isUniqueIndexesCheckedBeforeSave()) { 338 for (ResourceIndexedComboStringUnique theIndex : theParamsToAdd) { 339 ResourceIndexedComboStringUnique existing = 340 myResourceIndexedCompositeStringUniqueDao.findByQueryString(theIndex.getIndexString()); 341 if (existing != null) { 342 343 /* 344 * If we're reindexing, and the previous index row is being updated 345 * to add previously missing hashes, we may falsely detect that the index 346 * creation is going to fail. 347 */ 348 boolean existingIndexIsScheduledForRemoval = false; 349 for (var next : theParamsToRemove) { 350 if (existing == next) { 351 existingIndexIsScheduledForRemoval = true; 352 break; 353 } 354 } 355 if (existingIndexIsScheduledForRemoval) { 356 continue; 357 } 358 359 String searchParameterId = "(unknown)"; 360 if (theIndex.getSearchParameterId() != null) { 361 searchParameterId = theIndex.getSearchParameterId().getValue(); 362 } 363 364 String msg = myFhirContext 365 .getLocalizer() 366 .getMessage( 367 BaseHapiFhirDao.class, 368 "uniqueIndexConflictFailure", 369 existing.getResource().getResourceType(), 370 theIndex.getIndexString(), 371 existing.getResource() 372 .getIdDt() 373 .toUnqualifiedVersionless() 374 .getValue(), 375 searchParameterId); 376 377 // Use ResourceVersionConflictException here because the HapiTransactionService 378 // catches this and can retry it if needed 379 throw new ResourceVersionConflictException(Msg.code(1093) + msg); 380 } 381 } 382 } 383 } 384 } 385}