View Javadoc
1   package ca.uhn.fhir.jpa.term;
2   
3   /*
4    * #%L
5    * HAPI FHIR JPA Server
6    * %%
7    * Copyright (C) 2014 - 2018 University Health Network
8    * %%
9    * Licensed under the Apache License, Version 2.0 (the "License");
10   * you may not use this file except in compliance with the License.
11   * You may obtain a copy of the License at
12   * 
13   * http://www.apache.org/licenses/LICENSE-2.0
14   * 
15   * Unless required by applicable law or agreed to in writing, software
16   * distributed under the License is distributed on an "AS IS" BASIS,
17   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18   * See the License for the specific language governing permissions and
19   * limitations under the License.
20   * #L%
21   */
22  
23  import ca.uhn.fhir.context.FhirContext;
24  import ca.uhn.fhir.jpa.dao.BaseHapiFhirDao;
25  import ca.uhn.fhir.jpa.dao.DaoConfig;
26  import ca.uhn.fhir.jpa.dao.IFhirResourceDaoCodeSystem;
27  import ca.uhn.fhir.jpa.dao.data.*;
28  import ca.uhn.fhir.jpa.entity.*;
29  import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
30  import ca.uhn.fhir.jpa.util.ScrollableResultsIterator;
31  import ca.uhn.fhir.rest.api.server.RequestDetails;
32  import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
33  import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
34  import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
35  import ca.uhn.fhir.util.ObjectUtil;
36  import ca.uhn.fhir.util.StopWatch;
37  import ca.uhn.fhir.util.ValidateUtil;
38  import com.github.benmanes.caffeine.cache.Cache;
39  import com.github.benmanes.caffeine.cache.Caffeine;
40  import com.google.common.annotations.VisibleForTesting;
41  import com.google.common.base.Stopwatch;
42  import com.google.common.collect.ArrayListMultimap;
43  import org.apache.commons.lang3.Validate;
44  import org.apache.commons.lang3.time.DateUtils;
45  import org.apache.lucene.search.Query;
46  import org.hibernate.ScrollMode;
47  import org.hibernate.ScrollableResults;
48  import org.hibernate.search.jpa.FullTextEntityManager;
49  import org.hibernate.search.jpa.FullTextQuery;
50  import org.hibernate.search.query.dsl.BooleanJunction;
51  import org.hibernate.search.query.dsl.QueryBuilder;
52  import org.hl7.fhir.exceptions.FHIRException;
53  import org.hl7.fhir.instance.model.api.IIdType;
54  import org.hl7.fhir.r4.model.CodeSystem;
55  import org.hl7.fhir.r4.model.Coding;
56  import org.hl7.fhir.r4.model.ConceptMap;
57  import org.hl7.fhir.r4.model.ValueSet;
58  import org.springframework.beans.BeansException;
59  import org.springframework.beans.factory.annotation.Autowired;
60  import org.springframework.context.ApplicationContext;
61  import org.springframework.context.ApplicationContextAware;
62  import org.springframework.data.domain.Page;
63  import org.springframework.data.domain.PageRequest;
64  import org.springframework.scheduling.annotation.Scheduled;
65  import org.springframework.transaction.PlatformTransactionManager;
66  import org.springframework.transaction.TransactionStatus;
67  import org.springframework.transaction.annotation.Propagation;
68  import org.springframework.transaction.annotation.Transactional;
69  import org.springframework.transaction.support.TransactionCallbackWithoutResult;
70  import org.springframework.transaction.support.TransactionTemplate;
71  
72  import javax.annotation.PostConstruct;
73  import javax.persistence.EntityManager;
74  import javax.persistence.PersistenceContext;
75  import javax.persistence.PersistenceContextType;
76  import javax.persistence.TypedQuery;
77  import javax.persistence.criteria.*;
78  import java.util.*;
79  import java.util.concurrent.TimeUnit;
80  import java.util.stream.Collectors;
81  
82  import static org.apache.commons.lang3.StringUtils.isBlank;
83  import static org.apache.commons.lang3.StringUtils.isNotBlank;
84  
85  public abstract class BaseHapiTerminologySvcImpl implements IHapiTerminologySvc, ApplicationContextAware {
86  	public static final int DEFAULT_FETCH_SIZE = 250;
87  
88  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseHapiTerminologySvcImpl.class);
89  	private static final Object PLACEHOLDER_OBJECT = new Object();
90  	private static boolean ourForceSaveDeferredAlwaysForUnitTest;
91  	private static boolean ourLastResultsFromTranslationCache; // For testing.
92  	private static boolean ourLastResultsFromTranslationWithReverseCache; // For testing.
93  	@Autowired
94  	protected ITermCodeSystemDao myCodeSystemDao;
95  	@Autowired
96  	protected ITermConceptDao myConceptDao;
97  	@Autowired
98  	protected ITermConceptMapDao myConceptMapDao;
99  	@Autowired
100 	protected ITermConceptMapGroupDao myConceptMapGroupDao;
101 	@Autowired
102 	protected ITermConceptMapGroupElementDao myConceptMapGroupElementDao;
103 	@Autowired
104 	protected ITermConceptMapGroupElementTargetDao myConceptMapGroupElementTargetDao;
105 	@Autowired
106 	protected ITermConceptPropertyDao myConceptPropertyDao;
107 	@Autowired
108 	protected ITermConceptDesignationDao myConceptDesignationDao;
109 	@Autowired
110 	protected FhirContext myContext;
111 	@PersistenceContext(type = PersistenceContextType.TRANSACTION)
112 	protected EntityManager myEntityManager;
113 	private ArrayListMultimap<Long, Long> myChildToParentPidCache;
114 	@Autowired
115 	private ITermCodeSystemVersionDao myCodeSystemVersionDao;
116 	private List<TermConceptParentChildLink> myConceptLinksToSaveLater = new ArrayList<>();
117 	@Autowired
118 	private ITermConceptParentChildLinkDao myConceptParentChildLinkDao;
119 	private List<TermConcept> myDeferredConcepts = Collections.synchronizedList(new ArrayList<>());
120 	private List<ValueSet> myDeferredValueSets = Collections.synchronizedList(new ArrayList<>());
121 	private List<ConceptMap> myDeferredConceptMaps = Collections.synchronizedList(new ArrayList<>());
122 	@Autowired
123 	private DaoConfig myDaoConfig;
124 	private long myNextReindexPass;
125 	private boolean myProcessDeferred = true;
126 	@Autowired
127 	private PlatformTransactionManager myTransactionMgr;
128 	private IFhirResourceDaoCodeSystem<?, ?, ?> myCodeSystemResourceDao;
129 	private Cache<TranslationQuery, List<TermConceptMapGroupElementTarget>> myTranslationCache;
130 	private Cache<TranslationQuery, List<TermConceptMapGroupElement>> myTranslationWithReverseCache;
131 	private int myFetchSize = DEFAULT_FETCH_SIZE;
132 	private ApplicationContext myApplicationContext;
133 
134 	private void addCodeIfNotAlreadyAdded(String theCodeSystem, ValueSet.ValueSetExpansionComponent theExpansionComponent, Set<String> theAddedCodes, TermConcept theConcept) {
135 		if (theAddedCodes.add(theConcept.getCode())) {
136 			ValueSet.ValueSetExpansionContainsComponent contains = theExpansionComponent.addContains();
137 			contains.setCode(theConcept.getCode());
138 			contains.setSystem(theCodeSystem);
139 			contains.setDisplay(theConcept.getDisplay());
140 			for (TermConceptDesignation nextDesignation : theConcept.getDesignations()) {
141 				contains
142 					.addDesignation()
143 					.setValue(nextDesignation.getValue())
144 					.getUse()
145 					.setSystem(nextDesignation.getUseSystem())
146 					.setCode(nextDesignation.getUseCode())
147 					.setDisplay(nextDesignation.getUseDisplay());
148 			}
149 		}
150 	}
151 
152 	private void addConceptsToList(ValueSet.ValueSetExpansionComponent theExpansionComponent, Set<String> theAddedCodes, String theSystem, List<CodeSystem.ConceptDefinitionComponent> theConcept) {
153 		for (CodeSystem.ConceptDefinitionComponent next : theConcept) {
154 			if (!theAddedCodes.contains(next.getCode())) {
155 				theAddedCodes.add(next.getCode());
156 				ValueSet.ValueSetExpansionContainsComponent contains = theExpansionComponent.addContains();
157 				contains.setCode(next.getCode());
158 				contains.setSystem(theSystem);
159 				contains.setDisplay(next.getDisplay());
160 			}
161 			addConceptsToList(theExpansionComponent, theAddedCodes, theSystem, next.getConcept());
162 		}
163 	}
164 
165 	private void addDisplayFilterExact(QueryBuilder qb, BooleanJunction<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
166 		bool.must(qb.phrase().onField("myDisplay").sentence(nextFilter.getValue()).createQuery());
167 	}
168 
169 	private void addDisplayFilterInexact(QueryBuilder qb, BooleanJunction<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
170 		Query textQuery = qb
171 			.phrase()
172 			.withSlop(2)
173 			.onField("myDisplay").boostedTo(4.0f)
174 			.andField("myDisplayEdgeNGram").boostedTo(2.0f)
175 			// .andField("myDisplayNGram").boostedTo(1.0f)
176 			// .andField("myDisplayPhonetic").boostedTo(0.5f)
177 			.sentence(nextFilter.getValue().toLowerCase()).createQuery();
178 		bool.must(textQuery);
179 	}
180 
181 	private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) {
182 		boolean retVal = theSetToPopulate.add(theConcept);
183 		if (retVal) {
184 			if (theSetToPopulate.size() >= myDaoConfig.getMaximumExpansionSize()) {
185 				String msg = myContext.getLocalizer().getMessage(BaseHapiTerminologySvcImpl.class, "expansionTooLarge", myDaoConfig.getMaximumExpansionSize());
186 				throw new InvalidRequestException(msg);
187 			}
188 		}
189 		return retVal;
190 	}
191 
192 	@PostConstruct
193 	public void buildTranslationCaches() {
194 		Long timeout = myDaoConfig.getTranslationCachesExpireAfterWriteInMinutes();
195 
196 		myTranslationCache =
197 			Caffeine.newBuilder()
198 				.maximumSize(10000)
199 				.expireAfterWrite(timeout, TimeUnit.MINUTES)
200 				.build();
201 
202 		myTranslationWithReverseCache =
203 			Caffeine.newBuilder()
204 				.maximumSize(10000)
205 				.expireAfterWrite(timeout, TimeUnit.MINUTES)
206 				.build();
207 	}
208 
209 	/**
210 	 * This method is present only for unit tests, do not call from client code
211 	 */
212 	@VisibleForTesting
213 	public void clearDeferred() {
214 		myDeferredValueSets.clear();
215 		myDeferredConceptMaps.clear();
216 		myDeferredConcepts.clear();
217 	}
218 
219 	/**
220 	 * This method is present only for unit tests, do not call from client code
221 	 */
222 	@VisibleForTesting
223 	public void clearTranslationCache() {
224 		myTranslationCache.invalidateAll();
225 	}
226 
227 	/**
228 	 * This method is present only for unit tests, do not call from client code
229 	 */
230 	@VisibleForTesting()
231 	public void clearTranslationWithReverseCache() {
232 		myTranslationWithReverseCache.invalidateAll();
233 	}
234 
235 	protected abstract IIdType createOrUpdateCodeSystem(CodeSystem theCodeSystemResource);
236 
237 	protected abstract void createOrUpdateConceptMap(ConceptMap theNextConceptMap);
238 
239 	abstract void createOrUpdateValueSet(ValueSet theValueSet);
240 
241 	@Override
242 	public void deleteCodeSystem(TermCodeSystem theCodeSystem) {
243 		ourLog.info(" * Deleting code system {}", theCodeSystem.getPid());
244 
245 		myEntityManager.flush();
246 		TermCodeSystem cs = myCodeSystemDao.findOne(theCodeSystem.getPid());
247 		cs.setCurrentVersion(null);
248 		myCodeSystemDao.save(cs);
249 		myCodeSystemDao.flush();
250 
251 		int i = 0;
252 		for (TermCodeSystemVersion next : myCodeSystemVersionDao.findByCodeSystemResource(theCodeSystem.getPid())) {
253 			myConceptParentChildLinkDao.deleteByCodeSystemVersion(next.getPid());
254 			for (TermConcept nextConcept : myConceptDao.findByCodeSystemVersion(next.getPid())) {
255 				myConceptPropertyDao.delete(nextConcept.getProperties());
256 				myConceptDesignationDao.delete(nextConcept.getDesignations());
257 				myConceptDao.delete(nextConcept);
258 			}
259 			if (next.getCodeSystem().getCurrentVersion() == next) {
260 				next.getCodeSystem().setCurrentVersion(null);
261 				myCodeSystemDao.save(next.getCodeSystem());
262 			}
263 			myCodeSystemVersionDao.delete(next);
264 
265 			if (i++ % 1000 == 0) {
266 				myEntityManager.flush();
267 			}
268 		}
269 		myCodeSystemVersionDao.deleteForCodeSystem(theCodeSystem);
270 		myCodeSystemDao.delete(theCodeSystem);
271 
272 		myEntityManager.flush();
273 	}
274 
275 	private int ensureParentsSaved(Collection<TermConceptParentChildLink> theParents) {
276 		ourLog.trace("Checking {} parents", theParents.size());
277 		int retVal = 0;
278 
279 		for (TermConceptParentChildLink nextLink : theParents) {
280 			if (nextLink.getRelationshipType() == RelationshipTypeEnum.ISA) {
281 				TermConcept nextParent = nextLink.getParent();
282 				retVal += ensureParentsSaved(nextParent.getParents());
283 				if (nextParent.getId() == null) {
284 					myConceptDao.saveAndFlush(nextParent);
285 					retVal++;
286 					ourLog.debug("Saved parent code {} and got id {}", nextParent.getCode(), nextParent.getId());
287 				}
288 			}
289 		}
290 
291 		return retVal;
292 	}
293 
294 	@Override
295 	@Transactional(propagation = Propagation.REQUIRED)
296 	public ValueSet expandValueSet(ValueSet theValueSetToExpand) {
297 		ValueSet.ValueSetExpansionComponent expansionComponent = new ValueSet.ValueSetExpansionComponent();
298 		Set<String> addedCodes = new HashSet<>();
299 		boolean haveIncludeCriteria = false;
300 
301 		for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) {
302 			String system = include.getSystem();
303 			if (isNotBlank(system)) {
304 				ourLog.info("Starting expansion around code system: {}", system);
305 
306 				TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
307 				if (cs != null) {
308 					TermCodeSystemVersion csv = cs.getCurrentVersion();
309 
310 					/*
311 					 * Include Concepts
312 					 */
313 					for (ValueSet.ConceptReferenceComponent next : include.getConcept()) {
314 						String nextCode = next.getCode();
315 						if (isNotBlank(nextCode) && !addedCodes.contains(nextCode)) {
316 							haveIncludeCriteria = true;
317 							TermConcept code = findCode(system, nextCode);
318 							if (code != null) {
319 								addCodeIfNotAlreadyAdded(system, expansionComponent, addedCodes, code);
320 							}
321 						}
322 					}
323 
324 					/*
325 					 * Filters
326 					 */
327 
328 					if (include.getFilter().size() > 0) {
329 						haveIncludeCriteria = true;
330 
331 						FullTextEntityManager em = org.hibernate.search.jpa.Search.getFullTextEntityManager(myEntityManager);
332 						QueryBuilder qb = em.getSearchFactory().buildQueryBuilder().forEntity(TermConcept.class).get();
333 						BooleanJunction<?> bool = qb.bool();
334 
335 						bool.must(qb.keyword().onField("myCodeSystemVersionPid").matching(csv.getPid()).createQuery());
336 
337 						for (ValueSet.ConceptSetFilterComponent nextFilter : include.getFilter()) {
338 							if (isBlank(nextFilter.getValue()) && nextFilter.getOp() == null && isBlank(nextFilter.getProperty())) {
339 								continue;
340 							}
341 
342 							if (isBlank(nextFilter.getValue()) || nextFilter.getOp() == null || isBlank(nextFilter.getProperty())) {
343 								throw new InvalidRequestException("Invalid filter, must have fields populated: property op value");
344 							}
345 
346 
347 							if (nextFilter.getProperty().equals("display:exact") && nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
348 								addDisplayFilterExact(qb, bool, nextFilter);
349 							} else if ("display".equals(nextFilter.getProperty()) && nextFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
350 								if (nextFilter.getValue().trim().contains(" ")) {
351 									addDisplayFilterExact(qb, bool, nextFilter);
352 								} else {
353 									addDisplayFilterInexact(qb, bool, nextFilter);
354 								}
355 							} else if ((nextFilter.getProperty().equals("concept") || nextFilter.getProperty().equals("code")) && nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
356 
357 								TermConcept code = findCode(system, nextFilter.getValue());
358 								if (code == null) {
359 									throw new InvalidRequestException("Invalid filter criteria - code does not exist: {" + system + "}" + nextFilter.getValue());
360 								}
361 
362 								ourLog.info(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
363 								bool.must(qb.keyword().onField("myParentPids").matching("" + code.getId()).createQuery());
364 
365 							} else {
366 
367 								bool.must(qb.phrase().onField("myProperties").sentence(nextFilter.getProperty() + "=" + nextFilter.getValue()).createQuery());
368 
369 							}
370 						}
371 
372 						Query luceneQuery = bool.createQuery();
373 						FullTextQuery jpaQuery = em.createFullTextQuery(luceneQuery, TermConcept.class);
374 						jpaQuery.setMaxResults(1000);
375 
376 						StopWatch sw = new StopWatch();
377 
378 						@SuppressWarnings("unchecked")
379 						List<TermConcept> result = jpaQuery.getResultList();
380 
381 						ourLog.info("Expansion completed in {}ms", sw.getMillis());
382 
383 						for (TermConcept nextConcept : result) {
384 							addCodeIfNotAlreadyAdded(system, expansionComponent, addedCodes, nextConcept);
385 						}
386 
387 						expansionComponent.setTotal(jpaQuery.getResultSize());
388 					}
389 
390 					if (!haveIncludeCriteria) {
391 						List<TermConcept> allCodes = findCodes(system);
392 						for (TermConcept nextConcept : allCodes) {
393 							addCodeIfNotAlreadyAdded(system, expansionComponent, addedCodes, nextConcept);
394 						}
395 					}
396 
397 				} else {
398 					// No codesystem matching the URL found in the database
399 
400 					CodeSystem codeSystemFromContext = getCodeSystemFromContext(system);
401 					if (codeSystemFromContext == null) {
402 						throw new InvalidRequestException("Unknown code system: " + system);
403 					}
404 
405 					if (include.getConcept().isEmpty() == false) {
406 						for (ValueSet.ConceptReferenceComponent next : include.getConcept()) {
407 							String nextCode = next.getCode();
408 							if (isNotBlank(nextCode) && !addedCodes.contains(nextCode)) {
409 								CodeSystem.ConceptDefinitionComponent code = findCode(codeSystemFromContext.getConcept(), nextCode);
410 								if (code != null) {
411 									addedCodes.add(nextCode);
412 									ValueSet.ValueSetExpansionContainsComponent contains = expansionComponent.addContains();
413 									contains.setCode(nextCode);
414 									contains.setSystem(system);
415 									contains.setDisplay(code.getDisplay());
416 								}
417 							}
418 						}
419 					} else {
420 						List<CodeSystem.ConceptDefinitionComponent> concept = codeSystemFromContext.getConcept();
421 						addConceptsToList(expansionComponent, addedCodes, system, concept);
422 					}
423 
424 				}
425 			}
426 		}
427 
428 		ValueSet valueSet = new ValueSet();
429 		valueSet.setExpansion(expansionComponent);
430 		return valueSet;
431 	}
432 
433 	protected List<VersionIndependentConcept> expandValueSetAndReturnVersionIndependentConcepts(org.hl7.fhir.r4.model.ValueSet theValueSetToExpandR4) {
434 		org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent expandedR4 = expandValueSet(theValueSetToExpandR4).getExpansion();
435 
436 		ArrayList<VersionIndependentConcept> retVal = new ArrayList<>();
437 		for (org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent nextContains : expandedR4.getContains()) {
438 			retVal.add(
439 				new VersionIndependentConcept()
440 					.setSystem(nextContains.getSystem())
441 					.setCode(nextContains.getCode()));
442 		}
443 		return retVal;
444 	}
445 
446 	private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
447 		for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) {
448 			TermConcept nextChild = nextChildLink.getChild();
449 			if (addToSet(theSetToPopulate, nextChild)) {
450 				fetchChildren(nextChild, theSetToPopulate);
451 			}
452 		}
453 	}
454 
455 	private TermConcept fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) {
456 		TermCodeSystemVersion codeSystem = myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid);
457 		return myConceptDao.findByCodeSystemAndCode(codeSystem, theCode);
458 	}
459 
460 	private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
461 		for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) {
462 			TermConcept nextChild = nextChildLink.getParent();
463 			if (addToSet(theSetToPopulate, nextChild)) {
464 				fetchParents(nextChild, theSetToPopulate);
465 			}
466 		}
467 	}
468 
469 	private CodeSystem.ConceptDefinitionComponent findCode(List<CodeSystem.ConceptDefinitionComponent> theConcepts, String theCode) {
470 		for (CodeSystem.ConceptDefinitionComponent next : theConcepts) {
471 			if (theCode.equals(next.getCode())) {
472 				return next;
473 			}
474 			findCode(next.getConcept(), theCode);
475 		}
476 		return null;
477 	}
478 
479 	@Override
480 	public TermConcept findCode(String theCodeSystem, String theCode) {
481 		TermCodeSystemVersion csv = findCurrentCodeSystemVersionForSystem(theCodeSystem);
482 
483 		return myConceptDao.findByCodeSystemAndCode(csv, theCode);
484 	}
485 
486 	@Override
487 	public List<TermConcept> findCodes(String theSystem) {
488 		return myConceptDao.findByCodeSystemVersion(findCurrentCodeSystemVersionForSystem(theSystem));
489 	}
490 
491 	@Transactional(propagation = Propagation.REQUIRED)
492 	@Override
493 	public Set<TermConcept> findCodesAbove(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
494 		StopWatch stopwatch = new StopWatch();
495 
496 		TermConcept concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
497 		if (concept == null) {
498 			return Collections.emptySet();
499 		}
500 
501 		Set<TermConcept> retVal = new HashSet<>();
502 		retVal.add(concept);
503 
504 		fetchParents(concept, retVal);
505 
506 		ourLog.info("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
507 		return retVal;
508 	}
509 
510 	@Override
511 	public List<VersionIndependentConcept> findCodesAbove(String theSystem, String theCode) {
512 		TermCodeSystem cs = getCodeSystem(theSystem);
513 		if (cs == null) {
514 			return findCodesAboveUsingBuiltInSystems(theSystem, theCode);
515 		}
516 		TermCodeSystemVersion csv = cs.getCurrentVersion();
517 
518 		Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode);
519 		return toVersionIndependentConcepts(theSystem, codes);
520 	}
521 
522 	@Transactional(propagation = Propagation.REQUIRED)
523 	@Override
524 	public Set<TermConcept> findCodesBelow(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
525 		Stopwatch stopwatch = Stopwatch.createStarted();
526 
527 		TermConcept concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
528 		if (concept == null) {
529 			return Collections.emptySet();
530 		}
531 
532 		Set<TermConcept> retVal = new HashSet<>();
533 		retVal.add(concept);
534 
535 		fetchChildren(concept, retVal);
536 
537 		ourLog.info("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS));
538 		return retVal;
539 	}
540 
541 	@Override
542 	public List<VersionIndependentConcept> findCodesBelow(String theSystem, String theCode) {
543 		TermCodeSystem cs = getCodeSystem(theSystem);
544 		if (cs == null) {
545 			return findCodesBelowUsingBuiltInSystems(theSystem, theCode);
546 		}
547 		TermCodeSystemVersion csv = cs.getCurrentVersion();
548 
549 		Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode);
550 		return toVersionIndependentConcepts(theSystem, codes);
551 	}
552 
553 	private TermCodeSystemVersion findCurrentCodeSystemVersionForSystem(String theCodeSystem) {
554 		TermCodeSystem cs = getCodeSystem(theCodeSystem);
555 		if (cs == null || cs.getCurrentVersion() == null) {
556 			return null;
557 		}
558 		return cs.getCurrentVersion();
559 	}
560 
561 	private TermCodeSystem getCodeSystem(String theSystem) {
562 		return myCodeSystemDao.findByCodeSystemUri(theSystem);
563 	}
564 
565 	protected abstract CodeSystem getCodeSystemFromContext(String theSystem);
566 
567 	private void persistChildren(TermConcept theConcept, TermCodeSystemVersion theCodeSystem, IdentityHashMap<TermConcept, Object> theConceptsStack, int theTotalConcepts) {
568 		if (theConceptsStack.put(theConcept, PLACEHOLDER_OBJECT) != null) {
569 			return;
570 		}
571 
572 		if (theConceptsStack.size() == 1 || theConceptsStack.size() % 10000 == 0) {
573 			float pct = (float) theConceptsStack.size() / (float) theTotalConcepts;
574 			ourLog.info("Have processed {}/{} concepts ({}%)", theConceptsStack.size(), theTotalConcepts, (int) (pct * 100.0f));
575 		}
576 
577 		theConcept.setCodeSystemVersion(theCodeSystem);
578 		theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED);
579 
580 		if (theConceptsStack.size() <= myDaoConfig.getDeferIndexingForCodesystemsOfSize()) {
581 			saveConcept(theConcept);
582 		} else {
583 			myDeferredConcepts.add(theConcept);
584 		}
585 
586 		for (TermConceptParentChildLink next : theConcept.getChildren()) {
587 			persistChildren(next.getChild(), theCodeSystem, theConceptsStack, theTotalConcepts);
588 		}
589 
590 		for (TermConceptParentChildLink next : theConcept.getChildren()) {
591 			if (theConceptsStack.size() <= myDaoConfig.getDeferIndexingForCodesystemsOfSize()) {
592 				saveConceptLink(next);
593 			} else {
594 				myConceptLinksToSaveLater.add(next);
595 			}
596 		}
597 
598 	}
599 
600 	private void populateVersion(TermConcept theNext, TermCodeSystemVersion theCodeSystemVersion) {
601 		if (theNext.getCodeSystemVersion() != null) {
602 			return;
603 		}
604 		theNext.setCodeSystemVersion(theCodeSystemVersion);
605 		for (TermConceptParentChildLink next : theNext.getChildren()) {
606 			populateVersion(next.getChild(), theCodeSystemVersion);
607 		}
608 	}
609 
610 	private void processDeferredConceptMaps() {
611 		int count = Math.min(myDeferredConceptMaps.size(), 20);
612 		for (ConceptMap nextConceptMap : new ArrayList<>(myDeferredConceptMaps.subList(0, count))) {
613 			ourLog.info("Creating ConceptMap: {}", nextConceptMap.getId());
614 			createOrUpdateConceptMap(nextConceptMap);
615 			myDeferredConceptMaps.remove(nextConceptMap);
616 		}
617 		ourLog.info("Saved {} deferred ConceptMap resources, have {} remaining", count, myDeferredConceptMaps.size());
618 	}
619 
620 	private void processDeferredConcepts() {
621 		int codeCount = 0, relCount = 0;
622 		StopWatch stopwatch = new StopWatch();
623 
624 		int count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myDeferredConcepts.size());
625 		ourLog.info("Saving {} deferred concepts...", count);
626 		while (codeCount < count && myDeferredConcepts.size() > 0) {
627 			TermConcept next = myDeferredConcepts.remove(0);
628 			codeCount += saveConcept(next);
629 		}
630 
631 		if (codeCount > 0) {
632 			ourLog.info("Saved {} deferred concepts ({} codes remain and {} relationships remain) in {}ms ({}ms / code)",
633 				codeCount, myDeferredConcepts.size(), myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount));
634 		}
635 
636 		if (codeCount == 0) {
637 			count = Math.min(myDaoConfig.getDeferIndexingForCodesystemsOfSize(), myConceptLinksToSaveLater.size());
638 			ourLog.info("Saving {} deferred concept relationships...", count);
639 			while (relCount < count && myConceptLinksToSaveLater.size() > 0) {
640 				TermConceptParentChildLink next = myConceptLinksToSaveLater.remove(0);
641 
642 				if (myConceptDao.findOne(next.getChild().getId()) == null || myConceptDao.findOne(next.getParent().getId()) == null) {
643 					ourLog.warn("Not inserting link from child {} to parent {} because it appears to have been deleted", next.getParent().getCode(), next.getChild().getCode());
644 					continue;
645 				}
646 
647 				saveConceptLink(next);
648 				relCount++;
649 			}
650 		}
651 
652 		if (relCount > 0) {
653 			ourLog.info("Saved {} deferred relationships ({} remain) in {}ms ({}ms / code)",
654 				relCount, myConceptLinksToSaveLater.size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(codeCount));
655 		}
656 
657 		if ((myDeferredConcepts.size() + myConceptLinksToSaveLater.size()) == 0) {
658 			ourLog.info("All deferred concepts and relationships have now been synchronized to the database");
659 		}
660 	}
661 
662 	private void processDeferredValueSets() {
663 		int count = Math.min(myDeferredValueSets.size(), 20);
664 		for (ValueSet nextValueSet : new ArrayList<>(myDeferredValueSets.subList(0, count))) {
665 			ourLog.info("Creating ValueSet: {}", nextValueSet.getId());
666 			createOrUpdateValueSet(nextValueSet);
667 			myDeferredValueSets.remove(nextValueSet);
668 		}
669 		ourLog.info("Saved {} deferred ValueSet resources, have {} remaining", count, myDeferredValueSets.size());
670 	}
671 
672 	private void processReindexing() {
673 		if (System.currentTimeMillis() < myNextReindexPass && !ourForceSaveDeferredAlwaysForUnitTest) {
674 			return;
675 		}
676 
677 		TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
678 		tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
679 		tt.execute(new TransactionCallbackWithoutResult() {
680 			private void createParentsString(StringBuilder theParentsBuilder, Long theConceptPid) {
681 				Validate.notNull(theConceptPid, "theConceptPid must not be null");
682 				List<Long> parents = myChildToParentPidCache.get(theConceptPid);
683 				if (parents.contains(-1L)) {
684 					return;
685 				} else if (parents.isEmpty()) {
686 					Collection<Long> parentLinks = myConceptParentChildLinkDao.findAllWithChild(theConceptPid);
687 					if (parentLinks.isEmpty()) {
688 						myChildToParentPidCache.put(theConceptPid, -1L);
689 						ourLog.info("Found {} parent concepts of concept {} (cache has {})", 0, theConceptPid, myChildToParentPidCache.size());
690 						return;
691 					} else {
692 						for (Long next : parentLinks) {
693 							myChildToParentPidCache.put(theConceptPid, next);
694 						}
695 						int parentCount = myChildToParentPidCache.get(theConceptPid).size();
696 						ourLog.info("Found {} parent concepts of concept {} (cache has {})", parentCount, theConceptPid, myChildToParentPidCache.size());
697 					}
698 				}
699 
700 				for (Long nextParent : parents) {
701 					if (theParentsBuilder.length() > 0) {
702 						theParentsBuilder.append(' ');
703 					}
704 					theParentsBuilder.append(nextParent);
705 					createParentsString(theParentsBuilder, nextParent);
706 				}
707 
708 			}
709 
710 			@Override
711 			protected void doInTransactionWithoutResult(TransactionStatus theArg0) {
712 				int maxResult = 1000;
713 				Page<TermConcept> concepts = myConceptDao.findResourcesRequiringReindexing(new PageRequest(0, maxResult));
714 				if (concepts.hasContent() == false) {
715 					if (myChildToParentPidCache != null) {
716 						ourLog.info("Clearing parent concept cache");
717 						myNextReindexPass = System.currentTimeMillis() + DateUtils.MILLIS_PER_MINUTE;
718 						myChildToParentPidCache = null;
719 					}
720 					return;
721 				}
722 
723 				if (myChildToParentPidCache == null) {
724 					myChildToParentPidCache = ArrayListMultimap.create();
725 				}
726 
727 				ourLog.info("Indexing {} / {} concepts", concepts.getContent().size(), concepts.getTotalElements());
728 
729 				int count = 0;
730 				StopWatch stopwatch = new StopWatch();
731 
732 				for (TermConcept nextConcept : concepts) {
733 
734 					StringBuilder parentsBuilder = new StringBuilder();
735 					createParentsString(parentsBuilder, nextConcept.getId());
736 					nextConcept.setParentPids(parentsBuilder.toString());
737 
738 					saveConcept(nextConcept);
739 					count++;
740 				}
741 
742 				ourLog.info("Indexed {} / {} concepts in {}ms - Avg {}ms / resource", count, concepts.getContent().size(), stopwatch.getMillis(), stopwatch.getMillisPerOperation(count));
743 			}
744 		});
745 
746 	}
747 
748 	private int saveConcept(TermConcept theConcept) {
749 		int retVal = 0;
750 
751 		/*
752 		 * If the concept has an ID, we're reindexing, so there's no need to
753 		 * save parent concepts first (it's way too slow to do that)
754 		 */
755 		if (theConcept.getId() == null) {
756 			retVal += ensureParentsSaved(theConcept.getParents());
757 		}
758 
759 		if (theConcept.getId() == null || theConcept.getIndexStatus() == null) {
760 			retVal++;
761 			theConcept.setIndexStatus(BaseHapiFhirDao.INDEX_STATUS_INDEXED);
762 			myConceptDao.save(theConcept);
763 
764 			for (TermConceptProperty next : theConcept.getProperties()) {
765 				myConceptPropertyDao.save(next);
766 			}
767 
768 			for (TermConceptDesignation next : theConcept.getDesignations()) {
769 				myConceptDesignationDao.save(next);
770 			}
771 		}
772 
773 		ourLog.trace("Saved {} and got PID {}", theConcept.getCode(), theConcept.getId());
774 		return retVal;
775 	}
776 
777 	private void saveConceptLink(TermConceptParentChildLink next) {
778 		if (next.getId() == null) {
779 			myConceptParentChildLinkDao.save(next);
780 		}
781 	}
782 
783 	@Scheduled(fixedRate = 5000)
784 	@Transactional(propagation = Propagation.NEVER)
785 	@Override
786 	public synchronized void saveDeferred() {
787 		if (!myProcessDeferred) {
788 			return;
789 		} else if (myDeferredConcepts.isEmpty() && myConceptLinksToSaveLater.isEmpty()) {
790 			processReindexing();
791 			return;
792 		}
793 
794 		TransactionTemplate tt = new TransactionTemplate(myTransactionMgr);
795 		tt.setPropagationBehavior(TransactionTemplate.PROPAGATION_REQUIRES_NEW);
796 		tt.execute(t -> {
797 			processDeferredConcepts();
798 			return null;
799 		});
800 
801 		if (myDeferredValueSets.size() > 0) {
802 			tt.execute(t -> {
803 				processDeferredValueSets();
804 				return null;
805 			});
806 		}
807 		if (myDeferredConceptMaps.size() > 0) {
808 			tt.execute(t -> {
809 				processDeferredConceptMaps();
810 				return null;
811 			});
812 		}
813 
814 	}
815 
816 	@Override
817 	public void setApplicationContext(ApplicationContext theApplicationContext) throws BeansException {
818 		myApplicationContext = theApplicationContext;
819 	}
820 
821 	@Override
822 	public void setProcessDeferred(boolean theProcessDeferred) {
823 		myProcessDeferred = theProcessDeferred;
824 	}
825 
826 	@PostConstruct
827 	public void start() {
828 		myCodeSystemResourceDao = myApplicationContext.getBean(IFhirResourceDaoCodeSystem.class);
829 	}
830 
831 	@Override
832 	@Transactional(propagation = Propagation.REQUIRED)
833 	public void storeNewCodeSystemVersion(Long theCodeSystemResourcePid, String theSystemUri, String theSystemName, TermCodeSystemVersion theCodeSystemVersion) {
834 		ourLog.info("Storing code system");
835 
836 		ValidateUtil.isTrueOrThrowInvalidRequest(theCodeSystemVersion.getResource() != null, "No resource supplied");
837 		ValidateUtil.isNotBlankOrThrowInvalidRequest(theSystemUri, "No system URI supplied");
838 
839 		// Grab the existing versions so we can delete them later
840 		List<TermCodeSystemVersion> existing = myCodeSystemVersionDao.findByCodeSystemResource(theCodeSystemResourcePid);
841 
842 //		verifyNoDuplicates(theCodeSystemVersion.getConcepts(), new HashSet<String>());
843 
844 		/*
845 		 * For now we always delete old versions.. At some point it would be nice to allow configuration to keep old versions
846 		 */
847 
848 		ourLog.info("Deleting old code system versions");
849 		for (TermCodeSystemVersion next : existing) {
850 			ourLog.info(" * Deleting code system version {}", next.getPid());
851 			myConceptParentChildLinkDao.deleteByCodeSystemVersion(next.getPid());
852 			for (TermConcept nextConcept : myConceptDao.findByCodeSystemVersion(next.getPid())) {
853 				myConceptPropertyDao.delete(nextConcept.getProperties());
854 				myConceptDao.delete(nextConcept);
855 			}
856 		}
857 
858 		ourLog.info("Flushing...");
859 
860 		myConceptParentChildLinkDao.flush();
861 		myConceptPropertyDao.flush();
862 		myConceptDao.flush();
863 
864 		ourLog.info("Done flushing");
865 
866 		/*
867 		 * Do the upload
868 		 */
869 
870 		TermCodeSystem codeSystem = getCodeSystem(theSystemUri);
871 		if (codeSystem == null) {
872 			codeSystem = myCodeSystemDao.findByResourcePid(theCodeSystemResourcePid);
873 			if (codeSystem == null) {
874 				codeSystem = new TermCodeSystem();
875 			}
876 			codeSystem.setResource(theCodeSystemVersion.getResource());
877 			codeSystem.setCodeSystemUri(theSystemUri);
878 			codeSystem.setName(theSystemName);
879 			myCodeSystemDao.save(codeSystem);
880 		} else {
881 			if (!ObjectUtil.equals(codeSystem.getResource().getId(), theCodeSystemVersion.getResource().getId())) {
882 				String msg = myContext.getLocalizer().getMessage(BaseHapiTerminologySvcImpl.class, "cannotCreateDuplicateCodeSystemUri", theSystemUri,
883 					codeSystem.getResource().getIdDt().toUnqualifiedVersionless().getValue());
884 				throw new UnprocessableEntityException(msg);
885 			}
886 		}
887 		theCodeSystemVersion.setCodeSystem(codeSystem);
888 
889 		ourLog.info("Validating all codes in CodeSystem for storage (this can take some time for large sets)");
890 
891 		// Validate the code system
892 		ArrayList<String> conceptsStack = new ArrayList<>();
893 		IdentityHashMap<TermConcept, Object> allConcepts = new IdentityHashMap<>();
894 		int totalCodeCount = 0;
895 		for (TermConcept next : theCodeSystemVersion.getConcepts()) {
896 			totalCodeCount += validateConceptForStorage(next, theCodeSystemVersion, conceptsStack, allConcepts);
897 		}
898 
899 		ourLog.info("Saving version containing {} concepts", totalCodeCount);
900 
901 		TermCodeSystemVersion codeSystemVersion = myCodeSystemVersionDao.saveAndFlush(theCodeSystemVersion);
902 
903 		ourLog.info("Saving code system");
904 
905 		codeSystem.setCurrentVersion(theCodeSystemVersion);
906 		codeSystem = myCodeSystemDao.saveAndFlush(codeSystem);
907 
908 		ourLog.info("Setting codesystemversion on {} concepts...", totalCodeCount);
909 
910 		for (TermConcept next : theCodeSystemVersion.getConcepts()) {
911 			populateVersion(next, codeSystemVersion);
912 		}
913 
914 		ourLog.info("Saving {} concepts...", totalCodeCount);
915 
916 		IdentityHashMap<TermConcept, Object> conceptsStack2 = new IdentityHashMap<TermConcept, Object>();
917 		for (TermConcept next : theCodeSystemVersion.getConcepts()) {
918 			persistChildren(next, codeSystemVersion, conceptsStack2, totalCodeCount);
919 		}
920 
921 		ourLog.info("Done saving concepts, flushing to database");
922 
923 		myConceptDao.flush();
924 		myConceptParentChildLinkDao.flush();
925 
926 		ourLog.info("Done deleting old code system versions");
927 
928 		if (myDeferredConcepts.size() > 0 || myConceptLinksToSaveLater.size() > 0) {
929 			ourLog.info("Note that some concept saving was deferred - still have {} concepts and {} relationships", myDeferredConcepts.size(), myConceptLinksToSaveLater.size());
930 		}
931 	}
932 
933 	@Override
934 	@Transactional(propagation = Propagation.REQUIRED)
935 	public IIdType storeNewCodeSystemVersion(CodeSystem theCodeSystemResource, TermCodeSystemVersion theCodeSystemVersion, RequestDetails theRequestDetails, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
936 		Validate.notBlank(theCodeSystemResource.getUrl(), "theCodeSystemResource must have a URL");
937 
938 		IIdType csId = createOrUpdateCodeSystem(theCodeSystemResource);
939 
940 		ResourceTable resource = (ResourceTable) myCodeSystemResourceDao.readEntity(csId);
941 		Long codeSystemResourcePid = resource.getId();
942 
943 		ourLog.info("CodeSystem resource has ID: {}", csId.getValue());
944 
945 		theCodeSystemVersion.setResource(resource);
946 		storeNewCodeSystemVersion(codeSystemResourcePid, theCodeSystemResource.getUrl(), theCodeSystemResource.getName(), theCodeSystemVersion);
947 
948 		myDeferredConceptMaps.addAll(theConceptMaps);
949 		myDeferredValueSets.addAll(theValueSets);
950 
951 		return csId;
952 	}
953 
954 	@Override
955 	@Transactional
956 	public void storeTermConceptMapAndChildren(ResourceTable theResourceTable, ConceptMap theConceptMap) {
957 		ourLog.info("Storing TermConceptMap {}", theConceptMap.getIdElement().getValue());
958 
959 		ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied");
960 		ValidateUtil.isNotBlankOrThrowUnprocessableEntity(theConceptMap.getUrl(), "ConceptMap has no value for ConceptMap.url");
961 
962 		TermConceptMap termConceptMap = new TermConceptMap();
963 		termConceptMap.setResource(theResourceTable);
964 		termConceptMap.setUrl(theConceptMap.getUrl());
965 
966 		// Get existing entity so it can be deleted.
967 		Optional<TermConceptMap> optionalExistingTermConceptMapById = myConceptMapDao.findTermConceptMapByResourcePid(termConceptMap.getResourcePid());
968 
969 		/*
970 		 * For now we always delete old versions. At some point, it would be nice to allow configuration to keep old versions.
971 		 */
972 
973 		if (optionalExistingTermConceptMapById.isPresent()) {
974 			Long id = optionalExistingTermConceptMapById.get().getId();
975 			ourLog.info("Deleting existing TermConceptMap {} and its children...", id);
976 			myConceptMapGroupElementTargetDao.deleteTermConceptMapGroupElementTargetById(id);
977 			myConceptMapGroupElementDao.deleteTermConceptMapGroupElementById(id);
978 			myConceptMapGroupDao.deleteTermConceptMapGroupById(id);
979 			myConceptMapDao.deleteTermConceptMapById(id);
980 			ourLog.info("Done deleting existing TermConceptMap {} and its children.", id);
981 
982 			ourLog.info("Flushing...");
983 			myConceptMapGroupElementTargetDao.flush();
984 			myConceptMapGroupElementDao.flush();
985 			myConceptMapGroupDao.flush();
986 			myConceptMapDao.flush();
987 			ourLog.info("Done flushing.");
988 		}
989 
990 		/*
991 		 * Do the upload.
992 		 */
993 		String conceptMapUrl = termConceptMap.getUrl();
994 		Optional<TermConceptMap> optionalExistingTermConceptMapByUrl = myConceptMapDao.findTermConceptMapByUrl(conceptMapUrl);
995 		if (!optionalExistingTermConceptMapByUrl.isPresent()) {
996 			try {
997 				String source = theConceptMap.hasSourceUriType() ? theConceptMap.getSourceUriType().getValueAsString() : null;
998 				if (isNotBlank(source)) {
999 					termConceptMap.setSource(source);
1000 				}
1001 				String target = theConceptMap.hasTargetUriType() ? theConceptMap.getTargetUriType().getValueAsString() : null;
1002 				if (isNotBlank(target)) {
1003 					termConceptMap.setTarget(target);
1004 				}
1005 			} catch (FHIRException fe) {
1006 				throw new InternalErrorException(fe);
1007 			}
1008 			myConceptMapDao.save(termConceptMap);
1009 
1010 			if (theConceptMap.hasGroup()) {
1011 				TermConceptMapGroup termConceptMapGroup;
1012 				for (ConceptMap.ConceptMapGroupComponent group : theConceptMap.getGroup()) {
1013 					if (isBlank(group.getSource())) {
1014 						throw new UnprocessableEntityException("ConceptMap[url='" + theConceptMap.getUrl() + "'] contains at least one group without a value in ConceptMap.group.source");
1015 					}
1016 					if (isBlank(group.getTarget())) {
1017 						throw new UnprocessableEntityException("ConceptMap[url='" + theConceptMap.getUrl() + "'] contains at least one group without a value in ConceptMap.group.target");
1018 					}
1019 					termConceptMapGroup = new TermConceptMapGroup();
1020 					termConceptMapGroup.setConceptMap(termConceptMap);
1021 					termConceptMapGroup.setSource(group.getSource());
1022 					termConceptMapGroup.setSourceVersion(group.getSourceVersion());
1023 					termConceptMapGroup.setTarget(group.getTarget());
1024 					termConceptMapGroup.setTargetVersion(group.getTargetVersion());
1025 					myConceptMapGroupDao.save(termConceptMapGroup);
1026 
1027 					if (group.hasElement()) {
1028 						TermConceptMapGroupElement termConceptMapGroupElement;
1029 						for (ConceptMap.SourceElementComponent element : group.getElement()) {
1030 							termConceptMapGroupElement = new TermConceptMapGroupElement();
1031 							termConceptMapGroupElement.setConceptMapGroup(termConceptMapGroup);
1032 							termConceptMapGroupElement.setCode(element.getCode());
1033 							termConceptMapGroupElement.setDisplay(element.getDisplay());
1034 							myConceptMapGroupElementDao.save(termConceptMapGroupElement);
1035 
1036 							if (element.hasTarget()) {
1037 								TermConceptMapGroupElementTarget termConceptMapGroupElementTarget;
1038 								for (ConceptMap.TargetElementComponent target : element.getTarget()) {
1039 									termConceptMapGroupElementTarget = new TermConceptMapGroupElementTarget();
1040 									termConceptMapGroupElementTarget.setConceptMapGroupElement(termConceptMapGroupElement);
1041 									termConceptMapGroupElementTarget.setCode(target.getCode());
1042 									termConceptMapGroupElementTarget.setDisplay(target.getDisplay());
1043 									termConceptMapGroupElementTarget.setEquivalence(target.getEquivalence());
1044 									myConceptMapGroupElementTargetDao.saveAndFlush(termConceptMapGroupElementTarget);
1045 								}
1046 							}
1047 						}
1048 					}
1049 				}
1050 			}
1051 		} else {
1052 			TermConceptMap existingTermConceptMap = optionalExistingTermConceptMapByUrl.get();
1053 
1054 			String msg = myContext.getLocalizer().getMessage(
1055 				BaseHapiTerminologySvcImpl.class,
1056 				"cannotCreateDuplicateConceptMapUrl",
1057 				conceptMapUrl,
1058 				existingTermConceptMap.getResource().getIdDt().toUnqualifiedVersionless().getValue());
1059 
1060 			throw new UnprocessableEntityException(msg);
1061 		}
1062 
1063 		ourLog.info("Done storing TermConceptMap.");
1064 	}
1065 
1066 	@Override
1067 	public boolean supportsSystem(String theSystem) {
1068 		TermCodeSystem cs = getCodeSystem(theSystem);
1069 		return cs != null;
1070 	}
1071 
1072 	private ArrayList<VersionIndependentConcept> toVersionIndependentConcepts(String theSystem, Set<TermConcept> codes) {
1073 		ArrayList<VersionIndependentConcept> retVal = new ArrayList<>(codes.size());
1074 		for (TermConcept next : codes) {
1075 			retVal.add(new VersionIndependentConcept(theSystem, next.getCode()));
1076 		}
1077 		return retVal;
1078 	}
1079 
1080 	@Override
1081 	@Transactional(propagation = Propagation.REQUIRED)
1082 	public List<TermConceptMapGroupElementTarget> translate(TranslationRequest theTranslationRequest) {
1083 		List<TermConceptMapGroupElementTarget> retVal = new ArrayList<>();
1084 
1085 		CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder();
1086 		CriteriaQuery<TermConceptMapGroupElementTarget> query = criteriaBuilder.createQuery(TermConceptMapGroupElementTarget.class);
1087 		Root<TermConceptMapGroupElementTarget> root = query.from(TermConceptMapGroupElementTarget.class);
1088 
1089 		Join<TermConceptMapGroupElementTarget, TermConceptMapGroupElement> elementJoin = root.join("myConceptMapGroupElement");
1090 		Join<TermConceptMapGroupElement, TermConceptMapGroup> groupJoin = elementJoin.join("myConceptMapGroup");
1091 		Join<TermConceptMapGroup, TermConceptMap> conceptMapJoin = groupJoin.join("myConceptMap");
1092 
1093 		List<TranslationQuery> translationQueries = theTranslationRequest.getTranslationQueries();
1094 		List<TermConceptMapGroupElementTarget> cachedTargets;
1095 		ArrayList<Predicate> predicates;
1096 		Coding coding;
1097 		for (TranslationQuery translationQuery : translationQueries) {
1098 			cachedTargets = myTranslationCache.getIfPresent(translationQuery);
1099 			if (cachedTargets == null) {
1100 				final List<TermConceptMapGroupElementTarget> targets = new ArrayList<>();
1101 
1102 				predicates = new ArrayList<>();
1103 
1104 				coding = translationQuery.getCoding();
1105 				if (coding.hasCode()) {
1106 					predicates.add(criteriaBuilder.equal(elementJoin.get("myCode"), coding.getCode()));
1107 				} else {
1108 					throw new InvalidRequestException("A code must be provided for translation to occur.");
1109 				}
1110 
1111 				if (coding.hasSystem()) {
1112 					predicates.add(criteriaBuilder.equal(groupJoin.get("mySource"), coding.getSystem()));
1113 				}
1114 
1115 				if (coding.hasVersion()) {
1116 					predicates.add(criteriaBuilder.equal(groupJoin.get("mySourceVersion"), coding.getVersion()));
1117 				}
1118 
1119 				if (translationQuery.hasTargetSystem()) {
1120 					predicates.add(criteriaBuilder.equal(groupJoin.get("myTarget"), translationQuery.getTargetSystem().getValueAsString()));
1121 				}
1122 
1123 				if (translationQuery.hasSource()) {
1124 					predicates.add(criteriaBuilder.equal(conceptMapJoin.get("mySource"), translationQuery.getSource().getValueAsString()));
1125 				}
1126 
1127 				if (translationQuery.hasTarget()) {
1128 					predicates.add(criteriaBuilder.equal(conceptMapJoin.get("myTarget"), translationQuery.getTarget().getValueAsString()));
1129 				}
1130 
1131 				if (translationQuery.hasResourceId()) {
1132 					predicates.add(criteriaBuilder.equal(conceptMapJoin.get("myResourcePid"), translationQuery.getResourceId()));
1133 				}
1134 
1135 				Predicate outerPredicate = criteriaBuilder.and(predicates.toArray(new Predicate[0]));
1136 				query.where(outerPredicate);
1137 
1138 				// Use scrollable results.
1139 				final TypedQuery<TermConceptMapGroupElementTarget> typedQuery = myEntityManager.createQuery(query.select(root));
1140 				org.hibernate.query.Query<TermConceptMapGroupElementTarget> hibernateQuery = (org.hibernate.query.Query<TermConceptMapGroupElementTarget>) typedQuery;
1141 				hibernateQuery.setFetchSize(myFetchSize);
1142 				ScrollableResults scrollableResults = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
1143 				Iterator<TermConceptMapGroupElementTarget> scrollableResultsIterator = new ScrollableResultsIterator<>(scrollableResults);
1144 
1145 				while (scrollableResultsIterator.hasNext()) {
1146 					targets.add(scrollableResultsIterator.next());
1147 				}
1148 
1149 				ourLastResultsFromTranslationCache = false; // For testing.
1150 				myTranslationCache.get(translationQuery, k -> targets);
1151 				retVal.addAll(targets);
1152 			} else {
1153 				ourLastResultsFromTranslationCache = true; // For testing.
1154 				retVal.addAll(cachedTargets);
1155 			}
1156 		}
1157 
1158 		return retVal;
1159 	}
1160 
1161 	@Override
1162 	@Transactional(propagation = Propagation.REQUIRED)
1163 	public List<TermConceptMapGroupElement> translateWithReverse(TranslationRequest theTranslationRequest) {
1164 		List<TermConceptMapGroupElement> retVal = new ArrayList<>();
1165 
1166 		CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder();
1167 		CriteriaQuery<TermConceptMapGroupElement> query = criteriaBuilder.createQuery(TermConceptMapGroupElement.class);
1168 		Root<TermConceptMapGroupElement> root = query.from(TermConceptMapGroupElement.class);
1169 
1170 		Join<TermConceptMapGroupElement, TermConceptMapGroupElementTarget> targetJoin = root.join("myConceptMapGroupElementTargets");
1171 		Join<TermConceptMapGroupElement, TermConceptMapGroup> groupJoin = root.join("myConceptMapGroup");
1172 		Join<TermConceptMapGroup, TermConceptMap> conceptMapJoin = groupJoin.join("myConceptMap");
1173 
1174 		List<TranslationQuery> translationQueries = theTranslationRequest.getTranslationQueries();
1175 		List<TermConceptMapGroupElement> cachedElements;
1176 		ArrayList<Predicate> predicates;
1177 		Coding coding;
1178 		for (TranslationQuery translationQuery : translationQueries) {
1179 			cachedElements = myTranslationWithReverseCache.getIfPresent(translationQuery);
1180 			if (cachedElements == null) {
1181 				final List<TermConceptMapGroupElement> elements = new ArrayList<>();
1182 
1183 				predicates = new ArrayList<>();
1184 
1185 				coding = translationQuery.getCoding();
1186 				if (coding.hasCode()) {
1187 					predicates.add(criteriaBuilder.equal(targetJoin.get("myCode"), coding.getCode()));
1188 				} else {
1189 					throw new InvalidRequestException("A code must be provided for translation to occur.");
1190 				}
1191 
1192 				if (coding.hasSystem()) {
1193 					predicates.add(criteriaBuilder.equal(groupJoin.get("myTarget"), coding.getSystem()));
1194 				}
1195 
1196 				if (coding.hasVersion()) {
1197 					predicates.add(criteriaBuilder.equal(groupJoin.get("myTargetVersion"), coding.getVersion()));
1198 				}
1199 
1200 				if (translationQuery.hasTargetSystem()) {
1201 					predicates.add(criteriaBuilder.equal(groupJoin.get("mySource"), translationQuery.getTargetSystem().getValueAsString()));
1202 				}
1203 
1204 				if (translationQuery.hasSource()) {
1205 					predicates.add(criteriaBuilder.equal(conceptMapJoin.get("myTarget"), translationQuery.getSource().getValueAsString()));
1206 				}
1207 
1208 				if (translationQuery.hasTarget()) {
1209 					predicates.add(criteriaBuilder.equal(conceptMapJoin.get("mySource"), translationQuery.getTarget().getValueAsString()));
1210 				}
1211 
1212 				if (translationQuery.hasResourceId()) {
1213 					predicates.add(criteriaBuilder.equal(conceptMapJoin.get("myResourcePid"), translationQuery.getResourceId()));
1214 				}
1215 
1216 				Predicate outerPredicate = criteriaBuilder.and(predicates.toArray(new Predicate[0]));
1217 				query.where(outerPredicate);
1218 
1219 				// Use scrollable results.
1220 				final TypedQuery<TermConceptMapGroupElement> typedQuery = myEntityManager.createQuery(query.select(root));
1221 				org.hibernate.query.Query<TermConceptMapGroupElement> hibernateQuery = (org.hibernate.query.Query<TermConceptMapGroupElement>) typedQuery;
1222 				hibernateQuery.setFetchSize(myFetchSize);
1223 				ScrollableResults scrollableResults = hibernateQuery.scroll(ScrollMode.FORWARD_ONLY);
1224 				Iterator<TermConceptMapGroupElement> scrollableResultsIterator = new ScrollableResultsIterator<>(scrollableResults);
1225 
1226 				while (scrollableResultsIterator.hasNext()) {
1227 					elements.add(scrollableResultsIterator.next());
1228 				}
1229 
1230 				ourLastResultsFromTranslationWithReverseCache = false; // For testing.
1231 				myTranslationWithReverseCache.get(translationQuery, k -> elements);
1232 				retVal.addAll(elements);
1233 			} else {
1234 				ourLastResultsFromTranslationWithReverseCache = true; // For testing.
1235 				retVal.addAll(cachedElements);
1236 			}
1237 		}
1238 
1239 		return retVal;
1240 	}
1241 
1242 	private int validateConceptForStorage(TermConcept theConcept, TermCodeSystemVersion theCodeSystem, ArrayList<String> theConceptsStack,
1243 													  IdentityHashMap<TermConcept, Object> theAllConcepts) {
1244 		ValidateUtil.isTrueOrThrowInvalidRequest(theConcept.getCodeSystemVersion() != null, "CodesystemValue is null");
1245 		ValidateUtil.isTrueOrThrowInvalidRequest(theConcept.getCodeSystemVersion() == theCodeSystem, "CodeSystems are not equal");
1246 		ValidateUtil.isNotBlankOrThrowInvalidRequest(theConcept.getCode(), "Codesystem contains a code with no code value");
1247 
1248 		if (theConceptsStack.contains(theConcept.getCode())) {
1249 			throw new InvalidRequestException("CodeSystem contains circular reference around code " + theConcept.getCode());
1250 		}
1251 		theConceptsStack.add(theConcept.getCode());
1252 
1253 		int retVal = 0;
1254 		if (theAllConcepts.put(theConcept, theAllConcepts) == null) {
1255 			if (theAllConcepts.size() % 1000 == 0) {
1256 				ourLog.info("Have validated {} concepts", theAllConcepts.size());
1257 			}
1258 			retVal = 1;
1259 		}
1260 
1261 		for (TermConceptParentChildLink next : theConcept.getChildren()) {
1262 			next.setCodeSystem(theCodeSystem);
1263 			retVal += validateConceptForStorage(next.getChild(), theCodeSystem, theConceptsStack, theAllConcepts);
1264 		}
1265 
1266 		theConceptsStack.remove(theConceptsStack.size() - 1);
1267 
1268 		return retVal;
1269 	}
1270 
1271 	private void verifyNoDuplicates(Collection<TermConcept> theConcepts, Set<String> theCodes) {
1272 		for (TermConcept next : theConcepts) {
1273 			if (!theCodes.add(next.getCode())) {
1274 				throw new InvalidRequestException("Duplicate code " + next.getCode() + " found in codesystem after checking " + theCodes.size() + " codes");
1275 			}
1276 			verifyNoDuplicates(next.getChildren().stream().map(TermConceptParentChildLink::getChild).collect(Collectors.toList()), theCodes);
1277 		}
1278 	}
1279 
1280 	/**
1281 	 * This method is present only for unit tests, do not call from client code
1282 	 */
1283 	@VisibleForTesting
1284 	public static void clearOurLastResultsFromTranslationCache() {
1285 		ourLastResultsFromTranslationCache = false;
1286 	}
1287 
1288 	/**
1289 	 * This method is present only for unit tests, do not call from client code
1290 	 */
1291 	@VisibleForTesting
1292 	public static void clearOurLastResultsFromTranslationWithReverseCache() {
1293 		ourLastResultsFromTranslationWithReverseCache = false;
1294 	}
1295 
1296 	/**
1297 	 * This method is present only for unit tests, do not call from client code
1298 	 */
1299 	@VisibleForTesting
1300 	static boolean isOurLastResultsFromTranslationCache() {
1301 		return ourLastResultsFromTranslationCache;
1302 	}
1303 
1304 	/**
1305 	 * This method is present only for unit tests, do not call from client code
1306 	 */
1307 	@VisibleForTesting
1308 	static boolean isOurLastResultsFromTranslationWithReverseCache() {
1309 		return ourLastResultsFromTranslationWithReverseCache;
1310 	}
1311 
1312 	/**
1313 	 * This method is present only for unit tests, do not call from client code
1314 	 */
1315 	@VisibleForTesting
1316 	public static void setForceSaveDeferredAlwaysForUnitTest(boolean theForceSaveDeferredAlwaysForUnitTest) {
1317 		ourForceSaveDeferredAlwaysForUnitTest = theForceSaveDeferredAlwaysForUnitTest;
1318 	}
1319 }