Source code

001package ca.uhn.fhir.jpa.term;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2021 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 * http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.context.FhirContext;
024import ca.uhn.fhir.context.support.ConceptValidationOptions;
025import ca.uhn.fhir.context.support.IValidationSupport;
026import ca.uhn.fhir.context.support.ValidationSupportContext;
027import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
028import ca.uhn.fhir.jpa.api.config.DaoConfig;
029import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
030import ca.uhn.fhir.jpa.api.dao.IDao;
031import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
032import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem;
033import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
034import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
035import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
036import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
037import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
038import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
039import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
040import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
041import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao;
042import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewDao;
043import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewOracleDao;
044import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
045import ca.uhn.fhir.jpa.entity.ITermValueSetConceptView;
046import ca.uhn.fhir.jpa.entity.TermCodeSystem;
047import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
048import ca.uhn.fhir.jpa.entity.TermConcept;
049import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
050import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
051import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
052import ca.uhn.fhir.jpa.entity.TermConceptProperty;
053import ca.uhn.fhir.jpa.entity.TermConceptPropertyBinder;
054import ca.uhn.fhir.jpa.entity.TermConceptPropertyTypeEnum;
055import ca.uhn.fhir.jpa.entity.TermValueSet;
056import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
057import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
058import ca.uhn.fhir.jpa.model.entity.ForcedId;
059import ca.uhn.fhir.jpa.model.entity.ResourceTable;
060import ca.uhn.fhir.jpa.model.sched.HapiJob;
061import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
062import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
063import ca.uhn.fhir.jpa.model.util.JpaConstants;
064import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
065import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
066import ca.uhn.fhir.jpa.term.api.ITermConceptMappingSvc;
067import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
068import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
069import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException;
070import ca.uhn.fhir.jpa.util.LogicUtil;
071import ca.uhn.fhir.rest.api.Constants;
072import ca.uhn.fhir.rest.api.server.RequestDetails;
073import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
074import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
075import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
076import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
077import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
078import ca.uhn.fhir.util.CoverageIgnore;
079import ca.uhn.fhir.util.FhirVersionIndependentConcept;
080import ca.uhn.fhir.util.HapiExtensions;
081import ca.uhn.fhir.util.StopWatch;
082import ca.uhn.fhir.util.UrlUtil;
083import ca.uhn.fhir.util.ValidateUtil;
084import com.github.benmanes.caffeine.cache.Cache;
085import com.github.benmanes.caffeine.cache.Caffeine;
086import com.google.common.annotations.VisibleForTesting;
087import com.google.common.base.Stopwatch;
088import com.google.common.collect.ArrayListMultimap;
089import org.apache.commons.lang3.ObjectUtils;
090import org.apache.commons.lang3.StringUtils;
091import org.apache.commons.lang3.Validate;
092import org.apache.commons.lang3.time.DateUtils;
093import org.apache.lucene.index.Term;
094import org.apache.lucene.search.RegexpQuery;
095import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension;
096import org.hibernate.search.backend.lucene.LuceneExtension;
097import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep;
098import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
099import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
100import org.hibernate.search.engine.search.query.SearchQuery;
101import org.hibernate.search.mapper.orm.Search;
102import org.hibernate.search.mapper.orm.session.SearchSession;
103import org.hl7.fhir.common.hapi.validation.support.CommonCodeSystemsTerminologyService;
104import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport;
105import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50;
106import org.hl7.fhir.convertors.context.ConversionContext40_50;
107import org.hl7.fhir.convertors.conv40_50.VersionConvertor_40_50;
108import org.hl7.fhir.convertors.conv40_50.resources40_50.ValueSet40_50;
109import org.hl7.fhir.instance.model.api.IAnyResource;
110import org.hl7.fhir.instance.model.api.IBaseCoding;
111import org.hl7.fhir.instance.model.api.IBaseDatatype;
112import org.hl7.fhir.instance.model.api.IBaseResource;
113import org.hl7.fhir.instance.model.api.IIdType;
114import org.hl7.fhir.instance.model.api.IPrimitiveType;
115import org.hl7.fhir.r4.model.BooleanType;
116import org.hl7.fhir.r4.model.CanonicalType;
117import org.hl7.fhir.r4.model.CodeSystem;
118import org.hl7.fhir.r4.model.CodeableConcept;
119import org.hl7.fhir.r4.model.Coding;
120import org.hl7.fhir.r4.model.DomainResource;
121import org.hl7.fhir.r4.model.Enumerations;
122import org.hl7.fhir.r4.model.Extension;
123import org.hl7.fhir.r4.model.InstantType;
124import org.hl7.fhir.r4.model.IntegerType;
125import org.hl7.fhir.r4.model.StringType;
126import org.hl7.fhir.r4.model.ValueSet;
127import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome;
128import org.quartz.JobExecutionContext;
129import org.springframework.beans.factory.annotation.Autowired;
130import org.springframework.context.ApplicationContext;
131import org.springframework.data.domain.PageRequest;
132import org.springframework.data.domain.Pageable;
133import org.springframework.data.domain.Slice;
134import org.springframework.transaction.PlatformTransactionManager;
135import org.springframework.transaction.TransactionDefinition;
136import org.springframework.transaction.annotation.Propagation;
137import org.springframework.transaction.annotation.Transactional;
138import org.springframework.transaction.interceptor.NoRollbackRuleAttribute;
139import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute;
140import org.springframework.transaction.support.TransactionSynchronizationManager;
141import org.springframework.transaction.support.TransactionTemplate;
142import org.springframework.util.comparator.Comparators;
143
144import javax.annotation.Nonnull;
145import javax.annotation.Nullable;
146import javax.annotation.PostConstruct;
147import javax.persistence.EntityManager;
148import javax.persistence.NonUniqueResultException;
149import javax.persistence.PersistenceContext;
150import javax.persistence.PersistenceContextType;
151import javax.persistence.TypedQuery;
152import javax.persistence.criteria.CriteriaBuilder;
153import javax.persistence.criteria.CriteriaQuery;
154import javax.persistence.criteria.Fetch;
155import javax.persistence.criteria.Join;
156import javax.persistence.criteria.JoinType;
157import javax.persistence.criteria.Predicate;
158import javax.persistence.criteria.Root;
159import javax.validation.constraints.NotNull;
160import java.util.ArrayList;
161import java.util.Collection;
162import java.util.Collections;
163import java.util.Date;
164import java.util.HashMap;
165import java.util.HashSet;
166import java.util.LinkedHashMap;
167import java.util.List;
168import java.util.Locale;
169import java.util.Map;
170import java.util.Objects;
171import java.util.Optional;
172import java.util.Set;
173import java.util.StringTokenizer;
174import java.util.UUID;
175import java.util.concurrent.TimeUnit;
176import java.util.concurrent.atomic.AtomicInteger;
177import java.util.function.Consumer;
178import java.util.function.Supplier;
179import java.util.stream.Collectors;
180
181import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI;
182import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
183import static org.apache.commons.lang3.StringUtils.defaultString;
184import static org.apache.commons.lang3.StringUtils.isBlank;
185import static org.apache.commons.lang3.StringUtils.isEmpty;
186import static org.apache.commons.lang3.StringUtils.isNoneBlank;
187import static org.apache.commons.lang3.StringUtils.isNotBlank;
188import static org.apache.commons.lang3.StringUtils.lowerCase;
189import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase;
190import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_LOW;
191
192public abstract class BaseTermReadSvcImpl implements ITermReadSvc {
193        public static final int DEFAULT_FETCH_SIZE = 250;
194        private static final int SINGLE_FETCH_SIZE = 1;
195        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseTermReadSvcImpl.class);
196        private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions();
197        private static final TermCodeSystemVersion NO_CURRENT_VERSION = new TermCodeSystemVersion().setId(-1L);
198        private static Runnable myInvokeOnNextCallForUnitTest;
199        private final Cache<String, TermCodeSystemVersion> myCodeSystemCurrentVersionCache = Caffeine.newBuilder().expireAfterWrite(1, TimeUnit.MINUTES).build();
200        @Autowired
201        protected DaoRegistry myDaoRegistry;
202        @Autowired
203        protected ITermCodeSystemDao myCodeSystemDao;
204        @Autowired
205        protected ITermConceptDao myConceptDao;
206        @Autowired
207        protected ITermConceptPropertyDao myConceptPropertyDao;
208        @Autowired
209        protected ITermConceptDesignationDao myConceptDesignationDao;
210        @Autowired
211        protected ITermValueSetDao myTermValueSetDao;
212        @Autowired
213        protected ITermValueSetConceptDao myValueSetConceptDao;
214        @Autowired
215        protected ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao;
216        @Autowired
217        protected FhirContext myContext;
218        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
219        protected EntityManager myEntityManager;
220        @Autowired
221        private ITermCodeSystemVersionDao myCodeSystemVersionDao;
222        @Autowired
223        private DaoConfig myDaoConfig;
224        private TransactionTemplate myTxTemplate;
225        @Autowired
226        private PlatformTransactionManager myTransactionManager;
227        @Autowired(required = false)
228        private IFulltextSearchSvc myFulltextSearchSvc;
229        @Autowired
230        private PlatformTransactionManager myTxManager;
231        @Autowired
232        private ITermValueSetConceptViewDao myTermValueSetConceptViewDao;
233        @Autowired
234        private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao;
235        @Autowired
236        private ISchedulerService mySchedulerService;
237        @Autowired(required = false)
238        private ITermDeferredStorageSvc myDeferredStorageSvc;
239        @Autowired(required = false)
240        private ITermCodeSystemStorageSvc myConceptStorageSvc;
241        @Autowired
242        private ApplicationContext myApplicationContext;
243        @Autowired
244        private ITermConceptMappingSvc myTermConceptMappingSvc;
245
246        private volatile IValidationSupport myJpaValidationSupport;
247        private volatile IValidationSupport myValidationSupport;
248
249        //We need this bean so we can tell which mode hibernate search is running in.
250        @Autowired
251        private HibernatePropertiesProvider myHibernatePropertiesProvider;
252
253        private boolean isFullTextSetToUseElastic() {
254                return "elasticsearch".equalsIgnoreCase(myHibernatePropertiesProvider.getHibernateSearchBackend());
255        }
256
257        @Override
258        public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) {
259                TermCodeSystemVersion cs = getCurrentCodeSystemVersion(theSystem);
260                return cs != null;
261        }
262
263        private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, TermConcept theConcept, boolean theAdd, String theValueSetIncludeVersion) {
264                String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri();
265                String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId();
266                String code = theConcept.getCode();
267                String display = theConcept.getDisplay();
268                Long sourceConceptPid = theConcept.getId();
269
270                String directParentPids = theConcept
271                        .getParents()
272                        .stream()
273                        .map(t -> t.getParent().getId().toString())
274                        .collect(Collectors.joining(" "));
275
276                Collection<TermConceptDesignation> designations = theConcept.getDesignations();
277                if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) {
278                        return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem + "|" + theValueSetIncludeVersion, code, display, sourceConceptPid, directParentPids, codeSystemVersion);
279                } else {
280                        return addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, designations, theAdd, codeSystem, code, display, sourceConceptPid, directParentPids, codeSystemVersion);
281                }
282        }
283
284        private void addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theCodeSystem, String theCodeSystemVersion, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids) {
285                if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
286                        if (isNoneBlank(theCodeSystem, theCode)) {
287                                if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
288                                        theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem + "|" + theCodeSystemVersion, theCode, theDisplay, null, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
289                                }
290
291                                if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
292                                        theValueSetCodeAccumulator.excludeConcept(theCodeSystem + "|" + theCodeSystemVersion, theCode);
293                                }
294                        }
295                } else {
296                        if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
297                                theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, null, theSourceConceptPid, theSourceConceptDirectParentPids, theCodeSystemVersion);
298                        }
299
300                        if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
301                                theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
302                        }
303                }
304        }
305
306        private boolean addCodeIfNotAlreadyAdded(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, Collection<TermConceptDesignation> theDesignations, boolean theAdd, String theCodeSystem, String theCode, String theDisplay, Long theSourceConceptPid, String theSourceConceptDirectParentPids, String theSystemVersion) {
307                if (isNoneBlank(theCodeSystem, theCode)) {
308                        if (theAdd && theAddedCodes.add(theCodeSystem + "|" + theCode)) {
309                                theValueSetCodeAccumulator.includeConceptWithDesignations(theCodeSystem, theCode, theDisplay, theDesignations, theSourceConceptPid, theSourceConceptDirectParentPids, theSystemVersion);
310                                return true;
311                        }
312
313                        if (!theAdd && theAddedCodes.remove(theCodeSystem + "|" + theCode)) {
314                                theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
315                                return true;
316                        }
317                }
318
319                return false;
320        }
321
322        private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) {
323                boolean retVal = theSetToPopulate.add(theConcept);
324                if (retVal) {
325                        if (theSetToPopulate.size() >= myDaoConfig.getMaximumExpansionSize()) {
326                                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "expansionTooLarge", myDaoConfig.getMaximumExpansionSize());
327                                throw new ExpansionTooCostlyException(msg);
328                        }
329                }
330                return retVal;
331        }
332
333        /**
334         * This method is present only for unit tests, do not call from client code
335         */
336        @VisibleForTesting
337        public void clearCaches() {
338                myCodeSystemCurrentVersionCache.invalidateAll();
339        }
340
341
342        public void deleteValueSetForResource(ResourceTable theResourceTable) {
343                // Get existing entity so it can be deleted.
344                Optional<TermValueSet> optionalExistingTermValueSetById = myTermValueSetDao.findByResourcePid(theResourceTable.getId());
345
346                if (optionalExistingTermValueSetById.isPresent()) {
347                        TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get();
348
349                        ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId());
350                        deletePreCalculatedValueSetContents(existingTermValueSet);
351                        myTermValueSetDao.deleteById(existingTermValueSet.getId());
352                        ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId());
353                }
354        }
355
356        private void deletePreCalculatedValueSetContents(TermValueSet theValueSet) {
357                myValueSetConceptDesignationDao.deleteByTermValueSetId(theValueSet.getId());
358                myValueSetConceptDao.deleteByTermValueSetId(theValueSet.getId());
359        }
360
361        @Override
362        @Transactional
363        public void deleteValueSetAndChildren(ResourceTable theResourceTable) {
364                deleteValueSetForResource(theResourceTable);
365        }
366
367
368        @Override
369        @Transactional
370        public List<FhirVersionIndependentConcept> expandValueSetIntoConceptList(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) {
371                // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
372
373                ValueSet expanded = expandValueSet(theExpansionOptions, theValueSetCanonicalUrl);
374
375                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
376                for (ValueSet.ValueSetExpansionContainsComponent nextContains : expanded.getExpansion().getContains()) {
377                        retVal.add(new FhirVersionIndependentConcept(nextContains.getSystem(), nextContains.getCode(), nextContains.getDisplay(), nextContains.getVersion()));
378                }
379                return retVal;
380        }
381
382        @Override
383        @Transactional
384        public ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) {
385                ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSetCanonicalUrl);
386                if (valueSet == null) {
387                        throw new ResourceNotFoundException("Unknown ValueSet: " + UrlUtil.escapeUrlParam(theValueSetCanonicalUrl));
388                }
389
390                return expandValueSet(theExpansionOptions, valueSet);
391        }
392
393        @Override
394        @Transactional(propagation = Propagation.REQUIRED)
395        public ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull ValueSet theValueSetToExpand) {
396                String filter = null;
397                if (theExpansionOptions != null) {
398                        filter = theExpansionOptions.getFilter();
399                }
400                return expandValueSet(theExpansionOptions, theValueSetToExpand, ExpansionFilter.fromFilterString(filter));
401        }
402
403        private ValueSet expandValueSet(@Nullable ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, ExpansionFilter theFilter) {
404                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
405
406                ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
407                int offset = expansionOptions.getOffset();
408                int count = expansionOptions.getCount();
409
410                ValueSetExpansionComponentWithConceptAccumulator accumulator = new ValueSetExpansionComponentWithConceptAccumulator(myContext, count, expansionOptions.isIncludeHierarchy());
411                accumulator.setHardExpansionMaximumSize(myDaoConfig.getMaximumExpansionSize());
412                accumulator.setSkipCountRemaining(offset);
413                accumulator.setIdentifier(UUID.randomUUID().toString());
414                accumulator.setTimestamp(new Date());
415                accumulator.setOffset(offset);
416
417                if (theExpansionOptions != null && isHibernateSearchEnabled()) {
418                        accumulator.addParameter().setName("offset").setValue(new IntegerType(offset));
419                        accumulator.addParameter().setName("count").setValue(new IntegerType(count));
420                }
421
422
423                expandValueSetIntoAccumulator(theValueSetToExpand, theExpansionOptions, accumulator, theFilter, true);
424
425                if (accumulator.getTotalConcepts() != null) {
426                        accumulator.setTotal(accumulator.getTotalConcepts());
427                }
428
429                ValueSet valueSet = new ValueSet();
430                valueSet.setId(theValueSetToExpand.getId());
431                valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
432                valueSet.setCompose(theValueSetToExpand.getCompose());
433                valueSet.setExpansion(accumulator);
434
435                for (String next : accumulator.getMessages()) {
436                        valueSet.getMeta().addExtension()
437                                .setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
438                                .setValue(new StringType(next));
439                }
440
441                if (expansionOptions.isIncludeHierarchy()) {
442                        accumulator.applyHierarchy();
443                }
444
445                return valueSet;
446        }
447
448        private void expandValueSetIntoAccumulator(ValueSet theValueSetToExpand, ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theAccumulator, ExpansionFilter theFilter, boolean theAdd) {
449                Optional<TermValueSet> optionalTermValueSet;
450                if (theValueSetToExpand.hasUrl()) {
451                        if (theValueSetToExpand.hasVersion()) {
452                                optionalTermValueSet = myTermValueSetDao.findTermValueSetByUrlAndVersion(theValueSetToExpand.getUrl(), theValueSetToExpand.getVersion());
453                        } else {
454                                optionalTermValueSet = findCurrentTermValueSet(theValueSetToExpand.getUrl());
455                        }
456                } else {
457                        optionalTermValueSet = Optional.empty();
458                }
459
460                /*
461                 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion
462                 */
463                if (!optionalTermValueSet.isPresent()) {
464                        ourLog.debug("ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}", getValueSetInfo(theValueSetToExpand));
465                        String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetExpandedUsingInMemoryExpansion", getValueSetInfo(theValueSetToExpand));
466                        theAccumulator.addMessage(msg);
467                        expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
468                        return;
469                }
470
471                /*
472                 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion
473                 */
474                TermValueSet termValueSet = optionalTermValueSet.get();
475                if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
476                        String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded", getValueSetInfo(theValueSetToExpand), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
477                        theAccumulator.addMessage(msg);
478                        expandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter);
479                        return;
480                }
481
482                /*
483                 * ValueSet is pre-expanded in database so let's use that
484                 */
485                String expansionTimestamp = toHumanReadableExpansionTimestamp(termValueSet);
486                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetExpandedUsingPreExpansion", expansionTimestamp);
487                theAccumulator.addMessage(msg);
488                expandConcepts(theAccumulator, termValueSet, theFilter, theAdd, isOracleDialect());
489        }
490
491        @Nonnull
492        private String toHumanReadableExpansionTimestamp(TermValueSet termValueSet) {
493                String expansionTimestamp = "(unknown)";
494                if (termValueSet.getExpansionTimestamp() != null) {
495                        String timeElapsed = StopWatch.formatMillis(System.currentTimeMillis() - termValueSet.getExpansionTimestamp().getTime());
496                        expansionTimestamp = new InstantType(termValueSet.getExpansionTimestamp()).getValueAsString() + " (" + timeElapsed + " ago)";
497                }
498                return expansionTimestamp;
499        }
500
501        private boolean isOracleDialect() {
502                return myHibernatePropertiesProvider.getDialect() instanceof org.hibernate.dialect.Oracle12cDialect;
503        }
504
505        private void expandConcepts(IValueSetConceptAccumulator theAccumulator, TermValueSet theTermValueSet, ExpansionFilter theFilter, boolean theAdd, boolean theOracle) {
506                // NOTE: if you modifiy the logic here, look to `expandConceptsOracle` and see if your new code applies to its copy pasted sibling
507                Integer offset = theAccumulator.getSkipCountRemaining();
508                offset = ObjectUtils.defaultIfNull(offset, 0);
509                offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue());
510
511                Integer count = theAccumulator.getCapacityRemaining();
512                count = defaultIfNull(count, myDaoConfig.getMaximumExpansionSize());
513
514                int conceptsExpanded = 0;
515                int designationsExpanded = 0;
516                int toIndex = offset + count;
517
518                Collection<? extends ITermValueSetConceptView> conceptViews;
519                boolean wasFilteredResult = false;
520                String filterDisplayValue = null;
521                if (!theFilter.getFilters().isEmpty() && JpaConstants.VALUESET_FILTER_DISPLAY.equals(theFilter.getFilters().get(0).getProperty()) && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) {
522                        filterDisplayValue = lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]"));
523                        String displayValue = "%" + lowerCase(filterDisplayValue) + "%";
524                        if (theOracle) {
525                                conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
526                        } else {
527                                conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
528                        }
529                        wasFilteredResult = true;
530                } else {
531                        // TODO JA HS: I'm pretty sure we are overfetching here.  test says offset 3, count 4, but we are fetching index 3 -> 10 here, grabbing 7 concepts.
532                        //Specifically this test testExpandInline_IncludePreExpandedValueSetByUri_FilterOnDisplay_LeftMatch_SelectRange
533                        if (theOracle) {
534                                conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
535                        } else {
536                                conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
537                        }
538                        theAccumulator.consumeSkipCount(offset);
539                        if (theAdd) {
540                                theAccumulator.incrementOrDecrementTotalConcepts(true, theTermValueSet.getTotalConcepts().intValue());
541                        }
542                }
543
544                if (conceptViews.isEmpty()) {
545                        logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded);
546                        return;
547                }
548
549                Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>();
550                ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create();
551                Map<Long, Long> pidToSourcePid = new HashMap<>();
552                Map<Long, String> pidToSourceDirectParentPids = new HashMap<>();
553
554                for (ITermValueSetConceptView conceptView : conceptViews) {
555
556                        String system = conceptView.getConceptSystemUrl();
557                        String code = conceptView.getConceptCode();
558                        String display = conceptView.getConceptDisplay();
559                        String systemVersion = conceptView.getConceptSystemVersion();
560
561                        //-- this is quick solution, may need to revisit
562                        if (!applyFilter(display, filterDisplayValue))
563                                continue;
564
565                        Long conceptPid = conceptView.getConceptPid();
566                        if (!pidToConcept.containsKey(conceptPid)) {
567                                FhirVersionIndependentConcept concept = new FhirVersionIndependentConcept(system, code, display, systemVersion);
568                                pidToConcept.put(conceptPid, concept);
569                        }
570
571                        // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations optional.
572                        if (conceptView.getDesignationPid() != null) {
573                                TermConceptDesignation designation = new TermConceptDesignation();
574                                designation.setUseSystem(conceptView.getDesignationUseSystem());
575                                designation.setUseCode(conceptView.getDesignationUseCode());
576                                designation.setUseDisplay(conceptView.getDesignationUseDisplay());
577                                designation.setValue(conceptView.getDesignationVal());
578                                designation.setLanguage(conceptView.getDesignationLang());
579                                pidToDesignations.put(conceptPid, designation);
580
581                                if (++designationsExpanded % 250 == 0) {
582                                        logDesignationsExpanded("Expansion of designations in progress. ", theTermValueSet, designationsExpanded);
583                                }
584                        }
585
586                        if (theAccumulator.isTrackingHierarchy()) {
587                                pidToSourcePid.put(conceptPid, conceptView.getSourceConceptPid());
588                                pidToSourceDirectParentPids.put(conceptPid, conceptView.getSourceConceptDirectParentPids());
589                        }
590
591                        if (++conceptsExpanded % 250 == 0) {
592                                logConceptsExpanded("Expansion of concepts in progress. ", theTermValueSet, conceptsExpanded);
593                        }
594                }
595
596                for (Long nextPid : pidToConcept.keySet()) {
597                        FhirVersionIndependentConcept concept = pidToConcept.get(nextPid);
598                        List<TermConceptDesignation> designations = pidToDesignations.get(nextPid);
599                        String system = concept.getSystem();
600                        String code = concept.getCode();
601                        String display = concept.getDisplay();
602                        String systemVersion = concept.getSystemVersion();
603
604                        if (theAdd) {
605                                if (theAccumulator.getCapacityRemaining() != null) {
606                                        if (theAccumulator.getCapacityRemaining() == 0) {
607                                                break;
608                                        }
609                                }
610
611                                Long sourceConceptPid = pidToSourcePid.get(nextPid);
612                                String sourceConceptDirectParentPids = pidToSourceDirectParentPids.get(nextPid);
613                                theAccumulator.includeConceptWithDesignations(system, code, display, designations, sourceConceptPid, sourceConceptDirectParentPids, systemVersion);
614                        } else {
615                                boolean removed = theAccumulator.excludeConcept(system, code);
616                                if (removed) {
617                                        theAccumulator.incrementOrDecrementTotalConcepts(false, 1);
618                                }
619                        }
620                }
621
622                if (wasFilteredResult && theAdd) {
623                        theAccumulator.incrementOrDecrementTotalConcepts(true, pidToConcept.size());
624                }
625
626                logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded);
627                logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded);
628        }
629
630        private void logConceptsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) {
631                if (theConceptsExpanded > 0) {
632                        ourLog.debug("{}Have expanded {} concepts in ValueSet[{}]", theLogDescriptionPrefix, theConceptsExpanded, theTermValueSet.getUrl());
633                }
634        }
635
636        private void logDesignationsExpanded(String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) {
637                if (theDesignationsExpanded > 0) {
638                        ourLog.debug("{}Have expanded {} designations in ValueSet[{}]", theLogDescriptionPrefix, theDesignationsExpanded, theTermValueSet.getUrl());
639                }
640        }
641
642        public boolean applyFilter(final String theDisplay, final String theFilterDisplay) {
643
644                //-- safety check only, no need to apply filter
645                if (theDisplay == null || theFilterDisplay == null)
646                        return true;
647
648                // -- sentence case
649                if (startsWithIgnoreCase(theDisplay, theFilterDisplay))
650                        return true;
651
652                //-- token case
653                return startsWithByWordBoundaries(theDisplay, theFilterDisplay);
654        }
655
656        private boolean startsWithByWordBoundaries(String theDisplay, String theFilterDisplay) {
657                // return true only e.g. the input is 'Body height', theFilterDisplay is "he", or 'bo'
658                StringTokenizer tok = new StringTokenizer(theDisplay);
659                List<String> tokens = new ArrayList<>();
660                while (tok.hasMoreTokens()) {
661                        String token = tok.nextToken();
662                        if (startsWithIgnoreCase(token, theFilterDisplay))
663                                return true;
664                        tokens.add(token);
665                }
666
667                // Allow to search by the end of the phrase.  E.g.  "working proficiency" will match "Limited working proficiency"
668                for (int start = 0; start <= tokens.size() - 1; ++start) {
669                        for (int end = start + 1; end <= tokens.size(); ++end) {
670                                String sublist = String.join(" ", tokens.subList(start, end));
671                                if (startsWithIgnoreCase(sublist, theFilterDisplay))
672                                        return true;
673                        }
674                }
675                return false;
676        }
677
678        @Override
679        @Transactional(propagation = Propagation.REQUIRED)
680        public void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator) {
681                expandValueSet(theExpansionOptions, theValueSetToExpand, theValueSetCodeAccumulator, ExpansionFilter.NO_FILTER);
682        }
683
684        @SuppressWarnings("ConstantConditions")
685        private void expandValueSet(ValueSetExpansionOptions theExpansionOptions, ValueSet theValueSetToExpand, IValueSetConceptAccumulator theValueSetCodeAccumulator, @Nonnull ExpansionFilter theExpansionFilter) {
686                Set<String> addedCodes = new HashSet<>();
687
688                StopWatch sw = new StopWatch();
689                String valueSetInfo = getValueSetInfo(theValueSetToExpand);
690                ourLog.debug("Working with {}", valueSetInfo);
691
692                // Offset can't be combined with excludes
693                Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining();
694                if (skipCountRemaining != null && skipCountRemaining > 0) {
695                        if (theValueSetToExpand.getCompose().getExclude().size() > 0) {
696                                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo);
697                                throw new InvalidRequestException(msg);
698                        }
699                }
700
701                // Handle includes
702                ourLog.debug("Handling includes");
703                for (ValueSet.ConceptSetComponent include : theValueSetToExpand.getCompose().getInclude()) {
704                        for (int i = 0; ; i++) {
705                                int queryIndex = i;
706                                Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
707                                        boolean add = true;
708                                        return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, include, add, queryIndex, theExpansionFilter);
709                                });
710                                if (!shouldContinue) {
711                                        break;
712                                }
713                        }
714                }
715
716                // Handle excludes
717                ourLog.debug("Handling excludes");
718                for (ValueSet.ConceptSetComponent exclude : theValueSetToExpand.getCompose().getExclude()) {
719                        for (int i = 0; ; i++) {
720                                int queryIndex = i;
721                                Boolean shouldContinue = executeInNewTransactionIfNeeded(() -> {
722                                        boolean add = false;
723                                        ExpansionFilter expansionFilter = ExpansionFilter.NO_FILTER;
724                                        return expandValueSetHandleIncludeOrExclude(theExpansionOptions, theValueSetCodeAccumulator, addedCodes, exclude, add, queryIndex, expansionFilter);
725                                });
726                                if (!shouldContinue) {
727                                        break;
728                                }
729                        }
730                }
731
732                if (theValueSetCodeAccumulator instanceof ValueSetConceptAccumulator) {
733                        myTxTemplate.execute(t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder());
734                }
735
736                ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
737        }
738
739        /**
740         * Execute in a new transaction only if we aren't already in one. We do this because in some cases
741         * when performing a VS expansion we throw an {@link ExpansionTooCostlyException} and we don't want
742         * this to cause the TX to be marked a rollback prematurely.
743         */
744        private <T> T executeInNewTransactionIfNeeded(Supplier<T> theAction) {
745                if (TransactionSynchronizationManager.isSynchronizationActive()) {
746                        return theAction.get();
747                }
748                return myTxTemplate.execute(t -> theAction.get());
749        }
750
751        private String getValueSetInfo(ValueSet theValueSet) {
752                StringBuilder sb = new StringBuilder();
753                boolean isIdentified = false;
754                if (theValueSet.hasUrl()) {
755                        isIdentified = true;
756                        sb
757                                .append("ValueSet.url[")
758                                .append(theValueSet.getUrl())
759                                .append("]");
760                } else if (theValueSet.hasId()) {
761                        isIdentified = true;
762                        sb
763                                .append("ValueSet.id[")
764                                .append(theValueSet.getId())
765                                .append("]");
766                }
767
768                if (!isIdentified) {
769                        sb.append("Unidentified ValueSet");
770                }
771
772                return sb.toString();
773        }
774
775        /**
776         * @return Returns true if there are potentially more results to process.
777         */
778        private Boolean expandValueSetHandleIncludeOrExclude(@Nullable ValueSetExpansionOptions theExpansionOptions, IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter) {
779
780                String system = theIncludeOrExclude.getSystem();
781                boolean hasSystem = isNotBlank(system);
782                boolean hasValueSet = theIncludeOrExclude.getValueSet().size() > 0;
783
784                if (hasSystem) {
785
786                        if (theExpansionFilter.hasCode() && theExpansionFilter.getSystem() != null && !system.equals(theExpansionFilter.getSystem())) {
787                                return false;
788                        }
789
790                        ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system);
791
792                        TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(system);
793                        if (cs != null) {
794
795                                return expandValueSetHandleIncludeOrExcludeUsingDatabase(theValueSetCodeAccumulator, theAddedCodes, theIncludeOrExclude, theAdd, theQueryIndex, theExpansionFilter, system, cs);
796
797                        } else {
798
799                                if (theIncludeOrExclude.getConcept().size() > 0 && theExpansionFilter.hasCode()) {
800                                        if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) {
801                                                if (theIncludeOrExclude.getConcept().stream().noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) {
802                                                        return false;
803                                                }
804                                        }
805                                }
806
807                                Consumer<FhirVersionIndependentConcept> consumer = c -> {
808                                        addOrRemoveCode(theValueSetCodeAccumulator, theAddedCodes, theAdd, system, c.getCode(), c.getDisplay(), c.getSystemVersion());
809                                };
810
811                                try {
812                                        ConversionContext40_50.INSTANCE.init(new VersionConvertor_40_50(new BaseAdvisor_40_50()), "ValueSet");
813                                        org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent includeOrExclude = ValueSet40_50.convertConceptSetComponent(theIncludeOrExclude);
814                                        new InMemoryTerminologyServerValidationSupport(myContext).expandValueSetIncludeOrExclude(new ValidationSupportContext(provideValidationSupport()), consumer, includeOrExclude);
815                                } catch (InMemoryTerminologyServerValidationSupport.ExpansionCouldNotBeCompletedInternallyException e) {
816                                        if (!theExpansionOptions.isFailOnMissingCodeSystem() && e.getFailureType() == InMemoryTerminologyServerValidationSupport.FailureType.UNKNOWN_CODE_SYSTEM) {
817                                                return false;
818                                        }
819                                        throw new InternalErrorException(e);
820                                } finally {
821                                        ConversionContext40_50.INSTANCE.close("ValueSet");
822                                }
823
824                                return false;
825                        }
826
827                } else if (hasValueSet) {
828
829                        for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) {
830                                String valueSetUrl = nextValueSet.getValueAsString();
831                                ourLog.debug("Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl);
832
833                                ExpansionFilter subExpansionFilter = new ExpansionFilter(theExpansionFilter, theIncludeOrExclude.getFilter(), theValueSetCodeAccumulator.getCapacityRemaining());
834
835                                // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may time-out.
836
837                                ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl);
838                                if (valueSet == null) {
839                                        throw new ResourceNotFoundException("Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl));
840                                }
841
842                                expandValueSetIntoAccumulator(valueSet, theExpansionOptions, theValueSetCodeAccumulator, subExpansionFilter, theAdd);
843
844                        }
845
846                        return false;
847
848                } else {
849                        throw new InvalidRequestException("ValueSet contains " + (theAdd ? "include" : "exclude") + " criteria with no system defined");
850                }
851
852
853        }
854
855        private boolean isHibernateSearchEnabled() {
856                return myFulltextSearchSvc != null;
857        }
858
859        @Nonnull
860        private Boolean expandValueSetHandleIncludeOrExcludeUsingDatabase(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theIncludeOrExclude, boolean theAdd, int theQueryIndex, @Nonnull ExpansionFilter theExpansionFilter, String theSystem, TermCodeSystem theCs) {
861                String includeOrExcludeVersion = theIncludeOrExclude.getVersion();
862                TermCodeSystemVersion csv;
863                if (isEmpty(includeOrExcludeVersion)) {
864                        csv = theCs.getCurrentVersion();
865                } else {
866                        csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(theCs.getPid(), includeOrExcludeVersion);
867                }
868
869                SearchSession searchSession = Search.session(myEntityManager);
870                /*
871                 * If FullText searching is not enabled, we can handle only basic expansions
872                 * since we're going to do it without the database.
873                 */
874                if (!isHibernateSearchEnabled()) {
875                        expandWithoutHibernateSearch(theValueSetCodeAccumulator, csv, theAddedCodes, theIncludeOrExclude, theSystem, theAdd);
876                        return false;
877                }
878
879                /*
880                 * Ok, let's use hibernate search to build the expansion
881                 */
882                //Manually building a predicate since we need to throw it around.
883                SearchPredicateFactory predicate = searchSession.scope(TermConcept.class).predicate();
884
885                //Build the top-level expansion on filters.
886                PredicateFinalStep step = predicate.bool(b -> {
887                        b.must(predicate.match().field("myCodeSystemVersionPid").matching(csv.getPid()));
888
889                        if (theExpansionFilter.hasCode()) {
890                                b.must(predicate.match().field("myCode").matching(theExpansionFilter.getCode()));
891                        }
892
893                        String codeSystemUrlAndVersion = buildCodeSystemUrlAndVersion(theSystem, includeOrExcludeVersion);
894                        for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
895                                handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter);
896                        }
897                        for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) {
898                                handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter);
899                        }
900                });
901
902                List<String> codes = theIncludeOrExclude
903                        .getConcept()
904                        .stream()
905                        .filter(Objects::nonNull)
906                        .map(ValueSet.ConceptReferenceComponent::getCode)
907                        .filter(StringUtils::isNotBlank)
908                        .collect(Collectors.toList());
909
910                PredicateFinalStep expansionStep = buildExpansionPredicate(codes, predicate);
911                final PredicateFinalStep finishedQuery;
912                if (expansionStep == null) {
913                        finishedQuery = step;
914                } else {
915                        finishedQuery = predicate.bool().must(step).must(expansionStep);
916                }
917
918                /*
919                 * DM 2019-08-21 - Processing slows after any ValueSets with many codes explicitly identified. This might
920                 * be due to the dark arts that is memory management. Will monitor but not do anything about this right now.
921                 */
922                //BooleanQuery.setMaxClauseCount(SearchBuilder.getMaximumPageSize());
923                //TODO GGG HS looks like we can't set max clause count, but it can be set server side.
924                //BooleanQuery.setMaxClauseCount(10000);
925
926                StopWatch sw = new StopWatch();
927                AtomicInteger count = new AtomicInteger(0);
928
929                int maxResultsPerBatch = SearchBuilder.getMaximumPageSize();
930
931                /*
932                 * If the accumulator is bounded, we may reduce the size of the query to
933                 * Lucene in order to be more efficient.
934                 */
935                if (theAdd) {
936                        Integer accumulatorCapacityRemaining = theValueSetCodeAccumulator.getCapacityRemaining();
937                        if (accumulatorCapacityRemaining != null) {
938                                maxResultsPerBatch = Math.min(maxResultsPerBatch, accumulatorCapacityRemaining + 1);
939                        }
940                        if (maxResultsPerBatch <= 0) {
941                                return false;
942                        }
943                }
944
945                ourLog.debug("Beginning batch expansion for {} with max results per batch: {}", (theAdd ? "inclusion" : "exclusion"), maxResultsPerBatch);
946
947                StopWatch swForBatch = new StopWatch();
948                AtomicInteger countForBatch = new AtomicInteger(0);
949
950                SearchQuery<TermConcept> termConceptsQuery = searchSession.search(TermConcept.class)
951                        .where(f -> finishedQuery).toQuery();
952
953                ourLog.trace("About to query: {}", termConceptsQuery.queryString());
954                List<TermConcept> termConcepts = termConceptsQuery.fetchHits(theQueryIndex * maxResultsPerBatch, maxResultsPerBatch);
955
956                // If the include section had multiple codes, return the codes in the same order
957                if (codes.size() > 1) {
958                        termConcepts = new ArrayList<>(termConcepts);
959                        Map<String, Integer> codeToIndex = new HashMap<>(codes.size());
960                        for (int i = 0; i < codes.size(); i++) {
961                                codeToIndex.put(codes.get(i), i);
962                        }
963                        termConcepts.sort(((o1, o2) -> {
964                                Integer idx1 = codeToIndex.get(o1.getCode());
965                                Integer idx2 = codeToIndex.get(o2.getCode());
966                                return Comparators.nullsHigh().compare(idx1, idx2);
967                        }));
968                }
969
970                int resultsInBatch = termConcepts.size();
971                int firstResult = theQueryIndex * maxResultsPerBatch;// TODO GGG HS we lose the ability to check the index of the first result, so just best-guessing it here.
972                int delta = 0;
973                for (TermConcept concept : termConcepts) {
974                        count.incrementAndGet();
975                        countForBatch.incrementAndGet();
976                        if (theAdd && expansionStep != null) {
977                                ValueSet.ConceptReferenceComponent theIncludeConcept = getMatchedConceptIncludedInValueSet(theIncludeOrExclude, concept);
978                                if (theIncludeConcept != null && isNotBlank(theIncludeConcept.getDisplay())) {
979                                        concept.setDisplay(theIncludeConcept.getDisplay());
980                                }
981                        }
982                        boolean added = addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, concept, theAdd, includeOrExcludeVersion);
983                        if (added) {
984                                delta++;
985                        }
986                }
987
988                ourLog.debug("Batch expansion for {} with starting index of {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), firstResult, countForBatch, swForBatch.getMillis());
989                theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta);
990
991                if (resultsInBatch < maxResultsPerBatch) {
992                        ourLog.debug("Expansion for {} produced {} results in {}ms", (theAdd ? "inclusion" : "exclusion"), count, sw.getMillis());
993                        return false;
994                } else {
995                        return true;
996                }
997        }
998
999        private ValueSet.ConceptReferenceComponent getMatchedConceptIncludedInValueSet(ValueSet.ConceptSetComponent theIncludeOrExclude, TermConcept concept) {
1000                return theIncludeOrExclude
1001                        .getConcept()
1002                        .stream().filter(includedConcept -> includedConcept.getCode().equalsIgnoreCase(concept.getCode()))
1003                        .findFirst()
1004                        .orElse(null);
1005        }
1006
1007        /**
1008         * Helper method which builds a predicate for the expansion
1009         */
1010        private PredicateFinalStep buildExpansionPredicate(List<String> theCodes, SearchPredicateFactory thePredicate) {
1011                PredicateFinalStep expansionStep;
1012                /*
1013                 * Include/Exclude Concepts
1014                 */
1015                List<Term> codes = theCodes
1016                        .stream()
1017                        .map(t -> new Term("myCode", t))
1018                        .collect(Collectors.toList());
1019
1020                if (codes.size() > 0) {
1021                        expansionStep = thePredicate.bool(b -> {
1022                                b.minimumShouldMatchNumber(1);
1023                                for (Term code : codes) {
1024                                        b.should(thePredicate.match().field(code.field()).matching(code.text()));
1025                                }
1026                        });
1027                        return expansionStep;
1028                } else {
1029                        return null;
1030                }
1031        }
1032
1033        private String buildCodeSystemUrlAndVersion(String theSystem, String theIncludeOrExcludeVersion) {
1034                String codeSystemUrlAndVersion;
1035                if (theIncludeOrExcludeVersion != null) {
1036                        codeSystemUrlAndVersion = theSystem + "|" + theIncludeOrExcludeVersion;
1037                } else {
1038                        codeSystemUrlAndVersion = theSystem;
1039                }
1040                return codeSystemUrlAndVersion;
1041        }
1042
1043        private @Nonnull
1044        ValueSetExpansionOptions provideExpansionOptions(@Nullable ValueSetExpansionOptions theExpansionOptions) {
1045                if (theExpansionOptions != null) {
1046                        return theExpansionOptions;
1047                } else {
1048                        return DEFAULT_EXPANSION_OPTIONS;
1049                }
1050        }
1051
1052        private void addOrRemoveCode(IValueSetConceptAccumulator theValueSetCodeAccumulator, Set<String> theAddedCodes, boolean theAdd, String theSystem, String theCode, String theDisplay, String theSystemVersion) {
1053                if (theAdd && theAddedCodes.add(theSystem + "|" + theCode)) {
1054                        theValueSetCodeAccumulator.includeConcept(theSystem, theCode, theDisplay, null, null, theSystemVersion);
1055                }
1056                if (!theAdd && theAddedCodes.remove(theSystem + "|" + theCode)) {
1057                        theValueSetCodeAccumulator.excludeConcept(theSystem, theCode);
1058                }
1059        }
1060
1061        private void handleFilter(String theCodeSystemIdentifier, SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1062                if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) {
1063                        return;
1064                }
1065
1066                if (isBlank(theFilter.getValue()) || theFilter.getOp() == null || isBlank(theFilter.getProperty())) {
1067                        throw new InvalidRequestException("Invalid filter, must have fields populated: property op value");
1068                }
1069
1070                switch (theFilter.getProperty()) {
1071                        case "display:exact":
1072                        case "display":
1073                                handleFilterDisplay(theF, theB, theFilter);
1074                                break;
1075                        case "concept":
1076                        case "code":
1077                                handleFilterConceptAndCode(theCodeSystemIdentifier, theF, theB, theFilter);
1078                                break;
1079                        case "parent":
1080                        case "child":
1081                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1082                                handleFilterLoincParentChild(theF, theB, theFilter);
1083                                break;
1084                        case "ancestor":
1085                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1086                                handleFilterLoincAncestor2(theCodeSystemIdentifier, theF, theB, theFilter);
1087                                break;
1088                        case "descendant":
1089                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1090                                handleFilterLoincDescendant(theCodeSystemIdentifier, theF, theB, theFilter);
1091                                break;
1092                        case "copyright":
1093                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1094                                handleFilterLoincCopyright(theF, theB, theFilter);
1095                                break;
1096                        default:
1097                                handleFilterRegex(theF, theB, theFilter);
1098                                break;
1099                }
1100        }
1101
1102        private void handleFilterRegex(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1103                if (theFilter.getOp() == ValueSet.FilterOperator.REGEX) {
1104
1105                        /*
1106                         * We treat the regex filter as a match on the regex
1107                         * anywhere in the property string. The spec does not
1108                         * say whether or not this is the right behaviour, but
1109                         * there are examples that seem to suggest that it is.
1110                         */
1111                        String value = theFilter.getValue();
1112                        if (value.endsWith("$")) {
1113                                value = value.substring(0, value.length() - 1);
1114                        } else if (!value.endsWith(".*")) {
1115                                value = value + ".*";
1116                        }
1117                        if (!value.startsWith("^") && !value.startsWith(".*")) {
1118                                value = ".*" + value;
1119                        } else if (value.startsWith("^")) {
1120                                value = value.substring(1);
1121                        }
1122
1123                        Term term = new Term(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + theFilter.getProperty(), value);
1124
1125                        if (isFullTextSetToUseElastic()) {
1126                                String regexpQuery = "{'regexp':{'" + term.field() + "':{'value':'" + term.text() + "'}}}";
1127                                ourLog.debug("Build Elasticsearch Regexp Query: {}", regexpQuery);
1128                                theB.must(theF.extension(ElasticsearchExtension.get()).fromJson(regexpQuery));
1129                        } else {
1130                                RegexpQuery query = new RegexpQuery(term);
1131                                theB.must(theF.extension(LuceneExtension.get()).fromLuceneQuery(query));
1132                        }
1133                } else {
1134                        String value = theFilter.getValue();
1135                        Term term = new Term(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + theFilter.getProperty(), value);
1136                        theB.must(theF.match().field(term.field()).matching(term.text()));
1137
1138                }
1139        }
1140
1141        private void handleFilterLoincCopyright(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB, ValueSet.ConceptSetFilterComponent theFilter) {
1142                if (theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1143
1144                        String copyrightFilterValue = defaultString(theFilter.getValue()).toLowerCase();
1145                        switch (copyrightFilterValue) {
1146                                case "3rdparty":
1147                                        logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1148                                        addFilterLoincCopyright3rdParty(theF, theB);
1149                                        break;
1150                                case "loinc":
1151                                        logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1152                                        addFilterLoincCopyrightLoinc(theF, theB);
1153                                        break;
1154                                default:
1155                                        throwInvalidRequestForValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1156                        }
1157
1158                } else {
1159                        throwInvalidRequestForOpOnProperty(theFilter.getOp(), theFilter.getProperty());
1160                }
1161        }
1162
1163        private void addFilterLoincCopyrightLoinc(SearchPredicateFactory thePredicateFactory, BooleanPredicateClausesStep<?> theBooleanClause) {
1164                theBooleanClause.mustNot(thePredicateFactory.exists().field(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE"));
1165        }
1166
1167        private void addFilterLoincCopyright3rdParty(SearchPredicateFactory thePredicateFactory, BooleanPredicateClausesStep<?> theBooleanClause) {
1168                //TODO GGG HS These used to be Term term = new Term(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE", ".*");, which was lucene-specific.
1169                //TODO GGG HS ask diederik if this is equivalent.
1170                //This old .* regex is the same as an existence check on a field, which I've implemented here.
1171                theBooleanClause.must(thePredicateFactory.exists().field(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + "EXTERNAL_COPYRIGHT_NOTICE"));
1172        }
1173
1174        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1175        private void handleFilterLoincAncestor2(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1176                switch (theFilter.getOp()) {
1177                        case EQUAL:
1178                                addLoincFilterAncestorEqual(theSystem, f, b, theFilter);
1179                                break;
1180                        case IN:
1181                                addLoincFilterAncestorIn(theSystem, f, b, theFilter);
1182                                break;
1183                        default:
1184                                throw new InvalidRequestException("Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1185                }
1186
1187        }
1188
1189        private void addLoincFilterAncestorEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1190                addLoincFilterAncestorEqual(theSystem, f, b, theFilter.getProperty(), theFilter.getValue());
1191        }
1192
1193        private void addLoincFilterAncestorEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) {
1194                List<Term> terms = getAncestorTerms(theSystem, theProperty, theValue);
1195                b.must(f.bool(innerB -> terms.forEach(term -> innerB.should(f.match().field(term.field()).matching(term.text())))));
1196        }
1197
1198        private void addLoincFilterAncestorIn(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1199                String[] values = theFilter.getValue().split(",");
1200                List<Term> terms = new ArrayList<>();
1201                for (String value : values) {
1202                        terms.addAll(getAncestorTerms(theSystem, theFilter.getProperty(), value));
1203                }
1204                b.must(f.bool(innerB -> terms.forEach(term -> innerB.should(f.match().field(term.field()).matching(term.text())))));
1205
1206        }
1207
1208        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1209        private void handleFilterLoincParentChild(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1210                switch (theFilter.getOp()) {
1211                        case EQUAL:
1212                                addLoincFilterParentChildEqual(f, b, theFilter.getProperty(), theFilter.getValue());
1213                                break;
1214                        case IN:
1215                                addLoincFilterParentChildIn(f, b, theFilter);
1216                                break;
1217                        default:
1218                                throw new InvalidRequestException("Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1219                }
1220        }
1221
1222        private void addLoincFilterParentChildIn(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1223                String[] values = theFilter.getValue().split(",");
1224                List<Term> terms = new ArrayList<>();
1225                for (String value : values) {
1226                        logFilteringValueOnProperty(value, theFilter.getProperty());
1227                        terms.add(getPropertyTerm(theFilter.getProperty(), value));
1228                }
1229
1230                //TODO GGG HS: Not sure if this is the right equivalent...seems to be no equivalent to `TermsQuery` in HS6.
1231                //Far as I'm aware, this is a single element of a MUST portion of a bool, which itself should contain a list of OR'ed options, e.g.
1232                // shape == round && color == (green || blue)
1233                b.must(f.bool(innerB -> terms.forEach(term -> innerB.should(f.match().field(term.field()).matching(term.text())))));
1234        }
1235
1236        private void addLoincFilterParentChildEqual(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) {
1237                logFilteringValueOnProperty(theValue, theProperty);
1238                //TODO GGG HS: Not sure if this is the right equivalent...seems to be no equivalent to `TermsQuery` in HS6.
1239                //b.must(new TermsQuery(getPropertyTerm(theProperty, theValue)));
1240                //According to the DSL migration reference (https://docs.jboss.org/hibernate/search/6.0/migration/html_single/#queries-reference),
1241                //Since this property is handled with a specific analyzer, I'm not sure a terms match here is actually correct. The analyzer is literally just a whitespace tokenizer here.
1242
1243                b.must(f.match().field(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + theProperty).matching(theValue));
1244        }
1245
1246        private void handleFilterConceptAndCode(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1247
1248
1249                TermConcept code = findCode(theSystem, theFilter.getValue())
1250                        .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
1251
1252                if (theFilter.getOp() == ValueSet.FilterOperator.ISA) {
1253                        ourLog.debug(" * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
1254
1255                        b.must(f.match().field("myParentPids").matching("" + code.getId()));
1256                } else {
1257                        throw new InvalidRequestException("Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1258                }
1259        }
1260
1261
1262        private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) {
1263                String systemUrl = getUrlFromIdentifier(theSystemIdentifier);
1264                if (!isCodeSystemLoinc(systemUrl)) {
1265                        throw new InvalidRequestException("Invalid filter, property " + theProperty + " is LOINC-specific and cannot be used with system: " + systemUrl);
1266                }
1267        }
1268
1269        private boolean isCodeSystemLoinc(String theSystem) {
1270                return LOINC_URI.equals(theSystem);
1271        }
1272
1273        private void handleFilterDisplay(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1274                if (theFilter.getProperty().equals("display:exact") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1275                        addDisplayFilterExact(f, b, theFilter);
1276                } else if (theFilter.getProperty().equals("display") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1277                        if (theFilter.getValue().trim().contains(" ")) {
1278                                addDisplayFilterExact(f, b, theFilter);
1279                        } else {
1280                                addDisplayFilterInexact(f, b, theFilter);
1281                        }
1282                }
1283        }
1284
1285        private void addDisplayFilterExact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
1286                bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue()));
1287        }
1288
1289
1290        private void addDisplayFilterInexact(SearchPredicateFactory f, BooleanPredicateClausesStep<?> bool, ValueSet.ConceptSetFilterComponent nextFilter) {
1291                bool.must(f.phrase()
1292                        .field("myDisplay").boost(4.0f)
1293                        .field("myDisplayWordEdgeNGram").boost(1.0f)
1294                        .field("myDisplayEdgeNGram").boost(1.0f)
1295                        .matching(nextFilter.getValue().toLowerCase())
1296                        .slop(2)
1297                );
1298        }
1299
1300        private Term getPropertyTerm(String theProperty, String theValue) {
1301                return new Term(TermConceptPropertyBinder.CONCEPT_FIELD_PROPERTY_PREFIX + theProperty, theValue);
1302        }
1303
1304        private List<Term> getAncestorTerms(String theSystem, String theProperty, String theValue) {
1305                List<Term> retVal = new ArrayList<>();
1306
1307                TermConcept code = findCode(theSystem, theValue)
1308                        .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue));
1309
1310                retVal.add(new Term("myParentPids", "" + code.getId()));
1311                logFilteringValueOnProperty(theValue, theProperty);
1312
1313                return retVal;
1314        }
1315
1316        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1317        private void handleFilterLoincDescendant(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1318                switch (theFilter.getOp()) {
1319                        case EQUAL:
1320                                addLoincFilterDescendantEqual(theSystem, f, b, theFilter);
1321                                break;
1322                        case IN:
1323                                addLoincFilterDescendantIn(theSystem, f, b, theFilter);
1324                                break;
1325                        default:
1326                                throw new InvalidRequestException("Don't know how to handle op=" + theFilter.getOp() + " on property " + theFilter.getProperty());
1327                }
1328        }
1329
1330
1331        private void addLoincFilterDescendantEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1332                addLoincFilterDescendantEqual(theSystem, f, b, theFilter.getProperty(), theFilter.getValue());
1333        }
1334
1335        private void addLoincFilterDescendantIn(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1336                String[] values = theFilter.getValue().split(",");
1337                List<Term> terms = new ArrayList<>();
1338                for (String value : values) {
1339                        terms.addAll(getDescendantTerms(theSystem, theFilter.getProperty(), value));
1340                }
1341                searchByParentPids(f, b, terms);
1342        }
1343
1344        private void addLoincFilterDescendantEqual(String theSystem, SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) {
1345                List<Term> terms = getDescendantTerms(theSystem, theProperty, theValue);
1346                searchByParentPids(f, b, terms);
1347        }
1348
1349        private void searchByParentPids(SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, List<Term> theTerms) {
1350                List<Long> parentPids = convertTermsToParentPids(theTerms);
1351                b.must(f.bool(innerB -> {
1352                        parentPids.forEach(pid -> innerB.should(f.match().field(theTerms.get(0).field()).matching(pid)));
1353                }));
1354        }
1355
1356        private List<Long> convertTermsToParentPids(List<Term> theTerms) {
1357                return theTerms.stream().map(Term::text).map(Long::valueOf).collect(Collectors.toList());
1358        }
1359
1360
1361        private List<Term> getDescendantTerms(String theSystem, String theProperty, String theValue) {
1362                List<Term> retVal = new ArrayList<>();
1363
1364                TermConcept code = findCode(theSystem, theValue)
1365                        .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {" + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue));
1366
1367                String[] parentPids = code.getParentPidsAsString().split(" ");
1368                for (String parentPid : parentPids) {
1369                        if (!StringUtils.equals(parentPid, "NONE")) {
1370                                retVal.add(new Term("myId", parentPid));
1371                        }
1372                }
1373                logFilteringValueOnProperty(theValue, theProperty);
1374
1375                return retVal;
1376        }
1377
1378
1379        private void logFilteringValueOnProperty(String theValue, String theProperty) {
1380                ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty);
1381        }
1382
1383        private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) {
1384                throw new InvalidRequestException("Don't know how to handle op=" + theOp + " on property " + theProperty);
1385        }
1386
1387        private void throwInvalidRequestForValueOnProperty(String theValue, String theProperty) {
1388                throw new InvalidRequestException("Don't know how to handle value=" + theValue + " on property " + theProperty);
1389        }
1390
1391        private void expandWithoutHibernateSearch(IValueSetConceptAccumulator theValueSetCodeAccumulator, TermCodeSystemVersion theVersion, Set<String> theAddedCodes, ValueSet.ConceptSetComponent theInclude, String theSystem, boolean theAdd) {
1392                ourLog.trace("Hibernate search is not enabled");
1393
1394                if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {
1395                        Validate.isTrue(((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator).getParameter().isEmpty(), "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server.");
1396                }
1397
1398                Validate.isTrue(theInclude.getFilter().isEmpty(), "Can not expand ValueSet with filters - Hibernate Search is not enabled on this server.");
1399                Validate.isTrue(isNotBlank(theSystem), "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
1400
1401                if (theInclude.getConcept().isEmpty()) {
1402                        for (TermConcept next : theVersion.getConcepts()) {
1403                                addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), next.getId(), next.getParentPidsAsString());
1404                        }
1405                }
1406
1407                for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) {
1408                        if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) {
1409                                continue;
1410                        }
1411                        addCodeIfNotAlreadyAdded(theValueSetCodeAccumulator, theAddedCodes, theAdd, theSystem, theInclude.getVersion(), next.getCode(), next.getDisplay(), null, null);
1412                }
1413
1414
1415        }
1416
1417        @Override
1418        @Transactional
1419        public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) {
1420                IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId, theRequestDetails);
1421                ValueSet canonicalValueSet = toCanonicalValueSet(valueSet);
1422                Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(canonicalValueSet);
1423                if (!optionalTermValueSet.isPresent()) {
1424                        return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetNotFoundInTerminologyDatabase", theValueSetId);
1425                }
1426
1427                ourLog.info("Invalidating pre-calculated expansion on ValueSet {} / {}", theValueSetId, canonicalValueSet.getUrl());
1428
1429                TermValueSet termValueSet = optionalTermValueSet.get();
1430                if (termValueSet.getExpansionStatus() == TermValueSetPreExpansionStatusEnum.NOT_EXPANDED) {
1431                        return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetCantInvalidateNotYetPrecalculated", termValueSet.getUrl(), termValueSet.getExpansionStatus());
1432                }
1433
1434                Long totalConcepts = termValueSet.getTotalConcepts();
1435
1436                deletePreCalculatedValueSetContents(termValueSet);
1437
1438                termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.NOT_EXPANDED);
1439                termValueSet.setExpansionTimestamp(null);
1440                myTermValueSetDao.save(termValueSet);
1441                return myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts);
1442        }
1443
1444
1445        @Override
1446        @Transactional
1447        public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) {
1448                Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(theValueSet);
1449
1450                if (!optionalTermValueSet.isPresent()) {
1451                        ourLog.warn("ValueSet is not present in terminology tables. Will perform in-memory code validation. {}", getValueSetInfo(theValueSet));
1452                        return false;
1453                }
1454
1455                TermValueSet termValueSet = optionalTermValueSet.get();
1456
1457                if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
1458                        ourLog.warn("{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}",
1459                                getValueSetInfo(theValueSet), termValueSet.getExpansionStatus().name(), termValueSet.getExpansionStatus().getDescription());
1460                        return false;
1461                }
1462
1463                return true;
1464        }
1465
1466        private Optional<TermValueSet> fetchValueSetEntity(ValueSet theValueSet) {
1467                ResourcePersistentId valueSetResourcePid = myConceptStorageSvc.getValueSetResourcePid(theValueSet.getIdElement());
1468                Optional<TermValueSet> optionalTermValueSet = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getIdAsLong());
1469                return optionalTermValueSet;
1470        }
1471
1472        protected IValidationSupport.CodeValidationResult validateCodeIsInPreExpandedValueSet(
1473                ConceptValidationOptions theValidationOptions,
1474                ValueSet theValueSet, String theSystem, String theCode, String theDisplay, Coding theCoding, CodeableConcept theCodeableConcept) {
1475                assert TransactionSynchronizationManager.isSynchronizationActive();
1476
1477                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required");
1478                ResourcePersistentId valueSetResourcePid = myConceptStorageSvc.getValueSetResourcePid(theValueSet.getIdElement());
1479
1480                List<TermValueSetConcept> concepts = new ArrayList<>();
1481                if (isNotBlank(theCode)) {
1482                        if (theValidationOptions.isInferSystem()) {
1483                                concepts.addAll(myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid.getIdAsLong(), theCode));
1484                        } else if (isNotBlank(theSystem)) {
1485                                concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode));
1486                        }
1487                } else if (theCoding != null) {
1488                        if (theCoding.hasSystem() && theCoding.hasCode()) {
1489                                concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theCoding.getSystem(), theCoding.getCode()));
1490                        }
1491                } else if (theCodeableConcept != null) {
1492                        for (Coding coding : theCodeableConcept.getCoding()) {
1493                                if (coding.hasSystem() && coding.hasCode()) {
1494                                        concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, coding.getSystem(), coding.getCode()));
1495                                        if (!concepts.isEmpty()) {
1496                                                break;
1497                                        }
1498                                }
1499                        }
1500                } else {
1501                        return null;
1502                }
1503
1504                TermValueSet valueSetEntity = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getIdAsLong()).orElseThrow(() -> new IllegalStateException());
1505                Object timingDescription = toHumanReadableExpansionTimestamp(valueSetEntity);
1506                String msg = myContext.getLocalizer().getMessage(BaseTermReadSvcImpl.class, "validationPerformedAgainstPreExpansion", timingDescription);
1507
1508                if (theValidationOptions.isValidateDisplay() && concepts.size() > 0) {
1509                        String systemVersion = null;
1510                        for (TermValueSetConcept concept : concepts) {
1511                                systemVersion = concept.getSystemVersion();
1512                                if (isBlank(theDisplay) || isBlank(concept.getDisplay()) || theDisplay.equals(concept.getDisplay())) {
1513                                        return new IValidationSupport.CodeValidationResult()
1514                                                .setCode(concept.getCode())
1515                                                .setDisplay(concept.getDisplay())
1516                                                .setCodeSystemVersion(concept.getSystemVersion())
1517                                                .setMessage(msg);
1518                                }
1519                        }
1520
1521                        return createFailureCodeValidationResult(theSystem, theCode, systemVersion, " - Concept Display \"" + theDisplay + "\" does not match expected \"" + concepts.get(0).getDisplay() + "\". " + msg).setDisplay(concepts.get(0).getDisplay());
1522                }
1523
1524                if (!concepts.isEmpty()) {
1525                        return new IValidationSupport.CodeValidationResult()
1526                                .setCode(concepts.get(0).getCode())
1527                                .setDisplay(concepts.get(0).getDisplay())
1528                                .setCodeSystemVersion(concepts.get(0).getSystemVersion())
1529                                .setMessage(msg);
1530                }
1531
1532                return createFailureCodeValidationResult(theSystem, theCode, null, " - Unknown code " + theSystem + "#" + theCode + ". " + msg);
1533        }
1534
1535        private CodeValidationResult createFailureCodeValidationResult(String theSystem, String theCode, String theCodeSystemVersion, String theAppend) {
1536                return new CodeValidationResult()
1537                        .setSeverity(IssueSeverity.ERROR)
1538                        .setCodeSystemVersion(theCodeSystemVersion)
1539                        .setMessage("Unable to validate code " + theSystem + "#" + theCode + theAppend);
1540        }
1541
1542        private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode(ResourcePersistentId theResourcePid, String theSystem, String theCode) {
1543                assert TransactionSynchronizationManager.isSynchronizationActive();
1544
1545                List<TermValueSetConcept> retVal = new ArrayList<>();
1546                Optional<TermValueSetConcept> optionalTermValueSetConcept;
1547                int versionIndex = theSystem.indexOf("|");
1548                if (versionIndex >= 0) {
1549                        String systemUrl = theSystem.substring(0, versionIndex);
1550                        String systemVersion = theSystem.substring(versionIndex + 1);
1551                        optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCodeWithVersion(theResourcePid.getIdAsLong(), systemUrl, systemVersion, theCode);
1552                } else {
1553                        optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode(theResourcePid.getIdAsLong(), theSystem, theCode);
1554                }
1555                optionalTermValueSetConcept.ifPresent(retVal::add);
1556                return retVal;
1557        }
1558
1559        private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
1560                for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) {
1561                        TermConcept nextChild = nextChildLink.getChild();
1562                        if (addToSet(theSetToPopulate, nextChild)) {
1563                                fetchChildren(nextChild, theSetToPopulate);
1564                        }
1565                }
1566        }
1567
1568        private Optional<TermConcept> fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) {
1569                TermCodeSystemVersion codeSystem = myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid);
1570                return myConceptDao.findByCodeSystemAndCode(codeSystem, theCode);
1571        }
1572
1573        private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
1574                for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) {
1575                        TermConcept nextChild = nextChildLink.getParent();
1576                        if (addToSet(theSetToPopulate, nextChild)) {
1577                                fetchParents(nextChild, theSetToPopulate);
1578                        }
1579                }
1580        }
1581
1582        private CodeSystem.ConceptDefinitionComponent findCode(List<CodeSystem.ConceptDefinitionComponent> theConcepts, String theCode) {
1583                for (CodeSystem.ConceptDefinitionComponent next : theConcepts) {
1584                        if (theCode.equals(next.getCode())) {
1585                                return next;
1586                        }
1587                        CodeSystem.ConceptDefinitionComponent val = findCode(next.getConcept(), theCode);
1588                        if (val != null) {
1589                                return val;
1590                        }
1591                }
1592                return null;
1593        }
1594
1595        @Override
1596        public Optional<TermConcept> findCode(String theCodeSystem, String theCode) {
1597                /*
1598                 * Loading concepts without a transaction causes issues later on some
1599                 * platforms (e.g. PSQL) so this transactiontemplate is here to make
1600                 * sure that we always call this with an open transaction
1601                 */
1602                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
1603                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_MANDATORY);
1604
1605                return txTemplate.execute(t -> {
1606                        TermCodeSystemVersion csv = getCurrentCodeSystemVersion(theCodeSystem);
1607                        if (csv == null) {
1608                                return Optional.empty();
1609                        }
1610                        return myConceptDao.findByCodeSystemAndCode(csv, theCode);
1611                });
1612        }
1613
1614        @Nullable
1615        private TermCodeSystemVersion getCurrentCodeSystemVersion(String theCodeSystemIdentifier) {
1616                String version = getVersionFromIdentifier(theCodeSystemIdentifier);
1617                TermCodeSystemVersion retVal = myCodeSystemCurrentVersionCache.get(theCodeSystemIdentifier, t -> myTxTemplate.execute(tx -> {
1618                        TermCodeSystemVersion csv = null;
1619                        TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier));
1620                        if (cs != null) {
1621                                if (version != null) {
1622                                        csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version);
1623                                } else if (cs.getCurrentVersion() != null) {
1624                                        csv = cs.getCurrentVersion();
1625                                }
1626                        }
1627                        if (csv != null) {
1628                                return csv;
1629                        } else {
1630                                return NO_CURRENT_VERSION;
1631                        }
1632                }));
1633                if (retVal == NO_CURRENT_VERSION) {
1634                        return null;
1635                }
1636                return retVal;
1637        }
1638
1639        private String getVersionFromIdentifier(String theUri) {
1640                String retVal = null;
1641                if (StringUtils.isNotEmpty((theUri))) {
1642                        int versionSeparator = theUri.lastIndexOf('|');
1643                        if (versionSeparator != -1) {
1644                                retVal = theUri.substring(versionSeparator + 1);
1645                        }
1646                }
1647                return retVal;
1648        }
1649
1650        private String getUrlFromIdentifier(String theUri) {
1651                String retVal = theUri;
1652                if (StringUtils.isNotEmpty((theUri))) {
1653                        int versionSeparator = theUri.lastIndexOf('|');
1654                        if (versionSeparator != -1) {
1655                                retVal = theUri.substring(0, versionSeparator);
1656                        }
1657                }
1658                return retVal;
1659        }
1660
1661        @Transactional(propagation = Propagation.REQUIRED)
1662        @Override
1663        public Set<TermConcept> findCodesAbove(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
1664                StopWatch stopwatch = new StopWatch();
1665
1666                Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
1667                if (!concept.isPresent()) {
1668                        return Collections.emptySet();
1669                }
1670
1671                Set<TermConcept> retVal = new HashSet<>();
1672                retVal.add(concept.get());
1673
1674                fetchParents(concept.get(), retVal);
1675
1676                ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
1677                return retVal;
1678        }
1679
1680        @Transactional
1681        @Override
1682        public List<FhirVersionIndependentConcept> findCodesAbove(String theSystem, String theCode) {
1683                TermCodeSystem cs = getCodeSystem(theSystem);
1684                if (cs == null) {
1685                        return findCodesAboveUsingBuiltInSystems(theSystem, theCode);
1686                }
1687                TermCodeSystemVersion csv = cs.getCurrentVersion();
1688
1689                Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode);
1690                return toVersionIndependentConcepts(theSystem, codes);
1691        }
1692
1693        @Transactional(propagation = Propagation.REQUIRED)
1694        @Override
1695        public Set<TermConcept> findCodesBelow(Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
1696                Stopwatch stopwatch = Stopwatch.createStarted();
1697
1698                Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
1699                if (!concept.isPresent()) {
1700                        return Collections.emptySet();
1701                }
1702
1703                Set<TermConcept> retVal = new HashSet<>();
1704                retVal.add(concept.get());
1705
1706                fetchChildren(concept.get(), retVal);
1707
1708                ourLog.debug("Fetched {} codes below code {} in {}ms", retVal.size(), theCode, stopwatch.elapsed(TimeUnit.MILLISECONDS));
1709                return retVal;
1710        }
1711
1712        @Transactional
1713        @Override
1714        public List<FhirVersionIndependentConcept> findCodesBelow(String theSystem, String theCode) {
1715                TermCodeSystem cs = getCodeSystem(theSystem);
1716                if (cs == null) {
1717                        return findCodesBelowUsingBuiltInSystems(theSystem, theCode);
1718                }
1719                TermCodeSystemVersion csv = cs.getCurrentVersion();
1720
1721                Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode);
1722                return toVersionIndependentConcepts(theSystem, codes);
1723        }
1724
1725        private TermCodeSystem getCodeSystem(String theSystem) {
1726                return myCodeSystemDao.findByCodeSystemUri(theSystem);
1727        }
1728
1729        @PostConstruct
1730        public void start() {
1731                RuleBasedTransactionAttribute rules = new RuleBasedTransactionAttribute();
1732                rules.getRollbackRules().add(new NoRollbackRuleAttribute(ExpansionTooCostlyException.class));
1733                myTxTemplate = new TransactionTemplate(myTransactionManager, rules);
1734                scheduleJob();
1735        }
1736
1737        public void scheduleJob() {
1738                // Register scheduled job to pre-expand ValueSets
1739                // In the future it would be great to make this a cluster-aware task somehow
1740                ScheduledJobDefinition vsJobDefinition = new ScheduledJobDefinition();
1741                vsJobDefinition.setId(getClass().getName());
1742                vsJobDefinition.setJobClass(Job.class);
1743                mySchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition);
1744        }
1745
1746
1747        @Override
1748        public synchronized void preExpandDeferredValueSetsToTerminologyTables() {
1749                if (!myDaoConfig.isEnableTaskPreExpandValueSets()) {
1750                        return;
1751                }
1752                if (isNotSafeToPreExpandValueSets()) {
1753                        ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded.");
1754                        return;
1755                }
1756                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
1757
1758                while (true) {
1759                        StopWatch sw = new StopWatch();
1760                        TermValueSet valueSetToExpand = txTemplate.execute(t -> {
1761                                Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
1762                                if (!optionalTermValueSet.isPresent()) {
1763                                        return null;
1764                                }
1765
1766                                TermValueSet termValueSet = optionalTermValueSet.get();
1767                                termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANSION_IN_PROGRESS);
1768                                return myTermValueSetDao.saveAndFlush(termValueSet);
1769                        });
1770                        if (valueSetToExpand == null) {
1771                                return;
1772                        }
1773
1774                        // We have a ValueSet to pre-expand.
1775                        try {
1776                                ValueSet valueSet = txTemplate.execute(t -> {
1777                                        TermValueSet refreshedValueSetToExpand = myTermValueSetDao.findById(valueSetToExpand.getId()).orElseThrow(() -> new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId()));
1778                                        return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
1779                                });
1780                                assert valueSet != null;
1781
1782                                ValueSetConceptAccumulator accumulator = new ValueSetConceptAccumulator(valueSetToExpand, myTermValueSetDao, myValueSetConceptDao, myValueSetConceptDesignationDao);
1783                                expandValueSet(null, valueSet, accumulator);
1784
1785                                // We are done with this ValueSet.
1786                                txTemplate.execute(t -> {
1787                                        valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED);
1788                                        valueSetToExpand.setExpansionTimestamp(new Date());
1789                                        myTermValueSetDao.saveAndFlush(valueSetToExpand);
1790                                        return null;
1791                                });
1792
1793                                ourLog.info("Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}", valueSet.getId(), valueSet.getUrl(), accumulator.getConceptsSaved(), sw);
1794
1795                        } catch (Exception e) {
1796                                ourLog.error("Failed to pre-expand ValueSet: " + e.getMessage(), e);
1797                                txTemplate.execute(t -> {
1798                                        valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND);
1799                                        myTermValueSetDao.saveAndFlush(valueSetToExpand);
1800                                        return null;
1801                                });
1802                        }
1803                }
1804        }
1805
1806        @Override
1807        @Transactional
1808        public CodeValidationResult validateCode(ConceptValidationOptions theOptions, IIdType theValueSetId, String theValueSetIdentifier, String theCodeSystemIdentifierToValidate, String theCodeToValidate, String theDisplayToValidate, IBaseDatatype theCodingToValidate, IBaseDatatype theCodeableConceptToValidate) {
1809
1810                CodeableConcept codeableConcept = toCanonicalCodeableConcept(theCodeableConceptToValidate);
1811                boolean haveCodeableConcept = codeableConcept != null && codeableConcept.getCoding().size() > 0;
1812
1813                Coding canonicalCodingToValidate = toCanonicalCoding(theCodingToValidate);
1814                boolean haveCoding = canonicalCodingToValidate != null && canonicalCodingToValidate.isEmpty() == false;
1815
1816                boolean haveCode = theCodeToValidate != null && theCodeToValidate.isEmpty() == false;
1817
1818                if (!haveCodeableConcept && !haveCoding && !haveCode) {
1819                        throw new InvalidRequestException("No code, coding, or codeableConcept provided to validate");
1820                }
1821                if (!LogicUtil.multiXor(haveCodeableConcept, haveCoding, haveCode)) {
1822                        throw new InvalidRequestException("$validate-code can only validate (system AND code) OR (coding) OR (codeableConcept)");
1823                }
1824
1825                boolean haveIdentifierParam = isNotBlank(theValueSetIdentifier);
1826                String valueSetIdentifier;
1827                if (theValueSetId != null) {
1828                        IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId);
1829                        StringBuilder valueSetIdentifierBuilder = new StringBuilder(CommonCodeSystemsTerminologyService.getValueSetUrl(valueSet));
1830                        String valueSetVersion = CommonCodeSystemsTerminologyService.getValueSetVersion(valueSet);
1831                        if (valueSetVersion != null) {
1832                                valueSetIdentifierBuilder.append("|").append(valueSetVersion);
1833                        }
1834                        valueSetIdentifier = valueSetIdentifierBuilder.toString();
1835
1836                } else if (haveIdentifierParam) {
1837                        valueSetIdentifier = theValueSetIdentifier;
1838                } else {
1839                        throw new InvalidRequestException("Either ValueSet ID or ValueSet identifier or system and code must be provided. Unable to validate.");
1840                }
1841
1842                ValidationSupportContext validationContext = new ValidationSupportContext(provideValidationSupport());
1843
1844                String codeValueToValidate = theCodeToValidate;
1845                String codeSystemIdentifierValueToValidate = theCodeSystemIdentifierToValidate;
1846                String codeDisplayValueToValidate = theDisplayToValidate;
1847
1848                if (haveCodeableConcept) {
1849                        for (int i = 0; i < codeableConcept.getCoding().size(); i++) {
1850                                Coding nextCoding = codeableConcept.getCoding().get(i);
1851                                String codeSystemIdentifier;
1852                                if (nextCoding.hasVersion()) {
1853                                        codeSystemIdentifier = nextCoding.getSystem() + "|" + nextCoding.getVersion();
1854                                } else {
1855                                        codeSystemIdentifier = nextCoding.getSystem();
1856                                }
1857                                CodeValidationResult nextValidation = validateCode(validationContext, theOptions, codeSystemIdentifier, nextCoding.getCode(), nextCoding.getDisplay(), valueSetIdentifier);
1858                                if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) {
1859                                        return nextValidation;
1860                                }
1861                        }
1862                } else if (haveCoding) {
1863                        if (canonicalCodingToValidate.hasVersion()) {
1864                                codeSystemIdentifierValueToValidate = canonicalCodingToValidate.getSystem() + "|" + canonicalCodingToValidate.getVersion();
1865                        } else {
1866                                codeSystemIdentifierValueToValidate = canonicalCodingToValidate.getSystem();
1867                        }
1868                        codeValueToValidate = canonicalCodingToValidate.getCode();
1869                        codeDisplayValueToValidate = canonicalCodingToValidate.getDisplay();
1870                }
1871
1872                return validateCode(validationContext, theOptions, codeSystemIdentifierValueToValidate, codeValueToValidate, codeDisplayValueToValidate, valueSetIdentifier);
1873        }
1874
1875        private boolean isNotSafeToPreExpandValueSets() {
1876                return myDeferredStorageSvc != null && !myDeferredStorageSvc.isStorageQueueEmpty();
1877        }
1878
1879        protected abstract ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable);
1880
1881        private Optional<TermValueSet> getNextTermValueSetNotExpanded() {
1882                Optional<TermValueSet> retVal = Optional.empty();
1883                Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(PageRequest.of(0, 1), TermValueSetPreExpansionStatusEnum.NOT_EXPANDED);
1884
1885                if (!page.getContent().isEmpty()) {
1886                        retVal = Optional.of(page.getContent().get(0));
1887                }
1888
1889                return retVal;
1890        }
1891
1892        @Override
1893        @Transactional
1894        public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) {
1895
1896                ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied");
1897                if (isPlaceholder(theValueSet)) {
1898                        ourLog.info("Not storing TermValueSet for placeholder {}", theValueSet.getIdElement().toVersionless().getValueAsString());
1899                        return;
1900                }
1901
1902                ValidateUtil.isNotBlankOrThrowUnprocessableEntity(theValueSet.getUrl(), "ValueSet has no value for ValueSet.url");
1903                ourLog.info("Storing TermValueSet for {}", theValueSet.getIdElement().toVersionless().getValueAsString());
1904
1905                /*
1906                 * Get CodeSystem and validate CodeSystemVersion
1907                 */
1908                TermValueSet termValueSet = new TermValueSet();
1909                termValueSet.setResource(theResourceTable);
1910                termValueSet.setUrl(theValueSet.getUrl());
1911                termValueSet.setVersion(theValueSet.getVersion());
1912                termValueSet.setName(theValueSet.hasName() ? theValueSet.getName() : null);
1913
1914                // Delete version being replaced
1915                deleteValueSetForResource(theResourceTable);
1916
1917                /*
1918                 * Do the upload.
1919                 */
1920                String url = termValueSet.getUrl();
1921                String version = termValueSet.getVersion();
1922                Optional<TermValueSet> optionalExistingTermValueSetByUrl;
1923                if (version != null) {
1924                        optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndVersion(url, version);
1925                } else {
1926                        optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndNullVersion(url);
1927                }
1928                if (!optionalExistingTermValueSetByUrl.isPresent()) {
1929
1930                        myTermValueSetDao.save(termValueSet);
1931
1932                } else {
1933                        TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get();
1934                        String msg;
1935                        if (version != null) {
1936                                msg = myContext.getLocalizer().getMessage(
1937                                        BaseTermReadSvcImpl.class,
1938                                        "cannotCreateDuplicateValueSetUrlAndVersion",
1939                                        url, version, existingTermValueSet.getResource().getIdDt().toUnqualifiedVersionless().getValue());
1940                        } else {
1941                                msg = myContext.getLocalizer().getMessage(
1942                                        BaseTermReadSvcImpl.class,
1943                                        "cannotCreateDuplicateValueSetUrl",
1944                                        url, existingTermValueSet.getResource().getIdDt().toUnqualifiedVersionless().getValue());
1945                        }
1946                        throw new UnprocessableEntityException(msg);
1947                }
1948        }
1949
1950        @Override
1951        @Transactional
1952        public IFhirResourceDaoCodeSystem.SubsumesResult subsumes(IPrimitiveType<String> theCodeA, IPrimitiveType<String> theCodeB,
1953                                                                                                                                                                 IPrimitiveType<String> theSystem, IBaseCoding theCodingA, IBaseCoding theCodingB) {
1954                FhirVersionIndependentConcept conceptA = toConcept(theCodeA, theSystem, theCodingA);
1955                FhirVersionIndependentConcept conceptB = toConcept(theCodeB, theSystem, theCodingB);
1956
1957                if (!StringUtils.equals(conceptA.getSystem(), conceptB.getSystem())) {
1958                        throw new InvalidRequestException("Unable to test subsumption across different code systems");
1959                }
1960
1961                if (!StringUtils.equals(conceptA.getSystemVersion(), conceptB.getSystemVersion())) {
1962                        throw new InvalidRequestException("Unable to test subsumption across different code system versions");
1963                }
1964
1965                String codeASystemIdentifier;
1966                if (StringUtils.isNotEmpty(conceptA.getSystemVersion())) {
1967                        codeASystemIdentifier = conceptA.getSystem() + "|" + conceptA.getSystemVersion();
1968                } else {
1969                        codeASystemIdentifier = conceptA.getSystem();
1970                }
1971                TermConcept codeA = findCode(codeASystemIdentifier, conceptA.getCode())
1972                        .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptA));
1973
1974                String codeBSystemIdentifier;
1975                if (StringUtils.isNotEmpty(conceptB.getSystemVersion())) {
1976                        codeBSystemIdentifier = conceptB.getSystem() + "|" + conceptB.getSystemVersion();
1977                } else {
1978                        codeBSystemIdentifier = conceptB.getSystem();
1979                }
1980                TermConcept codeB = findCode(codeBSystemIdentifier, conceptB.getCode())
1981                        .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptB));
1982
1983                SearchSession searchSession = Search.session(myEntityManager);
1984
1985                ConceptSubsumptionOutcome subsumes;
1986                subsumes = testForSubsumption(searchSession, codeA, codeB, ConceptSubsumptionOutcome.SUBSUMES);
1987                if (subsumes == null) {
1988                        subsumes = testForSubsumption(searchSession, codeB, codeA, ConceptSubsumptionOutcome.SUBSUMEDBY);
1989                }
1990                if (subsumes == null) {
1991                        subsumes = ConceptSubsumptionOutcome.NOTSUBSUMED;
1992                }
1993
1994                return new IFhirResourceDaoCodeSystem.SubsumesResult(subsumes);
1995        }
1996
1997        protected abstract ValueSet toCanonicalValueSet(IBaseResource theValueSet);
1998
1999        protected IValidationSupport.LookupCodeResult lookupCode(String theSystem, String theCode, String theDisplayLanguage) {
2000                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
2001                return txTemplate.execute(t -> {
2002                        Optional<TermConcept> codeOpt = findCode(theSystem, theCode);
2003                        if (codeOpt.isPresent()) {
2004                                TermConcept code = codeOpt.get();
2005
2006                                IValidationSupport.LookupCodeResult result = new IValidationSupport.LookupCodeResult();
2007                                result.setCodeSystemDisplayName(code.getCodeSystemVersion().getCodeSystemDisplayName());
2008                                result.setCodeSystemVersion(code.getCodeSystemVersion().getCodeSystemVersionId());
2009                                result.setSearchedForSystem(theSystem);
2010                                result.setSearchedForCode(theCode);
2011                                result.setFound(true);
2012                                result.setCodeDisplay(code.getDisplay());
2013
2014                                for (TermConceptDesignation next : code.getDesignations()) {
2015                                        // filter out the designation based on displayLanguage if any
2016                                        if (isDisplayLanguageMatch(theDisplayLanguage, next.getLanguage())) {
2017                                                IValidationSupport.ConceptDesignation designation = new IValidationSupport.ConceptDesignation();
2018                                                designation.setLanguage(next.getLanguage());
2019                                                designation.setUseSystem(next.getUseSystem());
2020                                                designation.setUseCode(next.getUseCode());
2021                                                designation.setUseDisplay(next.getUseDisplay());
2022                                                designation.setValue(next.getValue());
2023                                                result.getDesignations().add(designation);
2024                                        }
2025                                }
2026
2027                                for (TermConceptProperty next : code.getProperties()) {
2028                                        if (next.getType() == TermConceptPropertyTypeEnum.CODING) {
2029                                                IValidationSupport.CodingConceptProperty property = new IValidationSupport.CodingConceptProperty(next.getKey(), next.getCodeSystem(), next.getValue(), next.getDisplay());
2030                                                result.getProperties().add(property);
2031                                        } else if (next.getType() == TermConceptPropertyTypeEnum.STRING) {
2032                                                IValidationSupport.StringConceptProperty property = new IValidationSupport.StringConceptProperty(next.getKey(), next.getValue());
2033                                                result.getProperties().add(property);
2034                                        } else {
2035                                                throw new InternalErrorException("Unknown type: " + next.getType());
2036                                        }
2037                                }
2038
2039                                return result;
2040
2041                        } else {
2042                                return null;
2043                        }
2044                });
2045        }
2046
2047
2048        @Nullable
2049        private ConceptSubsumptionOutcome testForSubsumption(SearchSession theSearchSession, TermConcept theLeft, TermConcept theRight, ConceptSubsumptionOutcome theOutput) {
2050                List<TermConcept> fetch = theSearchSession.search(TermConcept.class)
2051                        .where(f -> f.bool()
2052                                .must(f.match().field("myId").matching(theRight.getId()))
2053                                .must(f.match().field("myParentPids").matching(Long.toString(theLeft.getId())))
2054                        ).fetchHits(1);
2055
2056                if (fetch.size() > 0) {
2057                        return theOutput;
2058                } else {
2059                        return null;
2060                }
2061        }
2062
2063        private ArrayList<FhirVersionIndependentConcept> toVersionIndependentConcepts(String theSystem, Set<TermConcept> codes) {
2064                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(codes.size());
2065                for (TermConcept next : codes) {
2066                        retVal.add(new FhirVersionIndependentConcept(theSystem, next.getCode()));
2067                }
2068                return retVal;
2069        }
2070
2071        @Override
2072        @Transactional
2073        public CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, @Nonnull IBaseResource theValueSet) {
2074                invokeRunnableForUnitTest();
2075
2076                IPrimitiveType<?> urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class);
2077                String url = urlPrimitive.getValueAsString();
2078                if (isNotBlank(url)) {
2079                        return validateCode(theValidationSupportContext, theOptions, theCodeSystem, theCode, theDisplay, url);
2080                }
2081                return null;
2082        }
2083
2084        @CoverageIgnore
2085        @Override
2086        public IValidationSupport.CodeValidationResult validateCode(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theOptions, String theCodeSystem, String theCode, String theDisplay, String theValueSetUrl) {
2087                //TODO GGG TRY TO JUST AUTO_PASS HERE AND SEE WHAT HAPPENS.
2088                invokeRunnableForUnitTest();
2089
2090                if (isNotBlank(theValueSetUrl)) {
2091                        return validateCodeInValueSet(theValidationSupportContext, theOptions, theValueSetUrl, theCodeSystem, theCode, theDisplay);
2092                }
2093
2094                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
2095                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
2096                Optional<FhirVersionIndependentConcept> codeOpt = txTemplate.execute(t -> findCode(theCodeSystem, theCode).map(c -> new FhirVersionIndependentConcept(theCodeSystem, c.getCode())));
2097
2098                if (codeOpt != null && codeOpt.isPresent()) {
2099                        FhirVersionIndependentConcept code = codeOpt.get();
2100                        if (!theOptions.isValidateDisplay() || (isNotBlank(code.getDisplay()) && isNotBlank(theDisplay) && code.getDisplay().equals(theDisplay))) {
2101                                return new CodeValidationResult()
2102                                        .setCode(code.getCode())
2103                                        .setDisplay(code.getDisplay());
2104                        } else {
2105                                return createFailureCodeValidationResult(theCodeSystem, theCode, code.getSystemVersion(), " - Concept Display \"" + code.getDisplay() + "\" does not match expected \"" + code.getDisplay() + "\"").setDisplay(code.getDisplay());
2106                        }
2107                }
2108
2109                return createFailureCodeValidationResult(theCodeSystem, theCode, null, " - Code can not be found in CodeSystem");
2110        }
2111
2112        IValidationSupport.CodeValidationResult validateCodeInValueSet(ValidationSupportContext theValidationSupportContext, ConceptValidationOptions theValidationOptions, String theValueSetUrl, String theCodeSystem, String theCode, String theDisplay) {
2113                IBaseResource valueSet = theValidationSupportContext.getRootValidationSupport().fetchValueSet(theValueSetUrl);
2114
2115                // If we don't have a PID, this came from some source other than the JPA
2116                // database, so we don't need to check if it's pre-expanded or not
2117                if (valueSet instanceof IAnyResource) {
2118                        Long pid = IDao.RESOURCE_PID.get((IAnyResource) valueSet);
2119                        if (pid != null) {
2120                                if (isValueSetPreExpandedForCodeValidation(valueSet)) {
2121                                        return validateCodeIsInPreExpandedValueSet(theValidationOptions, valueSet, theCodeSystem, theCode, theDisplay, null, null);
2122                                }
2123                        }
2124                }
2125
2126                CodeValidationResult retVal;
2127                if (valueSet != null) {
2128                        retVal = new InMemoryTerminologyServerValidationSupport(myContext).validateCodeInValueSet(theValidationSupportContext, theValidationOptions, theCodeSystem, theCode, theDisplay, valueSet);
2129                } else {
2130                        String append = " - Unable to locate ValueSet[" + theValueSetUrl + "]";
2131                        retVal = createFailureCodeValidationResult(theCodeSystem, theCode, null, append);
2132                }
2133
2134                return retVal;
2135
2136        }
2137
2138        @Override
2139        public IBaseResource fetchCodeSystem(String theSystem) {
2140                IValidationSupport jpaValidationSupport = provideJpaValidationSupport();
2141                return jpaValidationSupport.fetchCodeSystem(theSystem);
2142        }
2143
2144        @Override
2145        public CodeSystem fetchCanonicalCodeSystemFromCompleteContext(String theSystem) {
2146                IValidationSupport validationSupport = provideValidationSupport();
2147                IBaseResource codeSystem = validationSupport.fetchCodeSystem(theSystem);
2148                if (codeSystem != null) {
2149                        codeSystem = toCanonicalCodeSystem(codeSystem);
2150                }
2151                return (CodeSystem) codeSystem;
2152        }
2153
2154        @Nonnull
2155        private IValidationSupport provideJpaValidationSupport() {
2156                IValidationSupport jpaValidationSupport = myJpaValidationSupport;
2157                if (jpaValidationSupport == null) {
2158                        jpaValidationSupport = myApplicationContext.getBean("myJpaValidationSupport", IValidationSupport.class);
2159                        myJpaValidationSupport = jpaValidationSupport;
2160                }
2161                return jpaValidationSupport;
2162        }
2163
2164        @Nonnull
2165        private IValidationSupport provideValidationSupport() {
2166                IValidationSupport validationSupport = myValidationSupport;
2167                if (validationSupport == null) {
2168                        validationSupport = myApplicationContext.getBean(IValidationSupport.class);
2169                        myValidationSupport = validationSupport;
2170                }
2171                return validationSupport;
2172        }
2173
2174        public ValueSet fetchCanonicalValueSetFromCompleteContext(String theSystem) {
2175                IValidationSupport validationSupport = provideValidationSupport();
2176                IBaseResource valueSet = validationSupport.fetchValueSet(theSystem);
2177                if (valueSet != null) {
2178                        valueSet = toCanonicalValueSet(valueSet);
2179                }
2180                return (ValueSet) valueSet;
2181        }
2182
2183        protected abstract CodeSystem toCanonicalCodeSystem(IBaseResource theCodeSystem);
2184
2185        @Override
2186        public IBaseResource fetchValueSet(String theValueSetUrl) {
2187                return provideJpaValidationSupport().fetchValueSet(theValueSetUrl);
2188        }
2189
2190        @Override
2191        public FhirContext getFhirContext() {
2192                return myContext;
2193        }
2194
2195        private void findCodesAbove(CodeSystem theSystem, String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2196                List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept();
2197                for (CodeSystem.ConceptDefinitionComponent next : conceptList) {
2198                        addTreeIfItContainsCode(theSystemString, next, theCode, theListToPopulate);
2199                }
2200        }
2201
2202        @Override
2203        public List<FhirVersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) {
2204                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
2205                CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem);
2206                if (system != null) {
2207                        findCodesAbove(system, theSystem, theCode, retVal);
2208                }
2209                return retVal;
2210        }
2211
2212        private void findCodesBelow(CodeSystem theSystem, String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2213                List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept();
2214                findCodesBelow(theSystemString, theCode, theListToPopulate, conceptList);
2215        }
2216
2217        private void findCodesBelow(String theSystemString, String theCode, List<FhirVersionIndependentConcept> theListToPopulate, List<CodeSystem.ConceptDefinitionComponent> conceptList) {
2218                for (CodeSystem.ConceptDefinitionComponent next : conceptList) {
2219                        if (theCode.equals(next.getCode())) {
2220                                addAllChildren(theSystemString, next, theListToPopulate);
2221                        } else {
2222                                findCodesBelow(theSystemString, theCode, theListToPopulate, next.getConcept());
2223                        }
2224                }
2225        }
2226
2227        @Override
2228        public List<FhirVersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) {
2229                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
2230                CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem);
2231                if (system != null) {
2232                        findCodesBelow(system, theSystem, theCode, retVal);
2233                }
2234                return retVal;
2235        }
2236
2237        private void addAllChildren(String theSystemString, CodeSystem.ConceptDefinitionComponent theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2238                if (isNotBlank(theCode.getCode())) {
2239                        theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theCode.getCode()));
2240                }
2241                for (CodeSystem.ConceptDefinitionComponent nextChild : theCode.getConcept()) {
2242                        addAllChildren(theSystemString, nextChild, theListToPopulate);
2243                }
2244        }
2245
2246        private boolean addTreeIfItContainsCode(String theSystemString, CodeSystem.ConceptDefinitionComponent theNext, String theCode, List<FhirVersionIndependentConcept> theListToPopulate) {
2247                boolean foundCodeInChild = false;
2248                for (CodeSystem.ConceptDefinitionComponent nextChild : theNext.getConcept()) {
2249                        foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate);
2250                }
2251
2252                if (theCode.equals(theNext.getCode()) || foundCodeInChild) {
2253                        theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theNext.getCode()));
2254                        return true;
2255                }
2256
2257                return false;
2258        }
2259
2260        @Nullable
2261        protected abstract Coding toCanonicalCoding(@Nullable IBaseDatatype theCoding);
2262
2263        @Nullable
2264        protected abstract Coding toCanonicalCoding(@Nullable IBaseCoding theCoding);
2265
2266        @Nullable
2267        protected abstract CodeableConcept toCanonicalCodeableConcept(@Nullable IBaseDatatype theCodeableConcept);
2268
2269        @NotNull
2270        private FhirVersionIndependentConcept toConcept(IPrimitiveType<String> theCodeType, IPrimitiveType<String> theCodeSystemIdentifierType, IBaseCoding theCodingType) {
2271                String code = theCodeType != null ? theCodeType.getValueAsString() : null;
2272                String system = theCodeSystemIdentifierType != null ? getUrlFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) : null;
2273                String systemVersion = theCodeSystemIdentifierType != null ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString()) : null;
2274                if (theCodingType != null) {
2275                        Coding canonicalizedCoding = toCanonicalCoding(theCodingType);
2276                        assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't
2277                        code = canonicalizedCoding.getCode();
2278                        system = canonicalizedCoding.getSystem();
2279                        systemVersion = canonicalizedCoding.getVersion();
2280                }
2281                return new FhirVersionIndependentConcept(system, code, null, systemVersion);
2282        }
2283
2284        @Override
2285        @Transactional
2286        public CodeValidationResult codeSystemValidateCode(IIdType theCodeSystemId, String theCodeSystemUrl, String theVersion, String theCode, String theDisplay, IBaseDatatype theCoding, IBaseDatatype theCodeableConcept) {
2287
2288                CodeableConcept codeableConcept = toCanonicalCodeableConcept(theCodeableConcept);
2289                boolean haveCodeableConcept = codeableConcept != null && codeableConcept.getCoding().size() > 0;
2290
2291                Coding coding = toCanonicalCoding(theCoding);
2292                boolean haveCoding = coding != null && coding.isEmpty() == false;
2293
2294                boolean haveCode = theCode != null && theCode.isEmpty() == false;
2295
2296                if (!haveCodeableConcept && !haveCoding && !haveCode) {
2297                        throw new InvalidRequestException("No code, coding, or codeableConcept provided to validate.");
2298                }
2299                if (!LogicUtil.multiXor(haveCodeableConcept, haveCoding, haveCode)) {
2300                        throw new InvalidRequestException("$validate-code can only validate (code) OR (coding) OR (codeableConcept)");
2301                }
2302
2303                boolean haveIdentifierParam = isNotBlank(theCodeSystemUrl);
2304                String codeSystemUrl;
2305                if (theCodeSystemId != null) {
2306                        IBaseResource codeSystem = myDaoRegistry.getResourceDao("CodeSystem").read(theCodeSystemId);
2307                        codeSystemUrl = CommonCodeSystemsTerminologyService.getCodeSystemUrl(codeSystem);
2308                } else if (haveIdentifierParam) {
2309                        codeSystemUrl = theCodeSystemUrl;
2310                } else {
2311                        throw new InvalidRequestException("Either CodeSystem ID or CodeSystem identifier must be provided. Unable to validate.");
2312                }
2313
2314
2315                String code = theCode;
2316                String display = theDisplay;
2317
2318                if (haveCodeableConcept) {
2319                        for (int i = 0; i < codeableConcept.getCoding().size(); i++) {
2320                                Coding nextCoding = codeableConcept.getCoding().get(i);
2321                                if (nextCoding.hasSystem()) {
2322                                        if (!codeSystemUrl.equalsIgnoreCase(nextCoding.getSystem())) {
2323                                                throw new InvalidRequestException("Coding.system '" + nextCoding.getSystem() + "' does not equal with CodeSystem.url '" + theCodeSystemUrl + "'. Unable to validate.");
2324                                        }
2325                                        codeSystemUrl = nextCoding.getSystem();
2326                                }
2327                                code = nextCoding.getCode();
2328                                display = nextCoding.getDisplay();
2329                                CodeValidationResult nextValidation = codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
2330                                if (nextValidation.isOk() || i == codeableConcept.getCoding().size() - 1) {
2331                                        return nextValidation;
2332                                }
2333                        }
2334                } else if (haveCoding) {
2335                        if (coding.hasSystem()) {
2336                                if (!codeSystemUrl.equalsIgnoreCase(coding.getSystem())) {
2337                                        throw new InvalidRequestException("Coding.system '" + coding.getSystem() + "' does not equal with CodeSystem.url '" + theCodeSystemUrl + "'. Unable to validate.");
2338                                }
2339                                codeSystemUrl = coding.getSystem();
2340                        }
2341                        code = coding.getCode();
2342                        display = coding.getDisplay();
2343                }
2344
2345                return codeSystemValidateCode(codeSystemUrl, theVersion, code, display);
2346        }
2347
2348
2349        /**
2350         * When the search is for unversioned loinc system it uses the forcedId to obtain the current
2351         * version, as it is not necessarily the last  one anymore.
2352         * For other cases it keeps on considering the last uploaded as the current
2353         */
2354        @Override
2355        public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) {
2356                if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) {
2357                        Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl);
2358                        if (! vsIdOpt.isPresent()) {
2359                                return Optional.empty();
2360                        }
2361
2362                        return myTermValueSetDao.findTermValueSetByForcedId(vsIdOpt.get());
2363                }
2364
2365                List<TermValueSet> termValueSetList = myTermValueSetDao.findTermValueSetByUrl(Pageable.ofSize(1), theUrl);
2366                if (termValueSetList.isEmpty()) {
2367                        return Optional.empty();
2368                }
2369
2370                return Optional.of(termValueSetList.get(0));
2371        }
2372
2373
2374        @SuppressWarnings("unchecked")
2375        private CodeValidationResult codeSystemValidateCode(String theCodeSystemUrl, String theCodeSystemVersion, String theCode, String theDisplay) {
2376
2377                CriteriaBuilder criteriaBuilder = myEntityManager.getCriteriaBuilder();
2378                CriteriaQuery<TermConcept> query = criteriaBuilder.createQuery(TermConcept.class);
2379                Root<TermConcept> root = query.from(TermConcept.class);
2380
2381                Fetch<TermCodeSystemVersion, TermConcept> systemVersionFetch = root.fetch("myCodeSystem", JoinType.INNER);
2382                Join<TermCodeSystemVersion, TermConcept> systemVersionJoin = (Join<TermCodeSystemVersion, TermConcept>) systemVersionFetch;
2383                Fetch<TermCodeSystem, TermCodeSystemVersion> systemFetch = systemVersionFetch.fetch("myCodeSystem", JoinType.INNER);
2384                Join<TermCodeSystem, TermCodeSystemVersion> systemJoin = (Join<TermCodeSystem, TermCodeSystemVersion>) systemFetch;
2385
2386                ArrayList<Predicate> predicates = new ArrayList<>();
2387
2388                if (isNotBlank(theCode)) {
2389                        predicates.add(criteriaBuilder.equal(root.get("myCode"), theCode));
2390                }
2391
2392                if (isNoneBlank(theCodeSystemUrl)) {
2393                        predicates.add(criteriaBuilder.equal(systemJoin.get("myCodeSystemUri"), theCodeSystemUrl));
2394                }
2395
2396                // for loinc CodeSystem last version is not necessarily the current anymore, so if no version is present
2397                // we need to query for the current, which is that which version is null
2398                if (isNoneBlank(theCodeSystemVersion)) {
2399                        predicates.add(criteriaBuilder.equal(systemVersionJoin.get("myCodeSystemVersionId"), theCodeSystemVersion));
2400                } else {
2401                        if (theCodeSystemUrl.toLowerCase(Locale.ROOT).contains(LOINC_LOW)) {
2402                                predicates.add(criteriaBuilder.isNull(systemVersionJoin.get("myCodeSystemVersionId")));
2403                        } else {
2404                                query.orderBy(criteriaBuilder.desc(root.get("myUpdated")));
2405                        }
2406                }
2407
2408                Predicate outerPredicate = criteriaBuilder.and(predicates.toArray(new Predicate[0]));
2409                query.where(outerPredicate);
2410
2411                final TypedQuery<TermConcept> typedQuery = myEntityManager.createQuery(query.select(root));
2412                org.hibernate.query.Query<TermConcept> hibernateQuery = (org.hibernate.query.Query<TermConcept>) typedQuery;
2413                hibernateQuery.setFetchSize(SINGLE_FETCH_SIZE);
2414                List<TermConcept> resultsList = hibernateQuery.getResultList();
2415
2416                if (!resultsList.isEmpty()) {
2417                        TermConcept concept = resultsList.get(0);
2418
2419                        if (isNotBlank(theDisplay) && !theDisplay.equals(concept.getDisplay())) {
2420                                String message = "Concept Display \"" + theDisplay + "\" does not match expected \"" + concept.getDisplay() + "\" for CodeSystem: " + theCodeSystemUrl;
2421                                return createFailureCodeValidationResult(theCodeSystemUrl, theCode, theCodeSystemVersion, message);
2422                        }
2423
2424                        return new CodeValidationResult().setCode(concept.getCode()).setDisplay(concept.getDisplay());
2425                }
2426
2427                return createFailureCodeValidationResult(theCodeSystemUrl, theCode, theCodeSystemVersion, " - Code is not found in CodeSystem: " + theCodeSystemUrl);
2428        }
2429
2430        @Override
2431        public Optional<IBaseResource> readCodeSystemByForcedId(String theForcedId) {
2432                @SuppressWarnings("unchecked")
2433                List<ResourceTable> resultList = (List<ResourceTable>) myEntityManager.createQuery(
2434                        "select f.myResource from ForcedId f " +
2435                                "where f.myResourceType = 'CodeSystem' and f.myForcedId = '" + theForcedId + "'").getResultList();
2436                if (resultList.isEmpty()) return Optional.empty();
2437
2438                if (resultList.size() > 1) throw new NonUniqueResultException(
2439                        "More than one CodeSystem is pointed by forcedId: " + theForcedId + ". Was constraint "
2440                                + ForcedId.IDX_FORCEDID_TYPE_FID + " removed?");
2441
2442                IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem");
2443                IBaseResource cs = csDao.toResource(resultList.get(0), false);
2444                return Optional.of(cs);
2445        }
2446
2447        public static class Job implements HapiJob {
2448                @Autowired
2449                private ITermReadSvc myTerminologySvc;
2450
2451                @Override
2452                public void execute(JobExecutionContext theContext) {
2453                        myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables();
2454                }
2455        }
2456
2457        static boolean isPlaceholder(DomainResource theResource) {
2458                boolean retVal = false;
2459                Extension extension = theResource.getExtensionByUrl(HapiExtensions.EXT_RESOURCE_PLACEHOLDER);
2460                if (extension != null && extension.hasValue() && extension.getValue() instanceof BooleanType) {
2461                        retVal = ((BooleanType) extension.getValue()).booleanValue();
2462                }
2463                return retVal;
2464        }
2465
2466        /**
2467         * This is only used for unit tests to test failure conditions
2468         */
2469        static void invokeRunnableForUnitTest() {
2470                if (myInvokeOnNextCallForUnitTest != null) {
2471                        Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest;
2472                        myInvokeOnNextCallForUnitTest = null;
2473                        invokeOnNextCallForUnitTest.run();
2474                }
2475        }
2476
2477        @VisibleForTesting
2478        public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) {
2479                myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest;
2480        }
2481
2482        static List<TermConcept> toPersistedConcepts(List<CodeSystem.ConceptDefinitionComponent> theConcept, TermCodeSystemVersion theCodeSystemVersion) {
2483                ArrayList<TermConcept> retVal = new ArrayList<>();
2484
2485                for (CodeSystem.ConceptDefinitionComponent next : theConcept) {
2486                        if (isNotBlank(next.getCode())) {
2487                                TermConcept termConcept = toTermConcept(next, theCodeSystemVersion);
2488                                retVal.add(termConcept);
2489                        }
2490                }
2491
2492                return retVal;
2493        }
2494
2495        @Nonnull
2496        static TermConcept toTermConcept(CodeSystem.ConceptDefinitionComponent theConceptDefinition, TermCodeSystemVersion theCodeSystemVersion) {
2497                TermConcept termConcept = new TermConcept();
2498                termConcept.setCode(theConceptDefinition.getCode());
2499                termConcept.setCodeSystemVersion(theCodeSystemVersion);
2500                termConcept.setDisplay(theConceptDefinition.getDisplay());
2501                termConcept.addChildren(toPersistedConcepts(theConceptDefinition.getConcept(), theCodeSystemVersion), RelationshipTypeEnum.ISA);
2502
2503                for (CodeSystem.ConceptDefinitionDesignationComponent designationComponent : theConceptDefinition.getDesignation()) {
2504                        if (isNotBlank(designationComponent.getValue())) {
2505                                TermConceptDesignation designation = termConcept.addDesignation();
2506                                designation.setLanguage(designationComponent.hasLanguage() ? designationComponent.getLanguage() : null);
2507                                if (designationComponent.hasUse()) {
2508                                        designation.setUseSystem(designationComponent.getUse().hasSystem() ? designationComponent.getUse().getSystem() : null);
2509                                        designation.setUseCode(designationComponent.getUse().hasCode() ? designationComponent.getUse().getCode() : null);
2510                                        designation.setUseDisplay(designationComponent.getUse().hasDisplay() ? designationComponent.getUse().getDisplay() : null);
2511                                }
2512                                designation.setValue(designationComponent.getValue());
2513                        }
2514                }
2515
2516                for (CodeSystem.ConceptPropertyComponent next : theConceptDefinition.getProperty()) {
2517                        TermConceptProperty property = new TermConceptProperty();
2518
2519                        property.setKey(next.getCode());
2520                        property.setConcept(termConcept);
2521                        property.setCodeSystemVersion(theCodeSystemVersion);
2522
2523                        if (next.getValue() instanceof StringType) {
2524                                property.setType(TermConceptPropertyTypeEnum.STRING);
2525                                property.setValue(next.getValueStringType().getValue());
2526                        } else if (next.getValue() instanceof Coding) {
2527                                Coding nextCoding = next.getValueCoding();
2528                                property.setType(TermConceptPropertyTypeEnum.CODING);
2529                                property.setCodeSystem(nextCoding.getSystem());
2530                                property.setValue(nextCoding.getCode());
2531                                property.setDisplay(nextCoding.getDisplay());
2532                        } else if (next.getValue() != null) {
2533                                // TODO: LOINC has properties of type BOOLEAN that we should handle
2534                                ourLog.warn("Don't know how to handle properties of type: " + next.getValue().getClass());
2535                                continue;
2536                        }
2537
2538                        termConcept.getProperties().add(property);
2539                }
2540                return termConcept;
2541        }
2542
2543    static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) {
2544                // NOTE: return the designation when one of then is not specified.
2545                if (theReqLang == null || theStoredLang == null)
2546                        return true;
2547
2548                return theReqLang.equalsIgnoreCase(theStoredLang);
2549    }
2550
2551}