001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.term;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.FhirVersionEnum;
024import ca.uhn.fhir.context.support.ConceptValidationOptions;
025import ca.uhn.fhir.context.support.IValidationSupport;
026import ca.uhn.fhir.context.support.LookupCodeRequest;
027import ca.uhn.fhir.context.support.ValidationSupportContext;
028import ca.uhn.fhir.context.support.ValueSetExpansionOptions;
029import ca.uhn.fhir.i18n.Msg;
030import ca.uhn.fhir.interceptor.model.RequestPartitionId;
031import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
032import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
033import ca.uhn.fhir.jpa.api.dao.IDao;
034import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
035import ca.uhn.fhir.jpa.api.dao.IFhirResourceDaoCodeSystem;
036import ca.uhn.fhir.jpa.api.svc.IIdHelperService;
037import ca.uhn.fhir.jpa.api.svc.ResolveIdentityMode;
038import ca.uhn.fhir.jpa.config.HibernatePropertiesProvider;
039import ca.uhn.fhir.jpa.config.util.ConnectionPoolInfoProvider;
040import ca.uhn.fhir.jpa.config.util.IConnectionPoolInfoProvider;
041import ca.uhn.fhir.jpa.dao.IFulltextSearchSvc;
042import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
043import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemDao;
044import ca.uhn.fhir.jpa.dao.data.ITermCodeSystemVersionDao;
045import ca.uhn.fhir.jpa.dao.data.ITermConceptDao;
046import ca.uhn.fhir.jpa.dao.data.ITermConceptDesignationDao;
047import ca.uhn.fhir.jpa.dao.data.ITermConceptPropertyDao;
048import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDao;
049import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptDesignationDao;
050import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewDao;
051import ca.uhn.fhir.jpa.dao.data.ITermValueSetConceptViewOracleDao;
052import ca.uhn.fhir.jpa.dao.data.ITermValueSetDao;
053import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
054import ca.uhn.fhir.jpa.entity.ITermValueSetConceptView;
055import ca.uhn.fhir.jpa.entity.TermCodeSystem;
056import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
057import ca.uhn.fhir.jpa.entity.TermConcept;
058import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
059import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
060import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink.RelationshipTypeEnum;
061import ca.uhn.fhir.jpa.entity.TermConceptProperty;
062import ca.uhn.fhir.jpa.entity.TermConceptPropertyTypeEnum;
063import ca.uhn.fhir.jpa.entity.TermValueSet;
064import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
065import ca.uhn.fhir.jpa.entity.TermValueSetPreExpansionStatusEnum;
066import ca.uhn.fhir.jpa.model.dao.JpaPid;
067import ca.uhn.fhir.jpa.model.entity.ResourceTable;
068import ca.uhn.fhir.jpa.model.sched.HapiJob;
069import ca.uhn.fhir.jpa.model.sched.IHasScheduledJobs;
070import ca.uhn.fhir.jpa.model.sched.ISchedulerService;
071import ca.uhn.fhir.jpa.model.sched.ScheduledJobDefinition;
072import ca.uhn.fhir.jpa.model.util.JpaConstants;
073import ca.uhn.fhir.jpa.search.builder.SearchBuilder;
074import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
075import ca.uhn.fhir.jpa.term.api.ITermReadSvc;
076import ca.uhn.fhir.jpa.term.api.ReindexTerminologyResult;
077import ca.uhn.fhir.jpa.term.ex.ExpansionTooCostlyException;
078import ca.uhn.fhir.rest.api.Constants;
079import ca.uhn.fhir.rest.api.server.RequestDetails;
080import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
081import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
082import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
083import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
084import ca.uhn.fhir.util.CoverageIgnore;
085import ca.uhn.fhir.util.FhirVersionIndependentConcept;
086import ca.uhn.fhir.util.HapiExtensions;
087import ca.uhn.fhir.util.StopWatch;
088import ca.uhn.fhir.util.UrlUtil;
089import ca.uhn.fhir.util.ValidateUtil;
090import ca.uhn.hapi.converters.canonical.VersionCanonicalizer;
091import com.google.common.annotations.VisibleForTesting;
092import com.google.common.base.Stopwatch;
093import com.google.common.collect.ArrayListMultimap;
094import jakarta.annotation.Nonnull;
095import jakarta.annotation.Nullable;
096import jakarta.annotation.PostConstruct;
097import jakarta.persistence.EntityManager;
098import jakarta.persistence.NonUniqueResultException;
099import jakarta.persistence.PersistenceContext;
100import jakarta.persistence.PersistenceContextType;
101import org.apache.commons.collections4.ListUtils;
102import org.apache.commons.lang3.ObjectUtils;
103import org.apache.commons.lang3.StringUtils;
104import org.apache.commons.lang3.Validate;
105import org.apache.commons.lang3.time.DateUtils;
106import org.apache.lucene.index.Term;
107import org.apache.lucene.search.IndexSearcher;
108import org.hibernate.CacheMode;
109import org.hibernate.search.engine.search.predicate.dsl.BooleanPredicateClausesStep;
110import org.hibernate.search.engine.search.predicate.dsl.PredicateFinalStep;
111import org.hibernate.search.engine.search.predicate.dsl.SearchPredicateFactory;
112import org.hibernate.search.engine.search.query.SearchQuery;
113import org.hibernate.search.engine.search.query.SearchScroll;
114import org.hibernate.search.engine.search.query.SearchScrollResult;
115import org.hibernate.search.mapper.orm.Search;
116import org.hibernate.search.mapper.orm.common.EntityReference;
117import org.hibernate.search.mapper.orm.session.SearchSession;
118import org.hibernate.search.mapper.pojo.massindexing.impl.PojoMassIndexingLoggingMonitor;
119import org.hl7.fhir.common.hapi.validation.support.InMemoryTerminologyServerValidationSupport;
120import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50;
121import org.hl7.fhir.convertors.context.ConversionContext40_50;
122import org.hl7.fhir.convertors.conv40_50.VersionConvertor_40_50;
123import org.hl7.fhir.convertors.conv40_50.resources40_50.ValueSet40_50;
124import org.hl7.fhir.instance.model.api.IAnyResource;
125import org.hl7.fhir.instance.model.api.IBaseCoding;
126import org.hl7.fhir.instance.model.api.IBaseDatatype;
127import org.hl7.fhir.instance.model.api.IBaseResource;
128import org.hl7.fhir.instance.model.api.IIdType;
129import org.hl7.fhir.instance.model.api.IPrimitiveType;
130import org.hl7.fhir.r4.model.BooleanType;
131import org.hl7.fhir.r4.model.CanonicalType;
132import org.hl7.fhir.r4.model.CodeSystem;
133import org.hl7.fhir.r4.model.CodeableConcept;
134import org.hl7.fhir.r4.model.Coding;
135import org.hl7.fhir.r4.model.DateTimeType;
136import org.hl7.fhir.r4.model.DecimalType;
137import org.hl7.fhir.r4.model.DomainResource;
138import org.hl7.fhir.r4.model.Enumerations;
139import org.hl7.fhir.r4.model.Extension;
140import org.hl7.fhir.r4.model.InstantType;
141import org.hl7.fhir.r4.model.IntegerType;
142import org.hl7.fhir.r4.model.StringType;
143import org.hl7.fhir.r4.model.ValueSet;
144import org.hl7.fhir.r4.model.codesystems.ConceptSubsumptionOutcome;
145import org.quartz.JobExecutionContext;
146import org.springframework.beans.factory.annotation.Autowired;
147import org.springframework.context.ApplicationContext;
148import org.springframework.data.domain.PageRequest;
149import org.springframework.data.domain.Pageable;
150import org.springframework.data.domain.Slice;
151import org.springframework.transaction.PlatformTransactionManager;
152import org.springframework.transaction.TransactionDefinition;
153import org.springframework.transaction.annotation.Propagation;
154import org.springframework.transaction.annotation.Transactional;
155import org.springframework.transaction.interceptor.NoRollbackRuleAttribute;
156import org.springframework.transaction.interceptor.RuleBasedTransactionAttribute;
157import org.springframework.transaction.support.TransactionSynchronizationManager;
158import org.springframework.transaction.support.TransactionTemplate;
159import org.springframework.util.comparator.Comparators;
160
161import java.util.ArrayList;
162import java.util.Arrays;
163import java.util.Collection;
164import java.util.Collections;
165import java.util.Date;
166import java.util.HashMap;
167import java.util.HashSet;
168import java.util.LinkedHashMap;
169import java.util.List;
170import java.util.Map;
171import java.util.Objects;
172import java.util.Optional;
173import java.util.Set;
174import java.util.StringTokenizer;
175import java.util.UUID;
176import java.util.concurrent.TimeUnit;
177import java.util.function.Consumer;
178import java.util.function.Supplier;
179import java.util.stream.Collectors;
180
181import static ca.uhn.fhir.jpa.entity.TermConceptPropertyBinder.CONCEPT_PROPERTY_PREFIX_NAME;
182import static ca.uhn.fhir.jpa.term.api.ITermLoaderSvc.LOINC_URI;
183import static java.lang.String.join;
184import static java.util.stream.Collectors.joining;
185import static java.util.stream.Collectors.toList;
186import static java.util.stream.Collectors.toSet;
187import static org.apache.commons.lang3.ObjectUtils.defaultIfNull;
188import static org.apache.commons.lang3.StringUtils.defaultString;
189import static org.apache.commons.lang3.StringUtils.isBlank;
190import static org.apache.commons.lang3.StringUtils.isEmpty;
191import static org.apache.commons.lang3.StringUtils.isNoneBlank;
192import static org.apache.commons.lang3.StringUtils.isNotBlank;
193import static org.apache.commons.lang3.StringUtils.lowerCase;
194import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase;
195
196public class TermReadSvcImpl implements ITermReadSvc, IHasScheduledJobs {
197        public static final int DEFAULT_FETCH_SIZE = 250;
198        public static final int DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS = 2;
199        // doesn't seem to be much gain by using more threads than this value
200        public static final int MAX_MASS_INDEXER_OBJECT_LOADING_THREADS = 6;
201        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TermReadSvcImpl.class);
202        private static final ValueSetExpansionOptions DEFAULT_EXPANSION_OPTIONS = new ValueSetExpansionOptions();
203        private static final TermCodeSystemVersionDetails NO_CURRENT_VERSION = new TermCodeSystemVersionDetails(-1L, null);
204        private static final String OUR_PIPE_CHARACTER = "|";
205        private static final int SECONDS_IN_MINUTE = 60;
206        private static final int INDEXED_ROOTS_LOGGING_COUNT = 50_000;
207        private static final String CS_USERDATA_CURRENT_VERSION = TermReadSvcImpl.class.getName() + "_CS_CURRENT_VERSION";
208        private static final String VS_USERDATA_CURRENT_VERSION = TermReadSvcImpl.class.getName() + "_VS_CURRENT_VERSION";
209        private static Runnable myInvokeOnNextCallForUnitTest;
210        private static boolean ourForceDisableHibernateSearchForUnitTest;
211
212        @Autowired
213        protected DaoRegistry myDaoRegistry;
214
215        @Autowired
216        protected ITermCodeSystemDao myCodeSystemDao;
217
218        @Autowired
219        protected ITermConceptDao myConceptDao;
220
221        @Autowired
222        protected ITermConceptPropertyDao myConceptPropertyDao;
223
224        @Autowired
225        protected ITermConceptDesignationDao myConceptDesignationDao;
226
227        @Autowired
228        protected ITermValueSetDao myTermValueSetDao;
229
230        @Autowired
231        protected ITermValueSetConceptDao myValueSetConceptDao;
232
233        @Autowired
234        protected ITermValueSetConceptDesignationDao myValueSetConceptDesignationDao;
235
236        @Autowired
237        protected FhirContext myContext;
238
239        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
240        protected EntityManager myEntityManager;
241
242        private boolean myPreExpandingValueSets = false;
243
244        @Autowired
245        private ITermCodeSystemVersionDao myCodeSystemVersionDao;
246
247        @Autowired
248        private JpaStorageSettings myStorageSettings;
249
250        private TransactionTemplate myTxTemplate;
251
252        @Autowired
253        private PlatformTransactionManager myTransactionManager;
254
255        @Autowired(required = false)
256        private IFulltextSearchSvc myFulltextSearchSvc;
257
258        @Autowired
259        private PlatformTransactionManager myTxManager;
260
261        @Autowired
262        private ITermConceptDao myTermConceptDao;
263
264        @Autowired
265        private ITermValueSetConceptViewDao myTermValueSetConceptViewDao;
266
267        @Autowired
268        private ITermValueSetConceptViewOracleDao myTermValueSetConceptViewOracleDao;
269
270        @Autowired(required = false)
271        private ITermDeferredStorageSvc myDeferredStorageSvc;
272
273        @Autowired
274        private IIdHelperService<JpaPid> myIdHelperService;
275
276        @Autowired
277        private ApplicationContext myApplicationContext;
278
279        private volatile IValidationSupport myJpaValidationSupport;
280        private volatile IValidationSupport myValidationSupport;
281        // We need this bean so we can tell which mode hibernate search is running in.
282        @Autowired
283        private HibernatePropertiesProvider myHibernatePropertiesProvider;
284
285        @Autowired
286        private VersionCanonicalizer myVersionCanonicalizer;
287
288        @Autowired
289        private IJpaStorageResourceParser myJpaStorageResourceParser;
290
291        @Autowired
292        private InMemoryTerminologyServerValidationSupport myInMemoryTerminologyServerValidationSupport;
293
294        @Autowired
295        private ValueSetConceptAccumulatorFactory myValueSetConceptAccumulatorFactory;
296
297        @Override
298        public boolean isCodeSystemSupported(ValidationSupportContext theValidationSupportContext, String theSystem) {
299                if (isBlank(theSystem)) {
300                        return false;
301                }
302                TermCodeSystemVersionDetails cs = getCurrentCodeSystemVersion(theValidationSupportContext, theSystem);
303                return cs != null;
304        }
305
306        @Override
307        public boolean isValueSetSupported(ValidationSupportContext theValidationSupportContext, String theValueSetUrl) {
308                return fetchValueSet(theValueSetUrl) != null;
309        }
310
311        private boolean addCodeIfNotAlreadyAdded(
312                        @Nullable ValueSetExpansionOptions theExpansionOptions,
313                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
314                        Set<String> theAddedCodes,
315                        TermConcept theConcept,
316                        boolean theAdd,
317                        String theValueSetIncludeVersion) {
318                String codeSystem = theConcept.getCodeSystemVersion().getCodeSystem().getCodeSystemUri();
319                String codeSystemVersion = theConcept.getCodeSystemVersion().getCodeSystemVersionId();
320                String code = theConcept.getCode();
321                String display = theConcept.getDisplay();
322                Long sourceConceptPid = theConcept.getId();
323                String directParentPids = "";
324
325                if (theExpansionOptions != null && theExpansionOptions.isIncludeHierarchy()) {
326                        directParentPids = theConcept.getParents().stream()
327                                        .map(t -> t.getParent().getId().toString())
328                                        .collect(joining(" "));
329                }
330
331                Collection<TermConceptDesignation> designations = theConcept.getDesignations();
332                if (StringUtils.isNotEmpty(theValueSetIncludeVersion)) {
333                        return addCodeIfNotAlreadyAdded(
334                                        theValueSetCodeAccumulator,
335                                        theAddedCodes,
336                                        designations,
337                                        theAdd,
338                                        codeSystem + OUR_PIPE_CHARACTER + theValueSetIncludeVersion,
339                                        code,
340                                        display,
341                                        sourceConceptPid,
342                                        directParentPids,
343                                        codeSystemVersion);
344                } else {
345                        return addCodeIfNotAlreadyAdded(
346                                        theValueSetCodeAccumulator,
347                                        theAddedCodes,
348                                        designations,
349                                        theAdd,
350                                        codeSystem,
351                                        code,
352                                        display,
353                                        sourceConceptPid,
354                                        directParentPids,
355                                        codeSystemVersion);
356                }
357        }
358
359        private boolean addCodeIfNotAlreadyAdded(
360                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
361                        Set<String> theAddedCodes,
362                        boolean theAdd,
363                        String theCodeSystem,
364                        String theCodeSystemVersion,
365                        String theCode,
366                        String theDisplay,
367                        Long theSourceConceptPid,
368                        String theSourceConceptDirectParentPids,
369                        Collection<TermConceptDesignation> theDesignations) {
370                if (StringUtils.isNotEmpty(theCodeSystemVersion)) {
371                        if (isNoneBlank(theCodeSystem, theCode)) {
372                                if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) {
373                                        theValueSetCodeAccumulator.includeConceptWithDesignations(
374                                                        theCodeSystem + OUR_PIPE_CHARACTER + theCodeSystemVersion,
375                                                        theCode,
376                                                        theDisplay,
377                                                        theDesignations,
378                                                        theSourceConceptPid,
379                                                        theSourceConceptDirectParentPids,
380                                                        theCodeSystemVersion);
381                                        return true;
382                                }
383
384                                if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) {
385                                        theValueSetCodeAccumulator.excludeConcept(
386                                                        theCodeSystem + OUR_PIPE_CHARACTER + theCodeSystemVersion, theCode);
387                                        return true;
388                                }
389                        }
390                } else {
391                        if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) {
392                                theValueSetCodeAccumulator.includeConceptWithDesignations(
393                                                theCodeSystem,
394                                                theCode,
395                                                theDisplay,
396                                                theDesignations,
397                                                theSourceConceptPid,
398                                                theSourceConceptDirectParentPids,
399                                                theCodeSystemVersion);
400                                return true;
401                        }
402
403                        if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) {
404                                theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
405                                return true;
406                        }
407                }
408
409                return false;
410        }
411
412        private boolean addCodeIfNotAlreadyAdded(
413                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
414                        Set<String> theAddedCodes,
415                        Collection<TermConceptDesignation> theDesignations,
416                        boolean theAdd,
417                        String theCodeSystem,
418                        String theCode,
419                        String theDisplay,
420                        Long theSourceConceptPid,
421                        String theSourceConceptDirectParentPids,
422                        String theSystemVersion) {
423                if (isNoneBlank(theCodeSystem, theCode)) {
424                        if (theAdd && theAddedCodes.add(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) {
425                                theValueSetCodeAccumulator.includeConceptWithDesignations(
426                                                theCodeSystem,
427                                                theCode,
428                                                theDisplay,
429                                                theDesignations,
430                                                theSourceConceptPid,
431                                                theSourceConceptDirectParentPids,
432                                                theSystemVersion);
433                                return true;
434                        }
435
436                        if (!theAdd && theAddedCodes.remove(theCodeSystem + OUR_PIPE_CHARACTER + theCode)) {
437                                theValueSetCodeAccumulator.excludeConcept(theCodeSystem, theCode);
438                                return true;
439                        }
440                }
441
442                return false;
443        }
444
445        private boolean addToSet(Set<TermConcept> theSetToPopulate, TermConcept theConcept) {
446                boolean retVal = theSetToPopulate.add(theConcept);
447                if (retVal) {
448                        if (theSetToPopulate.size() >= myStorageSettings.getMaximumExpansionSize()) {
449                                String msg = myContext
450                                                .getLocalizer()
451                                                .getMessage(
452                                                                TermReadSvcImpl.class,
453                                                                "expansionTooLarge",
454                                                                myStorageSettings.getMaximumExpansionSize());
455                                throw new ExpansionTooCostlyException(Msg.code(885) + msg);
456                        }
457                }
458                return retVal;
459        }
460
461        public Optional<TermValueSet> deleteValueSetForResource(ResourceTable theResourceTable) {
462                // Get existing entity so it can be deleted.
463                Optional<TermValueSet> optionalExistingTermValueSetById =
464                                myTermValueSetDao.findByResourcePid(theResourceTable.getId());
465
466                if (optionalExistingTermValueSetById.isPresent()) {
467                        TermValueSet existingTermValueSet = optionalExistingTermValueSetById.get();
468
469                        ourLog.info("Deleting existing TermValueSet[{}] and its children...", existingTermValueSet.getId());
470                        deletePreCalculatedValueSetContents(existingTermValueSet);
471                        myTermValueSetDao.deleteById(existingTermValueSet.getId());
472
473                        /*
474                         * If we're updating an existing ValueSet within a transaction, we need to make
475                         * sure to manually flush now since otherwise we'll try to create a new
476                         * TermValueSet entity and fail with a constraint error on the URL, since
477                         * this one won't be deleted yet
478                         */
479                        myTermValueSetDao.flush();
480
481                        ourLog.info("Done deleting existing TermValueSet[{}] and its children.", existingTermValueSet.getId());
482                }
483
484                return optionalExistingTermValueSetById;
485        }
486
487        private void deletePreCalculatedValueSetContents(TermValueSet theValueSet) {
488                myValueSetConceptDesignationDao.deleteByTermValueSetId(theValueSet.getId());
489                myValueSetConceptDao.deleteByTermValueSetId(theValueSet.getId());
490        }
491
492        @Override
493        @Transactional
494        public void deleteValueSetAndChildren(ResourceTable theResourceTable) {
495                deleteValueSetForResource(theResourceTable);
496        }
497
498        @Override
499        @Transactional
500        public List<FhirVersionIndependentConcept> expandValueSetIntoConceptList(
501                        @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) {
502                // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not be found
503                // in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the expansion may
504                // time-out.
505
506                ValueSet expanded = expandValueSet(theExpansionOptions, theValueSetCanonicalUrl);
507
508                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
509                for (ValueSet.ValueSetExpansionContainsComponent nextContains :
510                                expanded.getExpansion().getContains()) {
511                        retVal.add(new FhirVersionIndependentConcept(
512                                        nextContains.getSystem(),
513                                        nextContains.getCode(),
514                                        nextContains.getDisplay(),
515                                        nextContains.getVersion()));
516                }
517                return retVal;
518        }
519
520        @Override
521        public ValueSet expandValueSet(
522                        @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull String theValueSetCanonicalUrl) {
523                ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(theValueSetCanonicalUrl);
524                if (valueSet == null) {
525                        throw new ResourceNotFoundException(
526                                        Msg.code(886) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(theValueSetCanonicalUrl));
527                }
528
529                return expandValueSet(theExpansionOptions, valueSet);
530        }
531
532        @Override
533        public ValueSet expandValueSet(
534                        @Nullable ValueSetExpansionOptions theExpansionOptions, @Nonnull ValueSet theValueSetToExpand) {
535                String filter = null;
536                if (theExpansionOptions != null) {
537                        filter = theExpansionOptions.getFilter();
538                }
539                return doExpandValueSet(theExpansionOptions, theValueSetToExpand, ExpansionFilter.fromFilterString(filter));
540        }
541
542        private ValueSet doExpandValueSet(
543                        @Nullable ValueSetExpansionOptions theExpansionOptions,
544                        ValueSet theValueSetToExpand,
545                        ExpansionFilter theFilter) {
546                Set<String> addedCodes = new HashSet<>();
547                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSetToExpand, "ValueSet to expand can not be null");
548
549                ValueSetExpansionOptions expansionOptions = provideExpansionOptions(theExpansionOptions);
550                int offset = expansionOptions.getOffset();
551                int count = expansionOptions.getCount();
552
553                ValueSetExpansionComponentWithConceptAccumulator accumulator =
554                                new ValueSetExpansionComponentWithConceptAccumulator(
555                                                myContext, count, expansionOptions.isIncludeHierarchy());
556                accumulator.setHardExpansionMaximumSize(myStorageSettings.getMaximumExpansionSize());
557                accumulator.setSkipCountRemaining(offset);
558                accumulator.setIdentifier(UUID.randomUUID().toString());
559                accumulator.setTimestamp(new Date());
560                accumulator.setOffset(offset);
561
562                if (theExpansionOptions != null && isHibernateSearchEnabled()) {
563                        accumulator.addParameter().setName("offset").setValue(new IntegerType(offset));
564                        accumulator.addParameter().setName("count").setValue(new IntegerType(count));
565                }
566
567                myTxTemplate.executeWithoutResult(tx -> expandValueSetIntoAccumulator(
568                                theValueSetToExpand, theExpansionOptions, accumulator, theFilter, true, addedCodes));
569
570                if (accumulator.getTotalConcepts() != null) {
571                        accumulator.setTotal(accumulator.getTotalConcepts());
572                }
573
574                ValueSet valueSet = new ValueSet();
575                valueSet.setUrl(theValueSetToExpand.getUrl());
576                valueSet.setId(theValueSetToExpand.getId());
577                valueSet.setStatus(Enumerations.PublicationStatus.ACTIVE);
578                valueSet.setCompose(theValueSetToExpand.getCompose());
579                valueSet.setExpansion(accumulator);
580
581                for (String next : accumulator.getMessages()) {
582                        valueSet.getMeta()
583                                        .addExtension()
584                                        .setUrl(HapiExtensions.EXT_VALUESET_EXPANSION_MESSAGE)
585                                        .setValue(new StringType(next));
586                }
587
588                if (expansionOptions.isIncludeHierarchy()) {
589                        accumulator.applyHierarchy();
590                }
591
592                return valueSet;
593        }
594
595        private void expandValueSetIntoAccumulator(
596                        ValueSet theValueSetToExpand,
597                        ValueSetExpansionOptions theExpansionOptions,
598                        IValueSetConceptAccumulator theAccumulator,
599                        ExpansionFilter theFilter,
600                        boolean theAdd,
601                        Set<String> theAddedCodes) {
602                Optional<TermValueSet> optionalTermValueSet;
603                if (theValueSetToExpand.hasUrl()) {
604                        if (theValueSetToExpand.hasVersion()) {
605                                optionalTermValueSet = myTermValueSetDao.findTermValueSetByUrlAndVersion(
606                                                theValueSetToExpand.getUrl(), theValueSetToExpand.getVersion());
607                        } else {
608                                optionalTermValueSet = findCurrentTermValueSet(theValueSetToExpand.getUrl());
609                        }
610                } else {
611                        optionalTermValueSet = Optional.empty();
612                }
613
614                /*
615                 * ValueSet doesn't exist in pre-expansion database, so perform in-memory expansion
616                 */
617                if (optionalTermValueSet.isEmpty()) {
618                        ourLog.debug(
619                                        "ValueSet is not present in terminology tables. Will perform in-memory expansion without parameters. {}",
620                                        getValueSetInfo(theValueSetToExpand));
621                        String msg = myContext
622                                        .getLocalizer()
623                                        .getMessage(
624                                                        TermReadSvcImpl.class,
625                                                        "valueSetExpandedUsingInMemoryExpansion",
626                                                        getValueSetInfo(theValueSetToExpand));
627                        theAccumulator.addMessage(msg);
628                        doExpandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter, theAddedCodes);
629                        return;
630                }
631
632                /*
633                 * ValueSet exists in pre-expansion database, but pre-expansion is not yet complete so perform in-memory expansion
634                 */
635                TermValueSet termValueSet = optionalTermValueSet.get();
636                if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
637                        String msg = myContext
638                                        .getLocalizer()
639                                        .getMessage(
640                                                        TermReadSvcImpl.class,
641                                                        "valueSetNotYetExpanded",
642                                                        getValueSetInfo(theValueSetToExpand),
643                                                        termValueSet.getExpansionStatus().name(),
644                                                        termValueSet.getExpansionStatus().getDescription());
645                        theAccumulator.addMessage(msg);
646                        doExpandValueSet(theExpansionOptions, theValueSetToExpand, theAccumulator, theFilter, theAddedCodes);
647                        return;
648                }
649
650                /*
651                 * ValueSet is pre-expanded in database so let's use that
652                 */
653                String expansionTimestamp = toHumanReadableExpansionTimestamp(termValueSet);
654                String msg = myContext
655                                .getLocalizer()
656                                .getMessage(TermReadSvcImpl.class, "valueSetExpandedUsingPreExpansion", expansionTimestamp);
657                theAccumulator.addMessage(msg);
658                expandConcepts(
659                                theExpansionOptions,
660                                theAccumulator,
661                                termValueSet,
662                                theFilter,
663                                theAdd,
664                                theAddedCodes,
665                                myHibernatePropertiesProvider.isOracleDialect());
666        }
667
668        @Nonnull
669        private String toHumanReadableExpansionTimestamp(TermValueSet termValueSet) {
670                String expansionTimestamp = "(unknown)";
671                if (termValueSet.getExpansionTimestamp() != null) {
672                        // Note: We used to append "123ms ago" to the timestamp, but we cache the
673                        // results here, so it's just kind of weird to do that since the duration will
674                        // be out of date when the entry comes back from cache
675                        expansionTimestamp = new InstantType(termValueSet.getExpansionTimestamp()).getValueAsString();
676                }
677                return expansionTimestamp;
678        }
679
680        private void expandConcepts(
681                        ValueSetExpansionOptions theExpansionOptions,
682                        IValueSetConceptAccumulator theAccumulator,
683                        TermValueSet theTermValueSet,
684                        ExpansionFilter theFilter,
685                        boolean theAdd,
686                        Set<String> theAddedCodes,
687                        boolean theOracle) {
688                // NOTE: if you modifiy the logic here, look to `expandConceptsOracle` and see if your new code applies to its
689                // copy pasted sibling
690                Integer offset = theAccumulator.getSkipCountRemaining();
691                offset = ObjectUtils.defaultIfNull(offset, 0);
692                offset = Math.min(offset, theTermValueSet.getTotalConcepts().intValue());
693
694                Integer count = theAccumulator.getCapacityRemaining();
695                count = defaultIfNull(count, myStorageSettings.getMaximumExpansionSize());
696
697                int conceptsExpanded = 0;
698                int designationsExpanded = 0;
699                int toIndex = offset + count;
700
701                Collection<? extends ITermValueSetConceptView> conceptViews;
702                boolean wasFilteredResult = false;
703                String filterDisplayValue = null;
704                if (!theFilter.getFilters().isEmpty()
705                                && JpaConstants.VALUESET_FILTER_DISPLAY.equals(
706                                                theFilter.getFilters().get(0).getProperty())
707                                && theFilter.getFilters().get(0).getOp() == ValueSet.FilterOperator.EQUAL) {
708                        filterDisplayValue =
709                                        lowerCase(theFilter.getFilters().get(0).getValue().replace("%", "[%]"));
710                        String displayValue = "%" + lowerCase(filterDisplayValue) + "%";
711                        if (theOracle) {
712                                conceptViews =
713                                                myTermValueSetConceptViewOracleDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
714                        } else {
715                                conceptViews = myTermValueSetConceptViewDao.findByTermValueSetId(theTermValueSet.getId(), displayValue);
716                        }
717                        wasFilteredResult = true;
718                } else {
719                        if (theOracle) {
720                                conceptViews = myTermValueSetConceptViewOracleDao.findByTermValueSetId(
721                                                offset, toIndex, theTermValueSet.getId());
722                        } else {
723                                conceptViews =
724                                                myTermValueSetConceptViewDao.findByTermValueSetId(offset, toIndex, theTermValueSet.getId());
725                        }
726                        theAccumulator.consumeSkipCount(offset);
727                        if (theAdd) {
728                                theAccumulator.incrementOrDecrementTotalConcepts(
729                                                true, theTermValueSet.getTotalConcepts().intValue());
730                        }
731                }
732
733                if (conceptViews.isEmpty()) {
734                        logConceptsExpanded("No concepts to expand. ", theTermValueSet, conceptsExpanded);
735                        return;
736                }
737
738                Map<Long, FhirVersionIndependentConcept> pidToConcept = new LinkedHashMap<>();
739                ArrayListMultimap<Long, TermConceptDesignation> pidToDesignations = ArrayListMultimap.create();
740                Map<Long, Long> pidToSourcePid = new HashMap<>();
741                Map<Long, String> pidToSourceDirectParentPids = new HashMap<>();
742
743                for (ITermValueSetConceptView conceptView : conceptViews) {
744
745                        String system = conceptView.getConceptSystemUrl();
746                        String code = conceptView.getConceptCode();
747                        String display = conceptView.getConceptDisplay();
748                        String systemVersion = conceptView.getConceptSystemVersion();
749
750                        // -- this is quick solution, may need to revisit
751                        if (!applyFilter(display, filterDisplayValue)) {
752                                continue;
753                        }
754
755                        Long conceptPid = conceptView.getConceptPid();
756                        if (!pidToConcept.containsKey(conceptPid)) {
757                                FhirVersionIndependentConcept concept =
758                                                new FhirVersionIndependentConcept(system, code, display, systemVersion);
759                                pidToConcept.put(conceptPid, concept);
760                        }
761
762                        // TODO: DM 2019-08-17 - Implement includeDesignations parameter for $expand operation to designations
763                        // optional.
764                        if (conceptView.getDesignationPid() != null) {
765                                TermConceptDesignation designation = new TermConceptDesignation();
766
767                                if (isValueSetDisplayLanguageMatch(theExpansionOptions, conceptView.getDesignationLang())) {
768                                        designation.setUseSystem(conceptView.getDesignationUseSystem());
769                                        designation.setUseCode(conceptView.getDesignationUseCode());
770                                        designation.setUseDisplay(conceptView.getDesignationUseDisplay());
771                                        designation.setValue(conceptView.getDesignationVal());
772                                        designation.setLanguage(conceptView.getDesignationLang());
773                                        pidToDesignations.put(conceptPid, designation);
774                                }
775
776                                if (++designationsExpanded % 250 == 0) {
777                                        logDesignationsExpanded(
778                                                        "Expansion of designations in progress. ", theTermValueSet, designationsExpanded);
779                                }
780                        }
781
782                        if (theAccumulator.isTrackingHierarchy()) {
783                                pidToSourcePid.put(conceptPid, conceptView.getSourceConceptPid());
784                                pidToSourceDirectParentPids.put(conceptPid, conceptView.getSourceConceptDirectParentPids());
785                        }
786
787                        if (++conceptsExpanded % 250 == 0) {
788                                logConceptsExpanded("Expansion of concepts in progress. ", theTermValueSet, conceptsExpanded);
789                        }
790                }
791
792                for (Long nextPid : pidToConcept.keySet()) {
793                        FhirVersionIndependentConcept concept = pidToConcept.get(nextPid);
794                        List<TermConceptDesignation> designations = pidToDesignations.get(nextPid);
795                        String system = concept.getSystem();
796                        String code = concept.getCode();
797                        String display = concept.getDisplay();
798                        String systemVersion = concept.getSystemVersion();
799
800                        if (theAdd) {
801                                if (theAccumulator.getCapacityRemaining() != null) {
802                                        if (theAccumulator.getCapacityRemaining() == 0) {
803                                                break;
804                                        }
805                                }
806
807                                Long sourceConceptPid = pidToSourcePid.get(nextPid);
808                                String sourceConceptDirectParentPids = pidToSourceDirectParentPids.get(nextPid);
809                                if (theAddedCodes.add(system + OUR_PIPE_CHARACTER + code)) {
810                                        theAccumulator.includeConceptWithDesignations(
811                                                        system,
812                                                        code,
813                                                        display,
814                                                        designations,
815                                                        sourceConceptPid,
816                                                        sourceConceptDirectParentPids,
817                                                        systemVersion);
818                                        if (wasFilteredResult) {
819                                                theAccumulator.incrementOrDecrementTotalConcepts(true, 1);
820                                        }
821                                }
822                        } else {
823                                if (theAddedCodes.remove(system + OUR_PIPE_CHARACTER + code)) {
824                                        theAccumulator.excludeConcept(system, code);
825                                        theAccumulator.incrementOrDecrementTotalConcepts(false, 1);
826                                }
827                        }
828                }
829
830                logDesignationsExpanded("Finished expanding designations. ", theTermValueSet, designationsExpanded);
831                logConceptsExpanded("Finished expanding concepts. ", theTermValueSet, conceptsExpanded);
832        }
833
834        private void logConceptsExpanded(
835                        String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theConceptsExpanded) {
836                if (theConceptsExpanded > 0) {
837                        ourLog.debug(
838                                        "{}Have expanded {} concepts in ValueSet[{}]",
839                                        theLogDescriptionPrefix,
840                                        theConceptsExpanded,
841                                        theTermValueSet.getUrl());
842                }
843        }
844
845        private void logDesignationsExpanded(
846                        String theLogDescriptionPrefix, TermValueSet theTermValueSet, int theDesignationsExpanded) {
847                if (theDesignationsExpanded > 0) {
848                        ourLog.debug(
849                                        "{}Have expanded {} designations in ValueSet[{}]",
850                                        theLogDescriptionPrefix,
851                                        theDesignationsExpanded,
852                                        theTermValueSet.getUrl());
853                }
854        }
855
856        public boolean applyFilter(final String theDisplay, final String theFilterDisplay) {
857
858                // -- safety check only, no need to apply filter
859                if (theDisplay == null || theFilterDisplay == null) return true;
860
861                // -- sentence case
862                if (startsWithIgnoreCase(theDisplay, theFilterDisplay)) return true;
863
864                // -- token case
865                return startsWithByWordBoundaries(theDisplay, theFilterDisplay);
866        }
867
868        private boolean startsWithByWordBoundaries(String theDisplay, String theFilterDisplay) {
869                // return true only e.g. the input is 'Body height', theFilterDisplay is "he", or 'bo'
870                StringTokenizer tok = new StringTokenizer(theDisplay);
871                List<String> tokens = new ArrayList<>();
872                while (tok.hasMoreTokens()) {
873                        String token = tok.nextToken();
874                        if (startsWithIgnoreCase(token, theFilterDisplay)) return true;
875                        tokens.add(token);
876                }
877
878                // Allow to search by the end of the phrase.  E.g.  "working proficiency" will match "Limited working
879                // proficiency"
880                for (int start = 0; start <= tokens.size() - 1; ++start) {
881                        for (int end = start + 1; end <= tokens.size(); ++end) {
882                                String sublist = String.join(" ", tokens.subList(start, end));
883                                if (startsWithIgnoreCase(sublist, theFilterDisplay)) return true;
884                        }
885                }
886                return false;
887        }
888
889        @Override
890        public void expandValueSet(
891                        ValueSetExpansionOptions theExpansionOptions,
892                        ValueSet theValueSetToExpand,
893                        IValueSetConceptAccumulator theValueSetCodeAccumulator) {
894                Set<String> addedCodes = new HashSet<>();
895                doExpandValueSet(
896                                theExpansionOptions,
897                                theValueSetToExpand,
898                                theValueSetCodeAccumulator,
899                                ExpansionFilter.NO_FILTER,
900                                addedCodes);
901        }
902
903        /**
904         * Note: Not transactional because specific calls within this method
905         * get executed in a transaction
906         */
907        private void doExpandValueSet(
908                        ValueSetExpansionOptions theExpansionOptions,
909                        ValueSet theValueSetToExpand,
910                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
911                        @Nonnull ExpansionFilter theExpansionFilter,
912                        Set<String> theAddedCodes) {
913
914                StopWatch sw = new StopWatch();
915                String valueSetInfo = getValueSetInfo(theValueSetToExpand);
916                ourLog.debug("Working with {}", valueSetInfo);
917
918                // Offset can't be combined with excludes
919                Integer skipCountRemaining = theValueSetCodeAccumulator.getSkipCountRemaining();
920                if (skipCountRemaining != null && skipCountRemaining > 0) {
921                        if (!theValueSetToExpand.getCompose().getExclude().isEmpty()) {
922                                String msg = myContext
923                                                .getLocalizer()
924                                                .getMessage(TermReadSvcImpl.class, "valueSetNotYetExpanded_OffsetNotAllowed", valueSetInfo);
925                                throw new InvalidRequestException(Msg.code(887) + msg);
926                        }
927                }
928
929                // Handle includes
930                ourLog.debug("Handling includes");
931                for (ValueSet.ConceptSetComponent include :
932                                theValueSetToExpand.getCompose().getInclude()) {
933                        myTxTemplate.executeWithoutResult(tx -> expandValueSetHandleIncludeOrExclude(
934                                        theExpansionOptions, theValueSetCodeAccumulator, theAddedCodes, include, true, theExpansionFilter));
935                }
936
937                // Handle excludes
938                ourLog.debug("Handling excludes");
939                for (ValueSet.ConceptSetComponent exclude :
940                                theValueSetToExpand.getCompose().getExclude()) {
941                        myTxTemplate.executeWithoutResult(tx -> expandValueSetHandleIncludeOrExclude(
942                                        theExpansionOptions,
943                                        theValueSetCodeAccumulator,
944                                        theAddedCodes,
945                                        exclude,
946                                        false,
947                                        ExpansionFilter.NO_FILTER));
948                }
949
950                if (theValueSetCodeAccumulator instanceof ValueSetConceptAccumulator) {
951                        myTxTemplate.execute(
952                                        t -> ((ValueSetConceptAccumulator) theValueSetCodeAccumulator).removeGapsFromConceptOrder());
953                }
954
955                ourLog.debug("Done working with {} in {}ms", valueSetInfo, sw.getMillis());
956        }
957
958        private String getValueSetInfo(ValueSet theValueSet) {
959                StringBuilder sb = new StringBuilder();
960                boolean isIdentified = false;
961                if (theValueSet.hasUrl()) {
962                        isIdentified = true;
963                        sb.append("ValueSet.url[").append(theValueSet.getUrl()).append("]");
964                } else if (theValueSet.hasId()) {
965                        isIdentified = true;
966                        sb.append("ValueSet.id[").append(theValueSet.getId()).append("]");
967                }
968
969                if (!isIdentified) {
970                        sb.append("Unidentified ValueSet");
971                }
972
973                return sb.toString();
974        }
975
976        /**
977         * Returns true if there are potentially more results to process.
978         */
979        private void expandValueSetHandleIncludeOrExclude(
980                        @Nullable ValueSetExpansionOptions theExpansionOptions,
981                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
982                        Set<String> theAddedCodes,
983                        ValueSet.ConceptSetComponent theIncludeOrExclude,
984                        boolean theAdd,
985                        @Nonnull ExpansionFilter theExpansionFilter) {
986
987                String system = theIncludeOrExclude.getSystem();
988                boolean hasSystem = isNotBlank(system);
989                boolean hasValueSet = !theIncludeOrExclude.getValueSet().isEmpty();
990
991                if (hasSystem) {
992
993                        if (theExpansionFilter.hasCode()
994                                        && theExpansionFilter.getSystem() != null
995                                        && !system.equals(theExpansionFilter.getSystem())) {
996                                return;
997                        }
998
999                        ourLog.debug("Starting {} expansion around CodeSystem: {}", (theAdd ? "inclusion" : "exclusion"), system);
1000
1001                        Optional<TermCodeSystemVersion> termCodeSystemVersion =
1002                                        optionalFindTermCodeSystemVersion(theIncludeOrExclude);
1003                        if (termCodeSystemVersion.isPresent()) {
1004
1005                                expandValueSetHandleIncludeOrExcludeUsingDatabase(
1006                                                theExpansionOptions,
1007                                                theValueSetCodeAccumulator,
1008                                                theAddedCodes,
1009                                                theIncludeOrExclude,
1010                                                theAdd,
1011                                                theExpansionFilter,
1012                                                system,
1013                                                termCodeSystemVersion.get());
1014
1015                        } else {
1016
1017                                if (!theIncludeOrExclude.getConcept().isEmpty() && theExpansionFilter.hasCode()) {
1018                                        if (defaultString(theIncludeOrExclude.getSystem()).equals(theExpansionFilter.getSystem())) {
1019                                                if (theIncludeOrExclude.getConcept().stream()
1020                                                                .noneMatch(t -> t.getCode().equals(theExpansionFilter.getCode()))) {
1021                                                        return;
1022                                                }
1023                                        }
1024                                }
1025
1026                                Consumer<FhirVersionIndependentConcept> consumer = c -> addOrRemoveCode(
1027                                                theValueSetCodeAccumulator,
1028                                                theAddedCodes,
1029                                                theAdd,
1030                                                system,
1031                                                c.getCode(),
1032                                                c.getDisplay(),
1033                                                c.getSystemVersion());
1034
1035                                try {
1036                                        ConversionContext40_50.INSTANCE.init(
1037                                                        new VersionConvertor_40_50(new BaseAdvisor_40_50()), "ValueSet");
1038                                        org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent includeOrExclude =
1039                                                        ValueSet40_50.convertConceptSetComponent(theIncludeOrExclude);
1040                                        myInMemoryTerminologyServerValidationSupport.expandValueSetIncludeOrExclude(
1041                                                        new ValidationSupportContext(provideValidationSupport()), consumer, includeOrExclude);
1042                                } catch (InMemoryTerminologyServerValidationSupport.ExpansionCouldNotBeCompletedInternallyException e) {
1043                                        if (theExpansionOptions != null
1044                                                        && !theExpansionOptions.isFailOnMissingCodeSystem()
1045                                                        // Code system is unknown, therefore NOT_FOUND
1046                                                        && e.getCodeValidationIssue()
1047                                                                        .hasIssueDetailCode(CodeValidationIssueCoding.NOT_FOUND.getCode())) {
1048                                                return;
1049                                        }
1050                                        throw new InternalErrorException(Msg.code(888) + e);
1051                                } finally {
1052                                        ConversionContext40_50.INSTANCE.close("ValueSet");
1053                                }
1054                        }
1055
1056                } else if (hasValueSet) {
1057
1058                        for (CanonicalType nextValueSet : theIncludeOrExclude.getValueSet()) {
1059                                String valueSetUrl = nextValueSet.getValueAsString();
1060                                ourLog.debug(
1061                                                "Starting {} expansion around ValueSet: {}", (theAdd ? "inclusion" : "exclusion"), valueSetUrl);
1062
1063                                ExpansionFilter subExpansionFilter = new ExpansionFilter(
1064                                                theExpansionFilter,
1065                                                theIncludeOrExclude.getFilter(),
1066                                                theValueSetCodeAccumulator.getCapacityRemaining());
1067
1068                                // TODO: DM 2019-09-10 - This is problematic because an incorrect URL that matches ValueSet.id will not
1069                                // be found in the terminology tables but will yield a ValueSet here. Depending on the ValueSet, the
1070                                // expansion may time-out.
1071
1072                                ValueSet valueSet = fetchCanonicalValueSetFromCompleteContext(valueSetUrl);
1073                                if (valueSet == null) {
1074                                        throw new ResourceNotFoundException(
1075                                                        Msg.code(889) + "Unknown ValueSet: " + UrlUtil.escapeUrlParam(valueSetUrl));
1076                                }
1077
1078                                expandValueSetIntoAccumulator(
1079                                                valueSet,
1080                                                theExpansionOptions,
1081                                                theValueSetCodeAccumulator,
1082                                                subExpansionFilter,
1083                                                theAdd,
1084                                                theAddedCodes);
1085                        }
1086
1087                } else {
1088                        throw new InvalidRequestException(Msg.code(890) + "ValueSet contains " + (theAdd ? "include" : "exclude")
1089                                        + " criteria with no system defined");
1090                }
1091        }
1092
1093        private Optional<TermCodeSystemVersion> optionalFindTermCodeSystemVersion(
1094                        ValueSet.ConceptSetComponent theIncludeOrExclude) {
1095                if (isEmpty(theIncludeOrExclude.getVersion())) {
1096                        return Optional.ofNullable(myCodeSystemDao.findByCodeSystemUri(theIncludeOrExclude.getSystem()))
1097                                        .map(TermCodeSystem::getCurrentVersion);
1098                } else {
1099                        return Optional.ofNullable(myCodeSystemVersionDao.findByCodeSystemUriAndVersion(
1100                                        theIncludeOrExclude.getSystem(), theIncludeOrExclude.getVersion()));
1101                }
1102        }
1103
1104        private boolean isHibernateSearchEnabled() {
1105                return myFulltextSearchSvc != null && !ourForceDisableHibernateSearchForUnitTest;
1106        }
1107
1108        private void expandValueSetHandleIncludeOrExcludeUsingDatabase(
1109                        ValueSetExpansionOptions theExpansionOptions,
1110                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
1111                        Set<String> theAddedCodes,
1112                        ValueSet.ConceptSetComponent theIncludeOrExclude,
1113                        boolean theAdd,
1114                        @Nonnull ExpansionFilter theExpansionFilter,
1115                        String theSystem,
1116                        TermCodeSystemVersion theTermCodeSystemVersion) {
1117
1118                StopWatch fullOperationSw = new StopWatch();
1119                String includeOrExcludeVersion = theIncludeOrExclude.getVersion();
1120
1121                /*
1122                 * If FullText searching is not enabled, we can handle only basic expansions
1123                 * since we're going to do it without the database.
1124                 */
1125                if (!isHibernateSearchEnabled()) {
1126                        expandWithoutHibernateSearch(
1127                                        theValueSetCodeAccumulator,
1128                                        theTermCodeSystemVersion,
1129                                        theAddedCodes,
1130                                        theIncludeOrExclude,
1131                                        theSystem,
1132                                        theAdd);
1133                        return;
1134                }
1135
1136                /*
1137                 * Ok, let's use hibernate search to build the expansion
1138                 */
1139
1140                int count = 0;
1141
1142                Optional<Integer> chunkSizeOpt = getScrollChunkSize(theAdd, theValueSetCodeAccumulator);
1143                if (chunkSizeOpt.isEmpty()) {
1144                        return;
1145                }
1146                int chunkSize = chunkSizeOpt.get();
1147
1148                /*
1149                 * Turn the filter into one or more Hibernate Search queries. Ideally we want it
1150                 * to be handled by a single query, but Lucene/ES don't like it when we exceed
1151                 * 1024 different terms in a single query. So if we have that many terms (which
1152                 * can happen if a ValueSet has a lot of explicitly enumerated codes that it's
1153                 * including) we split this into multiple searches. The method below builds these
1154                 * searches lazily, returning a Supplier that creates and executes the search
1155                 * when it's actually time to.
1156                 */
1157                SearchProperties searchProps = buildSearchScrolls(
1158                                theTermCodeSystemVersion,
1159                                theExpansionFilter,
1160                                theSystem,
1161                                theIncludeOrExclude,
1162                                chunkSize,
1163                                includeOrExcludeVersion);
1164                int accumulatedBatchesSoFar = 0;
1165                for (var next : searchProps.getSearchScroll()) {
1166                        try (SearchScroll<EntityReference> scroll = next.get()) {
1167                                ourLog.debug(
1168                                                "Beginning batch expansion for {} with max results per batch: {}",
1169                                                (theAdd ? "inclusion" : "exclusion"),
1170                                                chunkSize);
1171                                for (SearchScrollResult<EntityReference> chunk = scroll.next();
1172                                                chunk.hasHits();
1173                                                chunk = scroll.next()) {
1174                                        int countForBatch = 0;
1175
1176                                        List<Long> pids =
1177                                                        chunk.hits().stream().map(t -> (Long) t.id()).collect(Collectors.toList());
1178
1179                                        List<TermConcept> termConcepts = myTermConceptDao.fetchConceptsAndDesignationsByPid(pids);
1180
1181                                        // If the include section had multiple codes, return the codes in the same order
1182                                        termConcepts = sortTermConcepts(searchProps, termConcepts);
1183
1184                                        //       int firstResult = theQueryIndex * maxResultsPerBatch;
1185                                        // TODO GGG HS we lose the ability to check the
1186                                        // index of the first result, so just best-guessing it here.
1187                                        int delta = 0;
1188                                        for (TermConcept concept : termConcepts) {
1189                                                count++;
1190                                                countForBatch++;
1191                                                if (theAdd && searchProps.hasIncludeOrExcludeCodes()) {
1192                                                        ValueSet.ConceptReferenceComponent theIncludeConcept =
1193                                                                        getMatchedConceptIncludedInValueSet(theIncludeOrExclude, concept);
1194                                                        if (theIncludeConcept != null && isNotBlank(theIncludeConcept.getDisplay())) {
1195                                                                concept.setDisplay(theIncludeConcept.getDisplay());
1196                                                        }
1197                                                }
1198                                                boolean added = addCodeIfNotAlreadyAdded(
1199                                                                theExpansionOptions,
1200                                                                theValueSetCodeAccumulator,
1201                                                                theAddedCodes,
1202                                                                concept,
1203                                                                theAdd,
1204                                                                includeOrExcludeVersion);
1205                                                if (added) {
1206                                                        delta++;
1207                                                }
1208                                        }
1209
1210                                        ourLog.debug(
1211                                                        "Batch expansion scroll for {} with offset {} produced {} results in {}ms",
1212                                                        (theAdd ? "inclusion" : "exclusion"),
1213                                                        accumulatedBatchesSoFar,
1214                                                        chunk.hits().size(),
1215                                                        chunk.took().toMillis());
1216
1217                                        theValueSetCodeAccumulator.incrementOrDecrementTotalConcepts(theAdd, delta);
1218                                        accumulatedBatchesSoFar += countForBatch;
1219
1220                                        // keep session bounded
1221                                        myEntityManager.flush();
1222                                        myEntityManager.clear();
1223                                }
1224
1225                                ourLog.debug(
1226                                                "Expansion for {} produced {} results in {}ms",
1227                                                (theAdd ? "inclusion" : "exclusion"),
1228                                                count,
1229                                                fullOperationSw.getMillis());
1230                        }
1231                }
1232        }
1233
1234        private List<TermConcept> sortTermConcepts(SearchProperties searchProps, List<TermConcept> termConcepts) {
1235                List<String> codes = searchProps.getIncludeOrExcludeCodes();
1236                if (codes.size() > 1) {
1237                        termConcepts = new ArrayList<>(termConcepts);
1238                        Map<String, Integer> codeToIndex = new HashMap<>(codes.size());
1239                        for (int i = 0; i < codes.size(); i++) {
1240                                codeToIndex.put(codes.get(i), i);
1241                        }
1242                        termConcepts.sort(((o1, o2) -> {
1243                                Integer idx1 = codeToIndex.get(o1.getCode());
1244                                Integer idx2 = codeToIndex.get(o2.getCode());
1245                                return Comparators.nullsHigh().compare(idx1, idx2);
1246                        }));
1247                }
1248                return termConcepts;
1249        }
1250
1251        private Optional<Integer> getScrollChunkSize(
1252                        boolean theAdd, IValueSetConceptAccumulator theValueSetCodeAccumulator) {
1253                int maxResultsPerBatch = SearchBuilder.getMaximumPageSize();
1254
1255                /*
1256                 * If the accumulator is bounded, we may reduce the size of the query to
1257                 * Lucene in order to be more efficient.
1258                 */
1259                if (theAdd) {
1260                        Integer accumulatorCapacityRemaining = theValueSetCodeAccumulator.getCapacityRemaining();
1261                        if (accumulatorCapacityRemaining != null) {
1262                                maxResultsPerBatch = Math.min(maxResultsPerBatch, accumulatorCapacityRemaining + 1);
1263                        }
1264                }
1265                return maxResultsPerBatch > 0 ? Optional.of(maxResultsPerBatch) : Optional.empty();
1266        }
1267
1268        private SearchProperties buildSearchScrolls(
1269                        TermCodeSystemVersion theTermCodeSystemVersion,
1270                        ExpansionFilter theExpansionFilter,
1271                        String theSystem,
1272                        ValueSet.ConceptSetComponent theIncludeOrExclude,
1273                        Integer theScrollChunkSize,
1274                        String theIncludeOrExcludeVersion) {
1275                SearchSession searchSession = Search.session(myEntityManager);
1276                // Manually building a predicate since we need to throw it around.
1277                SearchPredicateFactory predicate =
1278                                searchSession.scope(TermConcept.class).predicate();
1279
1280                List<String> allCodes = theIncludeOrExclude.getConcept().stream()
1281                                .filter(Objects::nonNull)
1282                                .map(ValueSet.ConceptReferenceComponent::getCode)
1283                                .filter(StringUtils::isNotBlank)
1284                                .collect(Collectors.toList());
1285                SearchProperties returnProps = new SearchProperties();
1286                returnProps.setIncludeOrExcludeCodes(allCodes);
1287
1288                /*
1289                 * Lucene/ES can't typically handle more than 1024 clauses per search, so if
1290                 * we have more than that number (e.g. because of a ValueSet that explicitly
1291                 * includes thousands of codes), we break this up into multiple searches.
1292                 */
1293                List<List<String>> partitionedCodes = ListUtils.partition(allCodes, IndexSearcher.getMaxClauseCount() - 10);
1294                if (partitionedCodes.isEmpty()) {
1295                        partitionedCodes = List.of(List.of());
1296                }
1297
1298                for (List<String> nextCodePartition : partitionedCodes) {
1299                        Supplier<SearchScroll<EntityReference>> nextScroll = () -> {
1300                                // Build the top-level expansion on filters.
1301                                PredicateFinalStep step = predicate.bool(b -> {
1302                                        b.must(predicate
1303                                                        .match()
1304                                                        .field("myCodeSystemVersionPid")
1305                                                        .matching(theTermCodeSystemVersion.getPid()));
1306
1307                                        if (theExpansionFilter.hasCode()) {
1308                                                b.must(predicate.match().field("myCode").matching(theExpansionFilter.getCode()));
1309                                        }
1310
1311                                        String codeSystemUrlAndVersion =
1312                                                        buildCodeSystemUrlAndVersion(theSystem, theIncludeOrExcludeVersion);
1313                                        for (ValueSet.ConceptSetFilterComponent nextFilter : theIncludeOrExclude.getFilter()) {
1314                                                handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter);
1315                                        }
1316                                        for (ValueSet.ConceptSetFilterComponent nextFilter : theExpansionFilter.getFilters()) {
1317                                                handleFilter(codeSystemUrlAndVersion, predicate, b, nextFilter);
1318                                        }
1319                                });
1320
1321                                // Add a selector on any explicitly enumerated codes in the VS component
1322                                final PredicateFinalStep finishedQuery;
1323                                if (nextCodePartition.isEmpty()) {
1324                                        finishedQuery = step;
1325                                } else {
1326                                        PredicateFinalStep expansionStep = buildExpansionPredicate(nextCodePartition, predicate);
1327                                        finishedQuery = predicate.bool().must(step).must(expansionStep);
1328                                }
1329
1330                                SearchQuery<EntityReference> termConceptsQuery = searchSession
1331                                                .search(TermConcept.class)
1332                                                .selectEntityReference()
1333                                                .where(f -> finishedQuery)
1334                                                .toQuery();
1335
1336                                return termConceptsQuery.scroll(theScrollChunkSize);
1337                        };
1338
1339                        returnProps.addSearchScroll(nextScroll);
1340                }
1341
1342                return returnProps;
1343        }
1344
1345        private ValueSet.ConceptReferenceComponent getMatchedConceptIncludedInValueSet(
1346                        ValueSet.ConceptSetComponent theIncludeOrExclude, TermConcept concept) {
1347                return theIncludeOrExclude.getConcept().stream()
1348                                .filter(includedConcept -> includedConcept.getCode().equalsIgnoreCase(concept.getCode()))
1349                                .findFirst()
1350                                .orElse(null);
1351        }
1352
1353        /**
1354         * Helper method which builds a predicate for the expansion
1355         */
1356        private PredicateFinalStep buildExpansionPredicate(List<String> theCodes, SearchPredicateFactory thePredicate) {
1357                assert !theCodes.isEmpty();
1358                return thePredicate.simpleQueryString().field("myCode").matching(String.join(" | ", theCodes));
1359        }
1360
1361        private String buildCodeSystemUrlAndVersion(String theSystem, String theIncludeOrExcludeVersion) {
1362                String codeSystemUrlAndVersion;
1363                if (theIncludeOrExcludeVersion != null) {
1364                        codeSystemUrlAndVersion = theSystem + OUR_PIPE_CHARACTER + theIncludeOrExcludeVersion;
1365                } else {
1366                        codeSystemUrlAndVersion = theSystem;
1367                }
1368                return codeSystemUrlAndVersion;
1369        }
1370
1371        private @Nonnull ValueSetExpansionOptions provideExpansionOptions(
1372                        @Nullable ValueSetExpansionOptions theExpansionOptions) {
1373                return Objects.requireNonNullElse(theExpansionOptions, DEFAULT_EXPANSION_OPTIONS);
1374        }
1375
1376        private void addOrRemoveCode(
1377                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
1378                        Set<String> theAddedCodes,
1379                        boolean theAdd,
1380                        String theSystem,
1381                        String theCode,
1382                        String theDisplay,
1383                        String theSystemVersion) {
1384                if (theAdd && theAddedCodes.add(theSystem + OUR_PIPE_CHARACTER + theCode)) {
1385                        theValueSetCodeAccumulator.includeConcept(theSystem, theCode, theDisplay, null, null, theSystemVersion);
1386                }
1387                if (!theAdd && theAddedCodes.remove(theSystem + OUR_PIPE_CHARACTER + theCode)) {
1388                        theValueSetCodeAccumulator.excludeConcept(theSystem, theCode);
1389                }
1390        }
1391
1392        private void handleFilter(
1393                        String theCodeSystemIdentifier,
1394                        SearchPredicateFactory theF,
1395                        BooleanPredicateClausesStep<?> theB,
1396                        ValueSet.ConceptSetFilterComponent theFilter) {
1397                if (isBlank(theFilter.getValue()) && theFilter.getOp() == null && isBlank(theFilter.getProperty())) {
1398                        return;
1399                }
1400
1401                // if filter type is EXISTS, there's no reason to worry about the value (we won't set it anyways)
1402                if ((isBlank(theFilter.getValue()) && theFilter.getOp() != ValueSet.FilterOperator.EXISTS)
1403                                || theFilter.getOp() == null
1404                                || isBlank(theFilter.getProperty())) {
1405                        throw new InvalidRequestException(
1406                                        Msg.code(891) + "Invalid filter, must have fields populated: property op value");
1407                }
1408
1409                switch (theFilter.getProperty()) {
1410                        case "display:exact":
1411                        case "display":
1412                                handleFilterDisplay(theF, theB, theFilter);
1413                                break;
1414                        case "concept":
1415                        case "code":
1416                                handleFilterConceptAndCode(theCodeSystemIdentifier, theF, theB, theFilter);
1417                                break;
1418                        case "parent":
1419                        case "child":
1420                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1421                                handleFilterLoincParentChild(theF, theB, theFilter);
1422                                break;
1423                        case "ancestor":
1424                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1425                                handleFilterLoincAncestor(theCodeSystemIdentifier, theF, theB, theFilter);
1426                                break;
1427                        case "descendant":
1428                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1429                                handleFilterLoincDescendant(theCodeSystemIdentifier, theF, theB, theFilter);
1430                                break;
1431                        case "copyright":
1432                                isCodeSystemLoincOrThrowInvalidRequestException(theCodeSystemIdentifier, theFilter.getProperty());
1433                                handleFilterLoincCopyright(theF, theB, theFilter);
1434                                break;
1435                        default:
1436                                if (theFilter.getOp() == ValueSet.FilterOperator.REGEX) {
1437                                        handleFilterRegex(theF, theB, theFilter);
1438                                } else {
1439                                        handleFilterPropertyDefault(theF, theB, theFilter);
1440                                }
1441                                break;
1442                }
1443        }
1444
1445        private void handleFilterPropertyDefault(
1446                        SearchPredicateFactory theF,
1447                        BooleanPredicateClausesStep<?> theB,
1448                        ValueSet.ConceptSetFilterComponent theFilter) {
1449
1450                String value = theFilter.getValue();
1451                if (theFilter.getOp() == ValueSet.FilterOperator.EXISTS) {
1452                        // EXISTS has no value and is thus handled differently
1453                        Term term = new Term(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty());
1454                        theB.must(theF.exists().field(term.field()));
1455                } else {
1456                        Term term = new Term(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty(), value);
1457                        switch (theFilter.getOp()) {
1458                                case EQUAL:
1459                                        theB.must(theF.match().field(term.field()).matching(term.text()));
1460                                        break;
1461                                case IN:
1462                                case NOTIN:
1463                                        boolean isNotFilter = theFilter.getOp() == ValueSet.FilterOperator.NOTIN;
1464                                        // IN and NOTIN expect comma separated lists
1465                                        String[] values = term.text().split(",");
1466                                        Set<String> valueSet = new HashSet<>(Arrays.asList(values));
1467                                        if (isNotFilter) {
1468                                                theB.filter(theF.not(theF.terms().field(term.field()).matchingAny(valueSet)));
1469                                        } else {
1470                                                theB.filter(theF.terms().field(term.field()).matchingAny(valueSet));
1471                                        }
1472                                        break;
1473                                case ISA:
1474                                case ISNOTA:
1475                                case DESCENDENTOF:
1476                                case GENERALIZES:
1477                                default:
1478                                        /*
1479                                         * We do not need to handle REGEX, because that's handled in parent
1480                                         * We also don't handle EXISTS because that's a separate area (with different term).
1481                                         * We add a match-none filter because otherwise it matches everything (not desired).
1482                                         */
1483                                        ourLog.error(
1484                                                        "Unsupported property filter {}. This may affect expansion, but will not cause errors.",
1485                                                        theFilter.getOp().getDisplay());
1486                                        theB.must(theF.matchNone());
1487                                        break;
1488                        }
1489                }
1490        }
1491
1492        private void handleFilterRegex(
1493                        SearchPredicateFactory theF,
1494                        BooleanPredicateClausesStep<?> theB,
1495                        ValueSet.ConceptSetFilterComponent theFilter) {
1496                /*
1497                 * We treat the regex filter as a match on the regex
1498                 * anywhere in the property string. The spec does not
1499                 * say whether this is the right behaviour or not, but
1500                 * there are examples that seem to suggest that it is.
1501                 */
1502                String value = theFilter.getValue();
1503                if (value.endsWith("$")) {
1504                        value = value.substring(0, value.length() - 1);
1505                } else if (!value.endsWith(".*")) {
1506                        value = value + ".*";
1507                }
1508                if (!value.startsWith("^") && !value.startsWith(".*")) {
1509                        value = ".*" + value;
1510                } else if (value.startsWith("^")) {
1511                        value = value.substring(1);
1512                }
1513
1514                theB.must(theF.regexp()
1515                                .field(CONCEPT_PROPERTY_PREFIX_NAME + theFilter.getProperty())
1516                                .matching(value));
1517        }
1518
1519        private void handleFilterLoincCopyright(
1520                        SearchPredicateFactory theF,
1521                        BooleanPredicateClausesStep<?> theB,
1522                        ValueSet.ConceptSetFilterComponent theFilter) {
1523
1524                if (theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1525
1526                        String copyrightFilterValue = defaultString(theFilter.getValue()).toLowerCase();
1527                        switch (copyrightFilterValue) {
1528                                case "3rdparty":
1529                                        logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1530                                        addFilterLoincCopyright3rdParty(theF, theB);
1531                                        break;
1532                                case "loinc":
1533                                        logFilteringValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1534                                        addFilterLoincCopyrightLoinc(theF, theB);
1535                                        break;
1536                                default:
1537                                        throwInvalidRequestForValueOnProperty(theFilter.getValue(), theFilter.getProperty());
1538                        }
1539
1540                } else {
1541                        throwInvalidRequestForOpOnProperty(theFilter.getOp(), theFilter.getProperty());
1542                }
1543        }
1544
1545        private void addFilterLoincCopyrightLoinc(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB) {
1546                theB.mustNot(theF.exists().field(CONCEPT_PROPERTY_PREFIX_NAME + "EXTERNAL_COPYRIGHT_NOTICE"));
1547        }
1548
1549        private void addFilterLoincCopyright3rdParty(SearchPredicateFactory theF, BooleanPredicateClausesStep<?> theB) {
1550                theB.must(theF.exists().field(CONCEPT_PROPERTY_PREFIX_NAME + "EXTERNAL_COPYRIGHT_NOTICE"));
1551        }
1552
1553        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1554        private void handleFilterLoincAncestor(
1555                        String theSystem,
1556                        SearchPredicateFactory f,
1557                        BooleanPredicateClausesStep<?> b,
1558                        ValueSet.ConceptSetFilterComponent theFilter) {
1559                switch (theFilter.getOp()) {
1560                        case EQUAL:
1561                                addLoincFilterAncestorEqual(theSystem, f, b, theFilter);
1562                                break;
1563                        case IN:
1564                                addLoincFilterAncestorIn(theSystem, f, b, theFilter);
1565                                break;
1566                        default:
1567                                throw new InvalidRequestException(Msg.code(892) + "Don't know how to handle op=" + theFilter.getOp()
1568                                                + " on property " + theFilter.getProperty());
1569                }
1570        }
1571
1572        private void addLoincFilterAncestorEqual(
1573                        String theSystem,
1574                        SearchPredicateFactory f,
1575                        BooleanPredicateClausesStep<?> b,
1576                        ValueSet.ConceptSetFilterComponent theFilter) {
1577                addLoincFilterAncestorEqual(theSystem, f, b, theFilter.getProperty(), theFilter.getValue());
1578        }
1579
1580        private void addLoincFilterAncestorEqual(
1581                        String theSystem,
1582                        SearchPredicateFactory f,
1583                        BooleanPredicateClausesStep<?> b,
1584                        String theProperty,
1585                        String theValue) {
1586                List<Term> terms = getAncestorTerms(theSystem, theProperty, theValue);
1587                b.must(f.bool(innerB -> terms.forEach(
1588                                term -> innerB.should(f.match().field(term.field()).matching(term.text())))));
1589        }
1590
1591        private void addLoincFilterAncestorIn(
1592                        String theSystem,
1593                        SearchPredicateFactory f,
1594                        BooleanPredicateClausesStep<?> b,
1595                        ValueSet.ConceptSetFilterComponent theFilter) {
1596                String[] values = theFilter.getValue().split(",");
1597                List<Term> terms = new ArrayList<>();
1598                for (String value : values) {
1599                        terms.addAll(getAncestorTerms(theSystem, theFilter.getProperty(), value));
1600                }
1601                b.must(f.bool(innerB -> terms.forEach(
1602                                term -> innerB.should(f.match().field(term.field()).matching(term.text())))));
1603        }
1604
1605        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1606        private void handleFilterLoincParentChild(
1607                        SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1608                switch (theFilter.getOp()) {
1609                        case EQUAL:
1610                                addLoincFilterParentChildEqual(f, b, theFilter.getProperty(), theFilter.getValue());
1611                                break;
1612                        case IN:
1613                                addLoincFilterParentChildIn(f, b, theFilter);
1614                                break;
1615                        default:
1616                                throw new InvalidRequestException(Msg.code(893) + "Don't know how to handle op=" + theFilter.getOp()
1617                                                + " on property " + theFilter.getProperty());
1618                }
1619        }
1620
1621        private void addLoincFilterParentChildIn(
1622                        SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1623                String[] values = theFilter.getValue().split(",");
1624                List<Term> terms = new ArrayList<>();
1625                for (String value : values) {
1626                        logFilteringValueOnProperty(value, theFilter.getProperty());
1627                        terms.add(getPropertyTerm(theFilter.getProperty(), value));
1628                }
1629
1630                b.must(f.bool(innerB -> terms.forEach(
1631                                term -> innerB.should(f.match().field(term.field()).matching(term.text())))));
1632        }
1633
1634        private void addLoincFilterParentChildEqual(
1635                        SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, String theProperty, String theValue) {
1636                logFilteringValueOnProperty(theValue, theProperty);
1637                b.must(f.match().field(CONCEPT_PROPERTY_PREFIX_NAME + theProperty).matching(theValue));
1638        }
1639
1640        private void handleFilterConceptAndCode(
1641                        String theSystem,
1642                        SearchPredicateFactory f,
1643                        BooleanPredicateClausesStep<?> b,
1644                        ValueSet.ConceptSetFilterComponent theFilter) {
1645                TermConcept code = findCodeForFilterCriteriaCodeOrConcept(theSystem, theFilter);
1646
1647                if (theFilter.getOp() == ValueSet.FilterOperator.ISA) {
1648                        ourLog.debug(
1649                                        " * Filtering on specific code and codes with a parent of {}/{}/{}",
1650                                        code.getId(),
1651                                        code.getCode(),
1652                                        code.getDisplay());
1653
1654                        b.must(f.bool()
1655                                        .should(f.match().field("myParentPids").matching("" + code.getId()))
1656                                        .should(f.match().field("myId").matching(code.getId())));
1657                } else if (theFilter.getOp() == ValueSet.FilterOperator.DESCENDENTOF) {
1658                        ourLog.debug(
1659                                        " * Filtering on codes with a parent of {}/{}/{}", code.getId(), code.getCode(), code.getDisplay());
1660
1661                        b.must(f.match().field("myParentPids").matching("" + code.getId()));
1662                } else {
1663                        throwInvalidFilter(theFilter, "");
1664                }
1665        }
1666
1667        @Nonnull
1668        private TermConcept findCodeForFilterCriteriaCodeOrConcept(
1669                        String theSystem, ValueSet.ConceptSetFilterComponent theFilter) {
1670                return findCode(theSystem, theFilter.getValue())
1671                                .orElseThrow(() ->
1672                                                new InvalidRequestException(Msg.code(2071) + "Invalid filter criteria - code does not exist: {"
1673                                                                + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theFilter.getValue()));
1674        }
1675
1676        private void throwInvalidFilter(ValueSet.ConceptSetFilterComponent theFilter, String theErrorSuffix) {
1677                throw new InvalidRequestException(Msg.code(894) + "Don't know how to handle op=" + theFilter.getOp()
1678                                + " on property " + theFilter.getProperty() + theErrorSuffix);
1679        }
1680
1681        private void isCodeSystemLoincOrThrowInvalidRequestException(String theSystemIdentifier, String theProperty) {
1682                String systemUrl = getUrlFromIdentifier(theSystemIdentifier);
1683                if (!isCodeSystemLoinc(systemUrl)) {
1684                        throw new InvalidRequestException(Msg.code(895) + "Invalid filter, property " + theProperty
1685                                        + " is LOINC-specific and cannot be used with system: " + systemUrl);
1686                }
1687        }
1688
1689        private boolean isCodeSystemLoinc(String theSystem) {
1690                return LOINC_URI.equals(theSystem);
1691        }
1692
1693        private void handleFilterDisplay(
1694                        SearchPredicateFactory f, BooleanPredicateClausesStep<?> b, ValueSet.ConceptSetFilterComponent theFilter) {
1695                if (theFilter.getProperty().equals("display:exact") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1696                        addDisplayFilterExact(f, b, theFilter);
1697                } else if (theFilter.getProperty().equals("display") && theFilter.getOp() == ValueSet.FilterOperator.EQUAL) {
1698                        if (theFilter.getValue().trim().contains(" ")) {
1699                                addDisplayFilterExact(f, b, theFilter);
1700                        } else {
1701                                addDisplayFilterInexact(f, b, theFilter);
1702                        }
1703                }
1704        }
1705
1706        private void addDisplayFilterExact(
1707                        SearchPredicateFactory f,
1708                        BooleanPredicateClausesStep<?> bool,
1709                        ValueSet.ConceptSetFilterComponent nextFilter) {
1710                bool.must(f.phrase().field("myDisplay").matching(nextFilter.getValue()));
1711        }
1712
1713        private void addDisplayFilterInexact(
1714                        SearchPredicateFactory f,
1715                        BooleanPredicateClausesStep<?> bool,
1716                        ValueSet.ConceptSetFilterComponent nextFilter) {
1717                bool.must(f.phrase()
1718                                .field("myDisplay")
1719                                .boost(4.0f)
1720                                .field("myDisplayWordEdgeNGram")
1721                                .boost(1.0f)
1722                                .field("myDisplayEdgeNGram")
1723                                .boost(1.0f)
1724                                .matching(nextFilter.getValue().toLowerCase())
1725                                .slop(2));
1726        }
1727
1728        private Term getPropertyTerm(String theProperty, String theValue) {
1729                return new Term(CONCEPT_PROPERTY_PREFIX_NAME + theProperty, theValue);
1730        }
1731
1732        private List<Term> getAncestorTerms(String theSystem, String theProperty, String theValue) {
1733                List<Term> retVal = new ArrayList<>();
1734
1735                TermConcept code = findCode(theSystem, theValue)
1736                                .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {"
1737                                                + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue));
1738
1739                retVal.add(new Term("myParentPids", "" + code.getId()));
1740                logFilteringValueOnProperty(theValue, theProperty);
1741
1742                return retVal;
1743        }
1744
1745        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
1746        private void handleFilterLoincDescendant(
1747                        String theSystem,
1748                        SearchPredicateFactory f,
1749                        BooleanPredicateClausesStep<?> b,
1750                        ValueSet.ConceptSetFilterComponent theFilter) {
1751                switch (theFilter.getOp()) {
1752                        case EQUAL:
1753                                addLoincFilterDescendantEqual(theSystem, f, b, theFilter);
1754                                break;
1755                        case IN:
1756                                addLoincFilterDescendantIn(theSystem, f, b, theFilter);
1757                                break;
1758                        default:
1759                                throw new InvalidRequestException(Msg.code(896) + "Don't know how to handle op=" + theFilter.getOp()
1760                                                + " on property " + theFilter.getProperty());
1761                }
1762        }
1763
1764        private void addLoincFilterDescendantEqual(
1765                        String theSystem,
1766                        SearchPredicateFactory f,
1767                        BooleanPredicateClausesStep<?> b,
1768                        ValueSet.ConceptSetFilterComponent theFilter) {
1769
1770                List<Long> parentPids = getCodeParentPids(theSystem, theFilter.getProperty(), theFilter.getValue());
1771                if (parentPids.isEmpty()) {
1772                        // Can't return empty must, because it wil match according to other predicates.
1773                        // Some day there will be a 'matchNone' predicate
1774                        // (https://discourse.hibernate.org/t/fail-fast-predicate/6062)
1775                        b.mustNot(f.matchAll());
1776                        return;
1777                }
1778
1779                b.must(f.bool(innerB -> {
1780                        innerB.minimumShouldMatchNumber(1);
1781                        parentPids.forEach(pid -> innerB.should(f.match().field("myId").matching(pid)));
1782                }));
1783        }
1784
1785        /**
1786         * We are looking for codes which have codes indicated in theFilter.getValue() as descendants.
1787         * Strategy is to find codes which have their pId(s) in the list of the parentId(s) of all the TermConcept(s)
1788         * representing the codes in theFilter.getValue()
1789         */
1790        private void addLoincFilterDescendantIn(
1791                        String theSystem,
1792                        SearchPredicateFactory f,
1793                        BooleanPredicateClausesStep<?> b,
1794                        ValueSet.ConceptSetFilterComponent theFilter) {
1795
1796                String[] values = theFilter.getValue().split(",");
1797                if (values.length == 0) {
1798                        throw new InvalidRequestException(Msg.code(2062) + "Invalid filter criteria - no codes specified");
1799                }
1800
1801                List<Long> descendantCodePidList = getMultipleCodeParentPids(theSystem, theFilter.getProperty(), values);
1802
1803                b.must(f.bool(innerB -> descendantCodePidList.forEach(
1804                                pId -> innerB.should(f.match().field("myId").matching(pId)))));
1805        }
1806
1807        /**
1808         * Returns the list of parentId(s) of the TermConcept representing theValue as a code
1809         */
1810        private List<Long> getCodeParentPids(String theSystem, String theProperty, String theValue) {
1811                TermConcept code = findCode(theSystem, theValue)
1812                                .orElseThrow(() -> new InvalidRequestException("Invalid filter criteria - code does not exist: {"
1813                                                + Constants.codeSystemWithDefaultDescription(theSystem) + "}" + theValue));
1814
1815                String[] parentPids = code.getParentPidsAsString().split(" ");
1816                List<Long> retVal = Arrays.stream(parentPids)
1817                                .filter(pid -> !StringUtils.equals(pid, "NONE"))
1818                                .map(Long::parseLong)
1819                                .collect(Collectors.toList());
1820                logFilteringValueOnProperty(theValue, theProperty);
1821                return retVal;
1822        }
1823
1824        /**
1825         * Returns the list of parentId(s) of the TermConcept representing theValue as a code
1826         */
1827        private List<Long> getMultipleCodeParentPids(String theSystem, String theProperty, String[] theValues) {
1828                List<String> valuesList = Arrays.asList(theValues);
1829                List<TermConcept> termConcepts = findCodes(theSystem, valuesList);
1830                if (valuesList.size() != termConcepts.size()) {
1831                        String exMsg = getTermConceptsFetchExceptionMsg(termConcepts, valuesList);
1832                        throw new InvalidRequestException(Msg.code(2064) + "Invalid filter criteria - {"
1833                                        + Constants.codeSystemWithDefaultDescription(theSystem) + "}: " + exMsg);
1834                }
1835
1836                List<Long> retVal = termConcepts.stream()
1837                                .flatMap(tc -> Arrays.stream(tc.getParentPidsAsString().split(" ")))
1838                                .filter(pid -> !StringUtils.equals(pid, "NONE"))
1839                                .map(Long::parseLong)
1840                                .collect(Collectors.toList());
1841
1842                logFilteringValueOnProperties(valuesList, theProperty);
1843
1844                return retVal;
1845        }
1846
1847        /**
1848         * Generate message indicating for which of theValues a TermConcept was not found
1849         */
1850        private String getTermConceptsFetchExceptionMsg(List<TermConcept> theTermConcepts, List<String> theValues) {
1851                // case: more TermConcept(s) retrieved than codes queried
1852                if (theTermConcepts.size() > theValues.size()) {
1853                        return "Invalid filter criteria - More TermConcepts were found than indicated codes. Queried codes: ["
1854                                        + join(
1855                                                        ",",
1856                                                        theValues + "]; Obtained TermConcept IDs, codes: ["
1857                                                                        + theTermConcepts.stream()
1858                                                                                        .map(tc -> tc.getId() + ", " + tc.getCode())
1859                                                                                        .collect(joining("; "))
1860                                                                        + "]");
1861                }
1862
1863                // case: less TermConcept(s) retrieved than codes queried
1864                Set<String> matchedCodes =
1865                                theTermConcepts.stream().map(TermConcept::getCode).collect(toSet());
1866                List<String> notMatchedValues =
1867                                theValues.stream().filter(v -> !matchedCodes.contains(v)).collect(toList());
1868
1869                return "Invalid filter criteria - No TermConcept(s) were found for the requested codes: ["
1870                                + join(",", notMatchedValues + "]");
1871        }
1872
1873        private void logFilteringValueOnProperty(String theValue, String theProperty) {
1874                ourLog.debug(" * Filtering with value={} on property {}", theValue, theProperty);
1875        }
1876
1877        private void logFilteringValueOnProperties(List<String> theValues, String theProperty) {
1878                ourLog.debug(" * Filtering with values={} on property {}", String.join(", ", theValues), theProperty);
1879        }
1880
1881        private void throwInvalidRequestForOpOnProperty(ValueSet.FilterOperator theOp, String theProperty) {
1882                throw new InvalidRequestException(
1883                                Msg.code(897) + "Don't know how to handle op=" + theOp + " on property " + theProperty);
1884        }
1885
1886        private void throwInvalidRequestForValueOnProperty(String theValue, String theProperty) {
1887                throw new InvalidRequestException(
1888                                Msg.code(898) + "Don't know how to handle value=" + theValue + " on property " + theProperty);
1889        }
1890
1891        private void expandWithoutHibernateSearch(
1892                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
1893                        TermCodeSystemVersion theVersion,
1894                        Set<String> theAddedCodes,
1895                        ValueSet.ConceptSetComponent theInclude,
1896                        String theSystem,
1897                        boolean theAdd) {
1898                ourLog.trace("Hibernate search is not enabled");
1899
1900                if (theValueSetCodeAccumulator instanceof ValueSetExpansionComponentWithConceptAccumulator) {
1901                        Validate.isTrue(
1902                                        ((ValueSetExpansionComponentWithConceptAccumulator) theValueSetCodeAccumulator)
1903                                                        .getParameter()
1904                                                        .isEmpty(),
1905                                        "Can not expand ValueSet with parameters - Hibernate Search is not enabled on this server.");
1906                }
1907
1908                Validate.isTrue(
1909                                isNotBlank(theSystem),
1910                                "Can not expand ValueSet without explicit system - Hibernate Search is not enabled on this server.");
1911
1912                for (ValueSet.ConceptSetFilterComponent nextFilter : theInclude.getFilter()) {
1913                        boolean handled = false;
1914                        switch (nextFilter.getProperty().toLowerCase()) {
1915                                case "concept":
1916                                case "code":
1917                                        if (nextFilter.getOp() == ValueSet.FilterOperator.ISA) {
1918                                                theValueSetCodeAccumulator.addMessage(
1919                                                                "Processing IS-A filter in database - Note that Hibernate Search is not enabled on this server, so this operation can be inefficient.");
1920                                                TermConcept code = findCodeForFilterCriteriaCodeOrConcept(theSystem, nextFilter);
1921                                                addConceptAndChildren(
1922                                                                theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, code);
1923                                                handled = true;
1924                                        }
1925                                        break;
1926                                default:
1927                                        // TODO - we need to handle other properties (fields)
1928                                        // and other operations (not just is-a)
1929                                        // in some (preferably generic) way
1930                                        break;
1931                        }
1932
1933                        if (!handled) {
1934                                throwInvalidFilter(
1935                                                nextFilter,
1936                                                " - Note that Hibernate Search is disabled on this server so not all ValueSet expansion functionality is available.");
1937                        }
1938                }
1939
1940                if (theInclude.getFilter().isEmpty() && theInclude.getConcept().isEmpty()) {
1941                        Collection<TermConcept> concepts =
1942                                        myConceptDao.fetchConceptsAndDesignationsByVersionPid(theVersion.getPid());
1943                        for (TermConcept next : concepts) {
1944                                addCodeIfNotAlreadyAdded(
1945                                                theValueSetCodeAccumulator,
1946                                                theAddedCodes,
1947                                                theAdd,
1948                                                theSystem,
1949                                                theInclude.getVersion(),
1950                                                next.getCode(),
1951                                                next.getDisplay(),
1952                                                next.getId(),
1953                                                next.getParentPidsAsString(),
1954                                                next.getDesignations());
1955                        }
1956                }
1957
1958                for (ValueSet.ConceptReferenceComponent next : theInclude.getConcept()) {
1959                        if (!theSystem.equals(theInclude.getSystem()) && isNotBlank(theSystem)) {
1960                                continue;
1961                        }
1962                        Collection<TermConceptDesignation> designations = next.getDesignation().stream()
1963                                        .map(t -> new TermConceptDesignation()
1964                                                        .setValue(t.getValue())
1965                                                        .setLanguage(t.getLanguage())
1966                                                        .setUseCode(t.getUse().getCode())
1967                                                        .setUseSystem(t.getUse().getSystem())
1968                                                        .setUseDisplay(t.getUse().getDisplay()))
1969                                        .collect(Collectors.toList());
1970                        addCodeIfNotAlreadyAdded(
1971                                        theValueSetCodeAccumulator,
1972                                        theAddedCodes,
1973                                        theAdd,
1974                                        theSystem,
1975                                        theInclude.getVersion(),
1976                                        next.getCode(),
1977                                        next.getDisplay(),
1978                                        null,
1979                                        null,
1980                                        designations);
1981                }
1982        }
1983
1984        private void addConceptAndChildren(
1985                        IValueSetConceptAccumulator theValueSetCodeAccumulator,
1986                        Set<String> theAddedCodes,
1987                        ValueSet.ConceptSetComponent theInclude,
1988                        String theSystem,
1989                        boolean theAdd,
1990                        TermConcept theConcept) {
1991                for (TermConcept nextChild : theConcept.getChildCodes()) {
1992                        boolean added = addCodeIfNotAlreadyAdded(
1993                                        theValueSetCodeAccumulator,
1994                                        theAddedCodes,
1995                                        theAdd,
1996                                        theSystem,
1997                                        theInclude.getVersion(),
1998                                        nextChild.getCode(),
1999                                        nextChild.getDisplay(),
2000                                        nextChild.getId(),
2001                                        nextChild.getParentPidsAsString(),
2002                                        nextChild.getDesignations());
2003                        if (added) {
2004                                addConceptAndChildren(
2005                                                theValueSetCodeAccumulator, theAddedCodes, theInclude, theSystem, theAdd, nextChild);
2006                        }
2007                }
2008        }
2009
2010        @Override
2011        @Transactional
2012        public String invalidatePreCalculatedExpansion(IIdType theValueSetId, RequestDetails theRequestDetails) {
2013                IBaseResource valueSet = myDaoRegistry.getResourceDao("ValueSet").read(theValueSetId, theRequestDetails);
2014                ValueSet canonicalValueSet = myVersionCanonicalizer.valueSetToCanonical(valueSet);
2015                Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(canonicalValueSet);
2016                if (optionalTermValueSet.isEmpty()) {
2017                        return myContext
2018                                        .getLocalizer()
2019                                        .getMessage(TermReadSvcImpl.class, "valueSetNotFoundInTerminologyDatabase", theValueSetId);
2020                }
2021
2022                ourLog.info(
2023                                "Invalidating pre-calculated expansion on ValueSet {} / {}", theValueSetId, canonicalValueSet.getUrl());
2024
2025                TermValueSet termValueSet = optionalTermValueSet.get();
2026                if (termValueSet.getExpansionStatus() == TermValueSetPreExpansionStatusEnum.NOT_EXPANDED) {
2027                        return myContext
2028                                        .getLocalizer()
2029                                        .getMessage(
2030                                                        TermReadSvcImpl.class,
2031                                                        "valueSetCantInvalidateNotYetPrecalculated",
2032                                                        termValueSet.getUrl(),
2033                                                        termValueSet.getExpansionStatus());
2034                }
2035
2036                Long totalConcepts = termValueSet.getTotalConcepts();
2037
2038                deletePreCalculatedValueSetContents(termValueSet);
2039
2040                termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.NOT_EXPANDED);
2041                termValueSet.setExpansionTimestamp(null);
2042                myTermValueSetDao.save(termValueSet);
2043
2044                afterValueSetExpansionStatusChange();
2045
2046                return myContext
2047                                .getLocalizer()
2048                                .getMessage(
2049                                                TermReadSvcImpl.class, "valueSetPreExpansionInvalidated", termValueSet.getUrl(), totalConcepts);
2050        }
2051
2052        @Override
2053        @Transactional(readOnly = true)
2054        public boolean isValueSetPreExpandedForCodeValidation(ValueSet theValueSet) {
2055                Optional<TermValueSet> optionalTermValueSet = fetchValueSetEntity(theValueSet);
2056
2057                if (optionalTermValueSet.isEmpty()) {
2058                        ourLog.warn(
2059                                        "ValueSet is not present in terminology tables. Will perform in-memory code validation. {}",
2060                                        getValueSetInfo(theValueSet));
2061                        return false;
2062                }
2063
2064                TermValueSet termValueSet = optionalTermValueSet.get();
2065
2066                if (termValueSet.getExpansionStatus() != TermValueSetPreExpansionStatusEnum.EXPANDED) {
2067                        ourLog.warn(
2068                                        "{} is present in terminology tables but not ready for persistence-backed invocation of operation $validation-code. Will perform in-memory code validation. Current status: {} | {}",
2069                                        getValueSetInfo(theValueSet),
2070                                        termValueSet.getExpansionStatus().name(),
2071                                        termValueSet.getExpansionStatus().getDescription());
2072                        return false;
2073                }
2074
2075                return true;
2076        }
2077
2078        @SuppressWarnings({"OptionalAssignedToNull", "unchecked"})
2079        private Optional<TermValueSet> fetchValueSetEntity(ValueSet theValueSet) {
2080                Optional<TermValueSet> retVal = (Optional<TermValueSet>) theValueSet.getUserData(VS_USERDATA_CURRENT_VERSION);
2081                if (retVal == null) {
2082                        synchronized (theValueSet) {
2083                                JpaPid valueSetResourcePid = getValueSetResourcePersistentId(theValueSet);
2084                                retVal = myTermValueSetDao.findByResourcePid(valueSetResourcePid.getId());
2085                                theValueSet.setUserData(VS_USERDATA_CURRENT_VERSION, retVal);
2086                        }
2087                }
2088
2089                return retVal;
2090        }
2091
2092        private JpaPid getValueSetResourcePersistentId(ValueSet theValueSet) {
2093                return myIdHelperService.resolveResourceIdentityPid(
2094                                RequestPartitionId.allPartitions(),
2095                                theValueSet.getIdElement().getResourceType(),
2096                                theValueSet.getIdElement().getIdPart(),
2097                                ResolveIdentityMode.includeDeleted().cacheOk());
2098        }
2099
2100        protected IValidationSupport.CodeValidationResult validateCodeIsInPreExpandedValueSet(
2101                        ValidationSupportContext theValidationSupportContext,
2102                        ConceptValidationOptions theValidationOptions,
2103                        ValueSet theValueSet,
2104                        String theSystem,
2105                        String theCode,
2106                        String theDisplay,
2107                        Coding theCoding,
2108                        CodeableConcept theCodeableConcept) {
2109                assert TransactionSynchronizationManager.isSynchronizationActive();
2110
2111                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet.hasId(), "ValueSet.id is required");
2112                JpaPid valueSetResourcePid = getValueSetResourcePersistentId(theValueSet);
2113
2114                List<TermValueSetConcept> concepts = new ArrayList<>();
2115                if (isNotBlank(theCode)) {
2116                        if (theValidationOptions.isInferSystem()) {
2117                                concepts.addAll(
2118                                                myValueSetConceptDao.findByValueSetResourcePidAndCode(valueSetResourcePid.getId(), theCode));
2119                        } else if (isNotBlank(theSystem)) {
2120                                concepts.addAll(findByValueSetResourcePidSystemAndCode(valueSetResourcePid, theSystem, theCode));
2121                        }
2122                } else if (theCoding != null) {
2123                        if (theCoding.hasSystem() && theCoding.hasCode()) {
2124                                concepts.addAll(findByValueSetResourcePidSystemAndCode(
2125                                                valueSetResourcePid, theCoding.getSystem(), theCoding.getCode()));
2126                        }
2127                } else if (theCodeableConcept != null) {
2128                        for (Coding coding : theCodeableConcept.getCoding()) {
2129                                if (coding.hasSystem() && coding.hasCode()) {
2130                                        concepts.addAll(findByValueSetResourcePidSystemAndCode(
2131                                                        valueSetResourcePid, coding.getSystem(), coding.getCode()));
2132                                        if (!concepts.isEmpty()) {
2133                                                break;
2134                                        }
2135                                }
2136                        }
2137                } else {
2138                        return null;
2139                }
2140
2141                TermValueSet valueSetEntity = fetchValueSetEntity(theValueSet).orElseThrow(IllegalStateException::new);
2142                String timingDescription = toHumanReadableExpansionTimestamp(valueSetEntity);
2143                String preExpansionMessage = myContext
2144                                .getLocalizer()
2145                                .getMessage(TermReadSvcImpl.class, "validationPerformedAgainstPreExpansion", timingDescription);
2146
2147                if (theValidationOptions.isValidateDisplay() && concepts.size() > 0) {
2148                        String systemVersion = null;
2149                        for (TermValueSetConcept concept : concepts) {
2150                                systemVersion = concept.getSystemVersion();
2151                                if (isBlank(theDisplay) || isBlank(concept.getDisplay()) || theDisplay.equals(concept.getDisplay())) {
2152                                        return new IValidationSupport.CodeValidationResult()
2153                                                        .setCode(concept.getCode())
2154                                                        .setDisplay(concept.getDisplay())
2155                                                        .setCodeSystemVersion(concept.getSystemVersion())
2156                                                        .setSourceDetails(preExpansionMessage);
2157                                }
2158                        }
2159
2160                        String expectedDisplay = concepts.get(0).getDisplay();
2161                        return InMemoryTerminologyServerValidationSupport.createResultForDisplayMismatch(
2162                                        myContext,
2163                                        theCode,
2164                                        theDisplay,
2165                                        expectedDisplay,
2166                                        theSystem,
2167                                        systemVersion,
2168                                        myStorageSettings.getIssueSeverityForCodeDisplayMismatch());
2169                }
2170
2171                if (!concepts.isEmpty()) {
2172                        return new IValidationSupport.CodeValidationResult()
2173                                        .setCode(concepts.get(0).getCode())
2174                                        .setDisplay(concepts.get(0).getDisplay())
2175                                        .setCodeSystemVersion(concepts.get(0).getSystemVersion())
2176                                        .setMessage(preExpansionMessage);
2177                }
2178
2179                // Ok, we failed
2180                List<TermValueSetConcept> outcome = myValueSetConceptDao.findByTermValueSetIdSystemOnly(
2181                                Pageable.ofSize(1), valueSetEntity.getId(), theSystem);
2182                String append;
2183                if (outcome.size() == 0) {
2184                        append = " - No codes in ValueSet belong to CodeSystem with URL " + theSystem;
2185                } else {
2186                        String unknownCodeMessage = myContext
2187                                        .getLocalizer()
2188                                        .getMessage(TermReadSvcImpl.class, "unknownCodeInSystem", theSystem, theCode);
2189                        append = " - " + unknownCodeMessage + ". " + preExpansionMessage;
2190                }
2191
2192                return createFailureCodeValidationResult(theSystem, theCode, null, append);
2193        }
2194
2195        private CodeValidationResult createFailureCodeValidationResult(
2196                        String theSystem, String theCode, String theCodeSystemVersion, String theAppend) {
2197                String theMessage = "Unable to validate code " + theSystem + "#" + theCode + theAppend;
2198                return new CodeValidationResult()
2199                                .setSeverity(IssueSeverity.ERROR)
2200                                .setCodeSystemVersion(theCodeSystemVersion)
2201                                .setMessage(theMessage)
2202                                .addIssue(new CodeValidationIssue(
2203                                                theMessage,
2204                                                IssueSeverity.ERROR,
2205                                                CodeValidationIssueCode.CODE_INVALID,
2206                                                CodeValidationIssueCoding.INVALID_CODE));
2207        }
2208
2209        private List<TermValueSetConcept> findByValueSetResourcePidSystemAndCode(
2210                        JpaPid theResourcePid, String theSystem, String theCode) {
2211                assert TransactionSynchronizationManager.isSynchronizationActive();
2212
2213                List<TermValueSetConcept> retVal = new ArrayList<>();
2214                Optional<TermValueSetConcept> optionalTermValueSetConcept;
2215                int versionIndex = theSystem.indexOf(OUR_PIPE_CHARACTER);
2216                if (versionIndex >= 0) {
2217                        String systemUrl = theSystem.substring(0, versionIndex);
2218                        String systemVersion = theSystem.substring(versionIndex + 1);
2219                        optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCodeWithVersion(
2220                                        theResourcePid.getId(), systemUrl, systemVersion, theCode);
2221                } else {
2222                        optionalTermValueSetConcept = myValueSetConceptDao.findByValueSetResourcePidSystemAndCode(
2223                                        theResourcePid.getId(), theSystem, theCode);
2224                }
2225                optionalTermValueSetConcept.ifPresent(retVal::add);
2226                return retVal;
2227        }
2228
2229        private void fetchChildren(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
2230                for (TermConceptParentChildLink nextChildLink : theConcept.getChildren()) {
2231                        TermConcept nextChild = nextChildLink.getChild();
2232                        if (addToSet(theSetToPopulate, nextChild)) {
2233                                fetchChildren(nextChild, theSetToPopulate);
2234                        }
2235                }
2236        }
2237
2238        private Optional<TermConcept> fetchLoadedCode(Long theCodeSystemResourcePid, String theCode) {
2239                TermCodeSystemVersion codeSystem =
2240                                myCodeSystemVersionDao.findCurrentVersionForCodeSystemResourcePid(theCodeSystemResourcePid);
2241                return myConceptDao.findByCodeSystemAndCode(codeSystem.getPid(), theCode);
2242        }
2243
2244        private void fetchParents(TermConcept theConcept, Set<TermConcept> theSetToPopulate) {
2245                for (TermConceptParentChildLink nextChildLink : theConcept.getParents()) {
2246                        TermConcept nextChild = nextChildLink.getParent();
2247                        if (addToSet(theSetToPopulate, nextChild)) {
2248                                fetchParents(nextChild, theSetToPopulate);
2249                        }
2250                }
2251        }
2252
2253        @Override
2254        public Optional<TermConcept> findCode(String theCodeSystem, String theCode) {
2255                /*
2256                 * Loading concepts without a transaction causes issues later on some
2257                 * platforms (e.g. PSQL) so make sure that we always call this with an open transaction
2258                 */
2259                HapiTransactionService.requireTransaction();
2260
2261                TermCodeSystemVersionDetails csv =
2262                                getCurrentCodeSystemVersion(new ValidationSupportContext(provideValidationSupport()), theCodeSystem);
2263                if (csv == null) {
2264                        return Optional.empty();
2265                }
2266                return myConceptDao.findByCodeSystemAndCode(csv.myPid, theCode);
2267        }
2268
2269        @Override
2270        public List<TermConcept> findCodes(String theCodeSystem, List<String> theCodeList) {
2271                HapiTransactionService.requireTransaction();
2272
2273                TermCodeSystemVersionDetails csv =
2274                                getCurrentCodeSystemVersion(new ValidationSupportContext(provideValidationSupport()), theCodeSystem);
2275                if (csv == null) {
2276                        return Collections.emptyList();
2277                }
2278
2279                return myConceptDao.findByCodeSystemAndCodeList(csv.myPid, theCodeList);
2280        }
2281
2282        @Nullable
2283        private TermCodeSystemVersionDetails getCurrentCodeSystemVersion(
2284                        ValidationSupportContext theValidationSupportContext, String theCodeSystemIdentifier) {
2285                String version = getVersionFromIdentifier(theCodeSystemIdentifier);
2286
2287                // Fetch the CodeSystem from ValidationSupport, which should return a cached copy. We
2288                // keep a copy of the current version entity in userData in that cached copy
2289                // to avoid repeated lookups
2290                TermCodeSystemVersionDetails retVal;
2291                IBaseResource codeSystem =
2292                                theValidationSupportContext.getRootValidationSupport().fetchCodeSystem(theCodeSystemIdentifier);
2293                if (codeSystem != null) {
2294
2295                        synchronized (codeSystem) {
2296                                retVal = (TermCodeSystemVersionDetails) codeSystem.getUserData(CS_USERDATA_CURRENT_VERSION);
2297                                if (retVal == null) {
2298                                        retVal = getCurrentCodeSystemVersion(theCodeSystemIdentifier, version);
2299                                        codeSystem.setUserData(CS_USERDATA_CURRENT_VERSION, retVal);
2300                                }
2301                        }
2302                } else {
2303                        retVal = getCurrentCodeSystemVersion(theCodeSystemIdentifier, version);
2304                }
2305
2306                return retVal;
2307        }
2308
2309        @Nullable
2310        private TermCodeSystemVersionDetails getCurrentCodeSystemVersion(String theCodeSystemIdentifier, String version) {
2311                TermCodeSystemVersionDetails retVal;
2312                retVal = myTxTemplate.execute(tx -> {
2313                        TermCodeSystemVersion csv = null;
2314                        TermCodeSystem cs = myCodeSystemDao.findByCodeSystemUri(getUrlFromIdentifier(theCodeSystemIdentifier));
2315                        if (cs != null) {
2316                                if (version != null) {
2317                                        csv = myCodeSystemVersionDao.findByCodeSystemPidAndVersion(cs.getPid(), version);
2318                                } else if (cs.getCurrentVersion() != null) {
2319                                        csv = cs.getCurrentVersion();
2320                                }
2321                        }
2322                        if (csv != null) {
2323                                return new TermCodeSystemVersionDetails(csv.getPid(), csv.getCodeSystemVersionId());
2324                        } else {
2325                                return null;
2326                        }
2327                });
2328                return retVal;
2329        }
2330
2331        private String getVersionFromIdentifier(String theUri) {
2332                String retVal = null;
2333                if (StringUtils.isNotEmpty((theUri))) {
2334                        int versionSeparator = theUri.lastIndexOf('|');
2335                        if (versionSeparator != -1) {
2336                                retVal = theUri.substring(versionSeparator + 1);
2337                        }
2338                }
2339                return retVal;
2340        }
2341
2342        private String getUrlFromIdentifier(String theUri) {
2343                String retVal = theUri;
2344                if (StringUtils.isNotEmpty((theUri))) {
2345                        int versionSeparator = theUri.lastIndexOf('|');
2346                        if (versionSeparator != -1) {
2347                                retVal = theUri.substring(0, versionSeparator);
2348                        }
2349                }
2350                return retVal;
2351        }
2352
2353        @Transactional(propagation = Propagation.REQUIRED, readOnly = true)
2354        @Override
2355        public Set<TermConcept> findCodesAbove(
2356                        Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
2357                StopWatch stopwatch = new StopWatch();
2358
2359                Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
2360                if (concept.isEmpty()) {
2361                        return Collections.emptySet();
2362                }
2363
2364                Set<TermConcept> retVal = new HashSet<>();
2365                retVal.add(concept.get());
2366
2367                fetchParents(concept.get(), retVal);
2368
2369                ourLog.debug("Fetched {} codes above code {} in {}ms", retVal.size(), theCode, stopwatch.getMillis());
2370                return retVal;
2371        }
2372
2373        @Transactional(readOnly = true)
2374        @Override
2375        public List<FhirVersionIndependentConcept> findCodesAbove(String theSystem, String theCode) {
2376                TermCodeSystem cs = getCodeSystem(theSystem);
2377                if (cs == null) {
2378                        return findCodesAboveUsingBuiltInSystems(theSystem, theCode);
2379                }
2380                TermCodeSystemVersion csv = cs.getCurrentVersion();
2381
2382                Set<TermConcept> codes = findCodesAbove(cs.getResource().getId(), csv.getPid(), theCode);
2383                return toVersionIndependentConcepts(theSystem, codes);
2384        }
2385
2386        @Transactional(propagation = Propagation.REQUIRED, readOnly = true)
2387        @Override
2388        public Set<TermConcept> findCodesBelow(
2389                        Long theCodeSystemResourcePid, Long theCodeSystemVersionPid, String theCode) {
2390                Stopwatch stopwatch = Stopwatch.createStarted();
2391
2392                Optional<TermConcept> concept = fetchLoadedCode(theCodeSystemResourcePid, theCode);
2393                if (concept.isEmpty()) {
2394                        return Collections.emptySet();
2395                }
2396
2397                Set<TermConcept> retVal = new HashSet<>();
2398                retVal.add(concept.get());
2399
2400                fetchChildren(concept.get(), retVal);
2401
2402                ourLog.debug(
2403                                "Fetched {} codes below code {} in {}ms",
2404                                retVal.size(),
2405                                theCode,
2406                                stopwatch.elapsed(TimeUnit.MILLISECONDS));
2407                return retVal;
2408        }
2409
2410        @Transactional(readOnly = true)
2411        @Override
2412        public List<FhirVersionIndependentConcept> findCodesBelow(String theSystem, String theCode) {
2413                TermCodeSystem cs = getCodeSystem(theSystem);
2414                if (cs == null) {
2415                        return findCodesBelowUsingBuiltInSystems(theSystem, theCode);
2416                }
2417                TermCodeSystemVersion csv = cs.getCurrentVersion();
2418
2419                Set<TermConcept> codes = findCodesBelow(cs.getResource().getId(), csv.getPid(), theCode);
2420                return toVersionIndependentConcepts(theSystem, codes);
2421        }
2422
2423        private TermCodeSystem getCodeSystem(String theSystem) {
2424                return myCodeSystemDao.findByCodeSystemUri(theSystem);
2425        }
2426
2427        @PostConstruct
2428        public void start() {
2429                RuleBasedTransactionAttribute rules = new RuleBasedTransactionAttribute();
2430                rules.getRollbackRules().add(new NoRollbackRuleAttribute(ExpansionTooCostlyException.class));
2431                myTxTemplate = new TransactionTemplate(myTransactionManager, rules);
2432        }
2433
2434        @Override
2435        public void scheduleJobs(ISchedulerService theSchedulerService) {
2436                // Register scheduled job to pre-expand ValueSets
2437                // In the future it would be great to make this a cluster-aware task somehow
2438                ScheduledJobDefinition vsJobDefinition = new ScheduledJobDefinition();
2439                vsJobDefinition.setId(getClass().getName());
2440                vsJobDefinition.setJobClass(Job.class);
2441                theSchedulerService.scheduleClusteredJob(10 * DateUtils.MILLIS_PER_MINUTE, vsJobDefinition);
2442        }
2443
2444        @Override
2445        public synchronized void preExpandDeferredValueSetsToTerminologyTables() {
2446                if (!myStorageSettings.isEnableTaskPreExpandValueSets()) {
2447                        return;
2448                }
2449                if (isNotSafeToPreExpandValueSets()) {
2450                        ourLog.info("Skipping scheduled pre-expansion of ValueSets while deferred entities are being loaded.");
2451                        return;
2452                }
2453                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
2454
2455                while (true) {
2456                        StopWatch sw = new StopWatch();
2457                        TermValueSet valueSetToExpand = txTemplate.execute(t -> {
2458                                Optional<TermValueSet> optionalTermValueSet = getNextTermValueSetNotExpanded();
2459                                if (optionalTermValueSet.isEmpty()) {
2460                                        return null;
2461                                }
2462
2463                                TermValueSet termValueSet = optionalTermValueSet.get();
2464                                termValueSet.setTotalConcepts(0L);
2465                                termValueSet.setTotalConceptDesignations(0L);
2466                                termValueSet.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANSION_IN_PROGRESS);
2467                                return myTermValueSetDao.saveAndFlush(termValueSet);
2468                        });
2469                        if (valueSetToExpand == null) {
2470                                return;
2471                        }
2472
2473                        // We have a ValueSet to pre-expand.
2474                        setPreExpandingValueSets(true);
2475                        try {
2476                                ValueSet valueSet = txTemplate.execute(t -> {
2477                                        TermValueSet refreshedValueSetToExpand = myTermValueSetDao
2478                                                        .findById(valueSetToExpand.getId())
2479                                                        .orElseThrow(() -> new IllegalStateException("Unknown VS ID: " + valueSetToExpand.getId()));
2480                                        return getValueSetFromResourceTable(refreshedValueSetToExpand.getResource());
2481                                });
2482                                assert valueSet != null;
2483
2484                                ValueSetConceptAccumulator valueSetConceptAccumulator =
2485                                                myValueSetConceptAccumulatorFactory.create(valueSetToExpand);
2486                                ValueSetExpansionOptions options = new ValueSetExpansionOptions();
2487                                options.setIncludeHierarchy(true);
2488                                expandValueSet(options, valueSet, valueSetConceptAccumulator);
2489
2490                                // We are done with this ValueSet.
2491                                txTemplate.executeWithoutResult(t -> {
2492                                        valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.EXPANDED);
2493                                        valueSetToExpand.setExpansionTimestamp(new Date());
2494                                        myTermValueSetDao.saveAndFlush(valueSetToExpand);
2495                                });
2496
2497                                afterValueSetExpansionStatusChange();
2498
2499                                ourLog.info(
2500                                                "Pre-expanded ValueSet[{}] with URL[{}] - Saved {} concepts in {}",
2501                                                valueSet.getId(),
2502                                                valueSet.getUrl(),
2503                                                valueSetConceptAccumulator.getConceptsSaved(),
2504                                                sw);
2505
2506                        } catch (Exception e) {
2507                                ourLog.error(
2508                                                "Failed to pre-expand ValueSet with URL[{}]: {}", valueSetToExpand.getUrl(), e.getMessage(), e);
2509                                txTemplate.executeWithoutResult(t -> {
2510                                        valueSetToExpand.setExpansionStatus(TermValueSetPreExpansionStatusEnum.FAILED_TO_EXPAND);
2511                                        myTermValueSetDao.saveAndFlush(valueSetToExpand);
2512                                });
2513
2514                        } finally {
2515                                setPreExpandingValueSets(false);
2516                        }
2517                }
2518        }
2519
2520        /*
2521         * If a ValueSet has just finished pre-expanding, let's flush the caches. This is
2522         * kind of a blunt tool, but it should ensure that users don't get unpredictable
2523         * results while they test changes, which is probably a worthwhile sacrifice
2524         */
2525        private void afterValueSetExpansionStatusChange() {
2526                provideValidationSupport().invalidateCaches();
2527        }
2528
2529        @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
2530        @Override
2531        public void invalidateCaches() {
2532                /*
2533                 * Clear out anything left in the userdata caches. We do this mostly because it messes
2534                 * up unit tests to have these things stick around between test runs, since many of
2535                 * these resources come from DefaultProfileValidationSupport and therefore live beyond
2536                 * any single test execution.
2537                 */
2538                for (IBaseResource next : provideValidationSupport().fetchAllConformanceResources()) {
2539                        if (next != null) {
2540                                synchronized (next) {
2541                                        if (next.getUserData(CS_USERDATA_CURRENT_VERSION) != null) {
2542                                                next.setUserData(CS_USERDATA_CURRENT_VERSION, null);
2543                                        }
2544                                        if (next.getUserData(VS_USERDATA_CURRENT_VERSION) != null) {
2545                                                next.setUserData(VS_USERDATA_CURRENT_VERSION, null);
2546                                        }
2547                                }
2548                        }
2549                }
2550        }
2551
2552        private synchronized boolean isPreExpandingValueSets() {
2553                return myPreExpandingValueSets;
2554        }
2555
2556        private synchronized void setPreExpandingValueSets(boolean thePreExpandingValueSets) {
2557                myPreExpandingValueSets = thePreExpandingValueSets;
2558        }
2559
2560        private boolean isNotSafeToPreExpandValueSets() {
2561                return myDeferredStorageSvc != null && !myDeferredStorageSvc.isStorageQueueEmpty(true);
2562        }
2563
2564        private Optional<TermValueSet> getNextTermValueSetNotExpanded() {
2565                Optional<TermValueSet> retVal = Optional.empty();
2566                Slice<TermValueSet> page = myTermValueSetDao.findByExpansionStatus(
2567                                PageRequest.of(0, 1), TermValueSetPreExpansionStatusEnum.NOT_EXPANDED);
2568
2569                if (!page.getContent().isEmpty()) {
2570                        retVal = Optional.of(page.getContent().get(0));
2571                }
2572
2573                return retVal;
2574        }
2575
2576        @Override
2577        @Transactional
2578        public void storeTermValueSet(ResourceTable theResourceTable, ValueSet theValueSet) {
2579                // If we're in a transaction, we need to flush now so that we can correctly detect
2580                // duplicates if there are multiple ValueSets in the same TX with the same URL
2581                // (which is an error, but we need to catch it). It'd be better to catch this by
2582                // inspecting the URLs in the bundle or something, since flushing hurts performance
2583                // but it's not expected that loading valuesets is going to be a huge high frequency
2584                // thing so it probably doesn't matter
2585                myEntityManager.flush();
2586
2587                ValidateUtil.isTrueOrThrowInvalidRequest(theResourceTable != null, "No resource supplied");
2588                if (isPlaceholder(theValueSet)) {
2589                        ourLog.info(
2590                                        "Not storing TermValueSet for placeholder {}",
2591                                        theValueSet.getIdElement().toVersionless().getValueAsString());
2592                        return;
2593                }
2594
2595                ValidateUtil.isNotBlankOrThrowUnprocessableEntity(
2596                                theValueSet.getUrl(), "ValueSet has no value for ValueSet.url");
2597                ourLog.info(
2598                                "Storing TermValueSet for {}",
2599                                theValueSet.getIdElement().toVersionless().getValueAsString());
2600
2601                /*
2602                 * Get CodeSystem and validate CodeSystemVersion
2603                 */
2604                TermValueSet termValueSet = new TermValueSet();
2605                termValueSet.setResource(theResourceTable);
2606                termValueSet.setUrl(theValueSet.getUrl());
2607                termValueSet.setVersion(theValueSet.getVersion());
2608                termValueSet.setName(theValueSet.hasName() ? theValueSet.getName() : null);
2609
2610                // Delete version being replaced
2611                Optional<TermValueSet> deletedTrmValueSet = deleteValueSetForResource(theResourceTable);
2612
2613                /*
2614                 * Do the upload.
2615                 */
2616                String url = termValueSet.getUrl();
2617                String version = termValueSet.getVersion();
2618                Optional<TermValueSet> optionalExistingTermValueSetByUrl;
2619
2620                if (deletedTrmValueSet.isPresent()
2621                                && Objects.equals(deletedTrmValueSet.get().getUrl(), url)
2622                                && Objects.equals(deletedTrmValueSet.get().getVersion(), version)) {
2623                        // If we just deleted the valueset marker, we don't need to check if it exists
2624                        // in the database
2625                        optionalExistingTermValueSetByUrl = Optional.empty();
2626                } else {
2627                        optionalExistingTermValueSetByUrl = getTermValueSet(version, url);
2628                }
2629
2630                if (optionalExistingTermValueSetByUrl.isEmpty()) {
2631
2632                        myTermValueSetDao.save(termValueSet);
2633
2634                } else {
2635                        TermValueSet existingTermValueSet = optionalExistingTermValueSetByUrl.get();
2636                        String msg;
2637                        if (version != null) {
2638                                msg = myContext
2639                                                .getLocalizer()
2640                                                .getMessage(
2641                                                                TermReadSvcImpl.class,
2642                                                                "cannotCreateDuplicateValueSetUrlAndVersion",
2643                                                                url,
2644                                                                version,
2645                                                                existingTermValueSet
2646                                                                                .getResource()
2647                                                                                .getIdDt()
2648                                                                                .toUnqualifiedVersionless()
2649                                                                                .getValue());
2650                        } else {
2651                                msg = myContext
2652                                                .getLocalizer()
2653                                                .getMessage(
2654                                                                TermReadSvcImpl.class,
2655                                                                "cannotCreateDuplicateValueSetUrl",
2656                                                                url,
2657                                                                existingTermValueSet
2658                                                                                .getResource()
2659                                                                                .getIdDt()
2660                                                                                .toUnqualifiedVersionless()
2661                                                                                .getValue());
2662                        }
2663                        throw new UnprocessableEntityException(Msg.code(902) + msg);
2664                }
2665        }
2666
2667        private Optional<TermValueSet> getTermValueSet(String version, String url) {
2668                Optional<TermValueSet> optionalExistingTermValueSetByUrl;
2669                if (version != null) {
2670                        optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndVersion(url, version);
2671                } else {
2672                        optionalExistingTermValueSetByUrl = myTermValueSetDao.findTermValueSetByUrlAndNullVersion(url);
2673                }
2674                return optionalExistingTermValueSetByUrl;
2675        }
2676
2677        @Override
2678        @Transactional
2679        public IFhirResourceDaoCodeSystem.SubsumesResult subsumes(
2680                        IPrimitiveType<String> theCodeA,
2681                        IPrimitiveType<String> theCodeB,
2682                        IPrimitiveType<String> theSystem,
2683                        IBaseCoding theCodingA,
2684                        IBaseCoding theCodingB) {
2685                FhirVersionIndependentConcept conceptA = toConcept(theCodeA, theSystem, theCodingA);
2686                FhirVersionIndependentConcept conceptB = toConcept(theCodeB, theSystem, theCodingB);
2687
2688                if (!StringUtils.equals(conceptA.getSystem(), conceptB.getSystem())) {
2689                        throw new InvalidRequestException(
2690                                        Msg.code(903) + "Unable to test subsumption across different code systems");
2691                }
2692
2693                if (!StringUtils.equals(conceptA.getSystemVersion(), conceptB.getSystemVersion())) {
2694                        throw new InvalidRequestException(
2695                                        Msg.code(904) + "Unable to test subsumption across different code system versions");
2696                }
2697
2698                String codeASystemIdentifier;
2699                if (StringUtils.isNotEmpty(conceptA.getSystemVersion())) {
2700                        codeASystemIdentifier = conceptA.getSystem() + OUR_PIPE_CHARACTER + conceptA.getSystemVersion();
2701                } else {
2702                        codeASystemIdentifier = conceptA.getSystem();
2703                }
2704                TermConcept codeA = findCode(codeASystemIdentifier, conceptA.getCode())
2705                                .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptA));
2706
2707                String codeBSystemIdentifier;
2708                if (StringUtils.isNotEmpty(conceptB.getSystemVersion())) {
2709                        codeBSystemIdentifier = conceptB.getSystem() + OUR_PIPE_CHARACTER + conceptB.getSystemVersion();
2710                } else {
2711                        codeBSystemIdentifier = conceptB.getSystem();
2712                }
2713                TermConcept codeB = findCode(codeBSystemIdentifier, conceptB.getCode())
2714                                .orElseThrow(() -> new InvalidRequestException("Unknown code: " + conceptB));
2715
2716                SearchSession searchSession = Search.session(myEntityManager);
2717
2718                ConceptSubsumptionOutcome subsumes;
2719                subsumes = testForSubsumption(searchSession, codeA, codeB, ConceptSubsumptionOutcome.SUBSUMES);
2720                if (subsumes == null) {
2721                        subsumes = testForSubsumption(searchSession, codeB, codeA, ConceptSubsumptionOutcome.SUBSUMEDBY);
2722                }
2723                if (subsumes == null) {
2724                        subsumes = ConceptSubsumptionOutcome.NOTSUBSUMED;
2725                }
2726
2727                return new IFhirResourceDaoCodeSystem.SubsumesResult(subsumes);
2728        }
2729
2730        @Override
2731        public IValidationSupport.LookupCodeResult lookupCode(
2732                        ValidationSupportContext theValidationSupportContext, @Nonnull LookupCodeRequest theLookupCodeRequest) {
2733                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
2734                return txTemplate.execute(t -> {
2735                        final String theSystem = theLookupCodeRequest.getSystem();
2736                        final String theCode = theLookupCodeRequest.getCode();
2737                        Optional<TermConcept> codeOpt = findCode(theSystem, theCode);
2738                        if (codeOpt.isPresent()) {
2739                                TermConcept code = codeOpt.get();
2740
2741                                IValidationSupport.LookupCodeResult result = new IValidationSupport.LookupCodeResult();
2742                                result.setCodeSystemDisplayName(code.getCodeSystemVersion().getCodeSystemDisplayName());
2743                                result.setCodeSystemVersion(code.getCodeSystemVersion().getCodeSystemVersionId());
2744                                result.setSearchedForSystem(theSystem);
2745                                result.setSearchedForCode(theCode);
2746                                result.setFound(true);
2747                                result.setCodeDisplay(code.getDisplay());
2748
2749                                for (TermConceptDesignation next : code.getDesignations()) {
2750                                        // filter out the designation based on displayLanguage if any
2751                                        if (isDisplayLanguageMatch(theLookupCodeRequest.getDisplayLanguage(), next.getLanguage())) {
2752                                                IValidationSupport.ConceptDesignation designation = new IValidationSupport.ConceptDesignation();
2753                                                designation.setLanguage(next.getLanguage());
2754                                                designation.setUseSystem(next.getUseSystem());
2755                                                designation.setUseCode(next.getUseCode());
2756                                                designation.setUseDisplay(next.getUseDisplay());
2757                                                designation.setValue(next.getValue());
2758                                                result.getDesignations().add(designation);
2759                                        }
2760                                }
2761
2762                                final Collection<String> propertyNames = theLookupCodeRequest.getPropertyNames();
2763                                for (TermConceptProperty next : code.getProperties()) {
2764                                        if (ObjectUtils.isNotEmpty(propertyNames) && !propertyNames.contains(next.getKey())) {
2765                                                continue;
2766                                        }
2767                                        if (next.getType() == TermConceptPropertyTypeEnum.CODING) {
2768                                                IValidationSupport.CodingConceptProperty property =
2769                                                                new IValidationSupport.CodingConceptProperty(
2770                                                                                next.getKey(), next.getCodeSystem(), next.getValue(), next.getDisplay());
2771                                                result.getProperties().add(property);
2772                                        } else if (next.getType() == TermConceptPropertyTypeEnum.STRING) {
2773                                                IValidationSupport.StringConceptProperty property =
2774                                                                new IValidationSupport.StringConceptProperty(next.getKey(), next.getValue());
2775                                                result.getProperties().add(property);
2776                                        } else {
2777                                                throw new InternalErrorException(Msg.code(905) + "Unknown type: " + next.getType());
2778                                        }
2779                                }
2780
2781                                return result;
2782
2783                        } else {
2784                                return new LookupCodeResult().setFound(false);
2785                        }
2786                });
2787        }
2788
2789        @Nullable
2790        private ConceptSubsumptionOutcome testForSubsumption(
2791                        SearchSession theSearchSession,
2792                        TermConcept theLeft,
2793                        TermConcept theRight,
2794                        ConceptSubsumptionOutcome theOutput) {
2795                List<TermConcept> fetch = theSearchSession
2796                                .search(TermConcept.class)
2797                                .where(f -> f.bool()
2798                                                .must(f.match().field("myId").matching(theRight.getId()))
2799                                                .must(f.match().field("myParentPids").matching(Long.toString(theLeft.getId()))))
2800                                .fetchHits(1);
2801
2802                if (fetch.size() > 0) {
2803                        return theOutput;
2804                } else {
2805                        return null;
2806                }
2807        }
2808
2809        private ArrayList<FhirVersionIndependentConcept> toVersionIndependentConcepts(
2810                        String theSystem, Set<TermConcept> codes) {
2811                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>(codes.size());
2812                for (TermConcept next : codes) {
2813                        retVal.add(new FhirVersionIndependentConcept(theSystem, next.getCode()));
2814                }
2815                return retVal;
2816        }
2817
2818        @Override
2819        @Transactional
2820        public CodeValidationResult validateCodeInValueSet(
2821                        ValidationSupportContext theValidationSupportContext,
2822                        ConceptValidationOptions theOptions,
2823                        String theCodeSystem,
2824                        String theCode,
2825                        String theDisplay,
2826                        @Nonnull IBaseResource theValueSet) {
2827                invokeRunnableForUnitTest();
2828
2829                IPrimitiveType<?> urlPrimitive;
2830                if (theValueSet instanceof org.hl7.fhir.dstu2.model.ValueSet) {
2831                        urlPrimitive = FhirContext.forDstu2Hl7OrgCached()
2832                                        .newTerser()
2833                                        .getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class);
2834                } else {
2835                        urlPrimitive = myContext.newTerser().getSingleValueOrNull(theValueSet, "url", IPrimitiveType.class);
2836                }
2837                String url = urlPrimitive.getValueAsString();
2838                if (isNotBlank(url)) {
2839                        return validateCode(theValidationSupportContext, theOptions, theCodeSystem, theCode, theDisplay, url);
2840                }
2841                return null;
2842        }
2843
2844        @CoverageIgnore
2845        @Override
2846        public IValidationSupport.CodeValidationResult validateCode(
2847                        @Nonnull ValidationSupportContext theValidationSupportContext,
2848                        @Nonnull ConceptValidationOptions theOptions,
2849                        String theCodeSystemUrl,
2850                        String theCode,
2851                        String theDisplay,
2852                        String theValueSetUrl) {
2853                // TODO GGG TRY TO JUST AUTO_PASS HERE AND SEE WHAT HAPPENS.
2854                invokeRunnableForUnitTest();
2855                theOptions.setValidateDisplay(isNotBlank(theDisplay));
2856
2857                if (isNotBlank(theValueSetUrl)) {
2858                        return validateCodeInValueSet(
2859                                        theValidationSupportContext, theOptions, theValueSetUrl, theCodeSystemUrl, theCode, theDisplay);
2860                }
2861
2862                TransactionTemplate txTemplate = new TransactionTemplate(myTransactionManager);
2863                txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED);
2864                txTemplate.setReadOnly(true);
2865                Optional<FhirVersionIndependentConcept> codeOpt =
2866                                txTemplate.execute(tx -> findCode(theCodeSystemUrl, theCode).map(c -> {
2867                                        String codeSystemVersionId = getCurrentCodeSystemVersion(
2868                                                                        theValidationSupportContext, theCodeSystemUrl)
2869                                                        .myCodeSystemVersionId;
2870                                        return new FhirVersionIndependentConcept(
2871                                                        theCodeSystemUrl, c.getCode(), c.getDisplay(), codeSystemVersionId);
2872                                }));
2873
2874                if (codeOpt != null && codeOpt.isPresent()) {
2875                        FhirVersionIndependentConcept code = codeOpt.get();
2876                        if (!theOptions.isValidateDisplay()
2877                                        || isBlank(code.getDisplay())
2878                                        || isBlank(theDisplay)
2879                                        || code.getDisplay().equals(theDisplay)) {
2880                                return new CodeValidationResult().setCode(code.getCode()).setDisplay(code.getDisplay());
2881                        } else {
2882                                return InMemoryTerminologyServerValidationSupport.createResultForDisplayMismatch(
2883                                                myContext,
2884                                                theCode,
2885                                                theDisplay,
2886                                                code.getDisplay(),
2887                                                code.getSystem(),
2888                                                code.getSystemVersion(),
2889                                                myStorageSettings.getIssueSeverityForCodeDisplayMismatch());
2890                        }
2891                }
2892
2893                return createFailureCodeValidationResult(
2894                                theCodeSystemUrl, theCode, null, createMessageAppendForCodeNotFoundInCodeSystem(theCodeSystemUrl));
2895        }
2896
2897        IValidationSupport.CodeValidationResult validateCodeInValueSet(
2898                        ValidationSupportContext theValidationSupportContext,
2899                        ConceptValidationOptions theValidationOptions,
2900                        String theValueSetUrl,
2901                        String theCodeSystem,
2902                        String theCode,
2903                        String theDisplay) {
2904                IBaseResource valueSet =
2905                                theValidationSupportContext.getRootValidationSupport().fetchValueSet(theValueSetUrl);
2906                CodeValidationResult retVal = null;
2907
2908                // If we don't have a PID, this came from some source other than the JPA
2909                // database, so we don't need to check if it's pre-expanded or not
2910                if (valueSet instanceof IAnyResource) {
2911                        Long pid = IDao.RESOURCE_PID.get(valueSet);
2912                        if (pid != null) {
2913                                TransactionTemplate txTemplate = new TransactionTemplate(myTxManager);
2914                                retVal = txTemplate.execute(tx -> {
2915                                        if (isValueSetPreExpandedForCodeValidation(valueSet)) {
2916                                                return validateCodeIsInPreExpandedValueSet(
2917                                                                theValidationSupportContext,
2918                                                                theValidationOptions,
2919                                                                valueSet,
2920                                                                theCodeSystem,
2921                                                                theCode,
2922                                                                theDisplay,
2923                                                                null,
2924                                                                null);
2925                                        } else {
2926                                                return null;
2927                                        }
2928                                });
2929                        }
2930                }
2931
2932                if (retVal == null) {
2933                        if (valueSet != null) {
2934                                retVal = myInMemoryTerminologyServerValidationSupport.validateCodeInValueSet(
2935                                                theValidationSupportContext,
2936                                                theValidationOptions,
2937                                                theCodeSystem,
2938                                                theCode,
2939                                                theDisplay,
2940                                                valueSet);
2941                        } else {
2942                                String append = " - Unable to locate ValueSet[" + theValueSetUrl + "]";
2943                                retVal = createFailureCodeValidationResult(theCodeSystem, theCode, null, append);
2944                        }
2945                }
2946
2947                // Check if someone is accidentally using a VS url where it should be a CS URL
2948                if (retVal != null
2949                                && retVal.getCode() == null
2950                                && theCodeSystem != null
2951                                && myContext.getVersion().getVersion().isNewerThan(FhirVersionEnum.DSTU2)) {
2952                        if (isValueSetSupported(theValidationSupportContext, theCodeSystem)) {
2953                                if (!isCodeSystemSupported(theValidationSupportContext, theCodeSystem)) {
2954                                        String newMessage = "Unable to validate code " + theCodeSystem + "#" + theCode
2955                                                        + " - Supplied system URL is a ValueSet URL and not a CodeSystem URL, check if it is correct: "
2956                                                        + theCodeSystem;
2957                                        retVal.setMessage(newMessage);
2958                                }
2959                        }
2960                }
2961
2962                return retVal;
2963        }
2964
2965        @Override
2966        public CodeSystem fetchCanonicalCodeSystemFromCompleteContext(String theSystem) {
2967                IValidationSupport validationSupport = provideValidationSupport();
2968                IBaseResource codeSystem = validationSupport.fetchCodeSystem(theSystem);
2969                if (codeSystem != null) {
2970                        codeSystem = myVersionCanonicalizer.codeSystemToCanonical(codeSystem);
2971                }
2972                return (CodeSystem) codeSystem;
2973        }
2974
2975        @Nonnull
2976        private IValidationSupport provideJpaValidationSupport() {
2977                IValidationSupport jpaValidationSupport = myJpaValidationSupport;
2978                if (jpaValidationSupport == null) {
2979                        jpaValidationSupport = myApplicationContext.getBean("myJpaValidationSupport", IValidationSupport.class);
2980                        myJpaValidationSupport = jpaValidationSupport;
2981                }
2982                return jpaValidationSupport;
2983        }
2984
2985        @Nonnull
2986        protected IValidationSupport provideValidationSupport() {
2987                IValidationSupport validationSupport = myValidationSupport;
2988                if (validationSupport == null) {
2989                        validationSupport = myApplicationContext.getBean(IValidationSupport.class);
2990                        myValidationSupport = validationSupport;
2991                }
2992                return validationSupport;
2993        }
2994
2995        public ValueSet fetchCanonicalValueSetFromCompleteContext(String theSystem) {
2996                IValidationSupport validationSupport = provideValidationSupport();
2997                IBaseResource valueSet = validationSupport.fetchValueSet(theSystem);
2998                if (valueSet != null) {
2999                        valueSet = myVersionCanonicalizer.valueSetToCanonical(valueSet);
3000                }
3001                return (ValueSet) valueSet;
3002        }
3003
3004        @Override
3005        public IBaseResource fetchValueSet(String theValueSetUrl) {
3006                return provideJpaValidationSupport().fetchValueSet(theValueSetUrl);
3007        }
3008
3009        @Override
3010        public FhirContext getFhirContext() {
3011                return myContext;
3012        }
3013
3014        private void findCodesAbove(
3015                        CodeSystem theSystem,
3016                        String theSystemString,
3017                        String theCode,
3018                        List<FhirVersionIndependentConcept> theListToPopulate) {
3019                List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept();
3020                for (CodeSystem.ConceptDefinitionComponent next : conceptList) {
3021                        addTreeIfItContainsCode(theSystemString, next, theCode, theListToPopulate);
3022                }
3023        }
3024
3025        @Override
3026        public List<FhirVersionIndependentConcept> findCodesAboveUsingBuiltInSystems(String theSystem, String theCode) {
3027                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
3028                CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem);
3029                if (system != null) {
3030                        findCodesAbove(system, theSystem, theCode, retVal);
3031                }
3032                return retVal;
3033        }
3034
3035        private void findCodesBelow(
3036                        CodeSystem theSystem,
3037                        String theSystemString,
3038                        String theCode,
3039                        List<FhirVersionIndependentConcept> theListToPopulate) {
3040                List<CodeSystem.ConceptDefinitionComponent> conceptList = theSystem.getConcept();
3041                findCodesBelow(theSystemString, theCode, theListToPopulate, conceptList);
3042        }
3043
3044        private void findCodesBelow(
3045                        String theSystemString,
3046                        String theCode,
3047                        List<FhirVersionIndependentConcept> theListToPopulate,
3048                        List<CodeSystem.ConceptDefinitionComponent> conceptList) {
3049                for (CodeSystem.ConceptDefinitionComponent next : conceptList) {
3050                        if (theCode.equals(next.getCode())) {
3051                                addAllChildren(theSystemString, next, theListToPopulate);
3052                        } else {
3053                                findCodesBelow(theSystemString, theCode, theListToPopulate, next.getConcept());
3054                        }
3055                }
3056        }
3057
3058        @Override
3059        public List<FhirVersionIndependentConcept> findCodesBelowUsingBuiltInSystems(String theSystem, String theCode) {
3060                ArrayList<FhirVersionIndependentConcept> retVal = new ArrayList<>();
3061                CodeSystem system = fetchCanonicalCodeSystemFromCompleteContext(theSystem);
3062                if (system != null) {
3063                        findCodesBelow(system, theSystem, theCode, retVal);
3064                }
3065                return retVal;
3066        }
3067
3068        private void addAllChildren(
3069                        String theSystemString,
3070                        CodeSystem.ConceptDefinitionComponent theCode,
3071                        List<FhirVersionIndependentConcept> theListToPopulate) {
3072                if (isNotBlank(theCode.getCode())) {
3073                        theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theCode.getCode()));
3074                }
3075                for (CodeSystem.ConceptDefinitionComponent nextChild : theCode.getConcept()) {
3076                        addAllChildren(theSystemString, nextChild, theListToPopulate);
3077                }
3078        }
3079
3080        private boolean addTreeIfItContainsCode(
3081                        String theSystemString,
3082                        CodeSystem.ConceptDefinitionComponent theNext,
3083                        String theCode,
3084                        List<FhirVersionIndependentConcept> theListToPopulate) {
3085                boolean foundCodeInChild = false;
3086                for (CodeSystem.ConceptDefinitionComponent nextChild : theNext.getConcept()) {
3087                        foundCodeInChild |= addTreeIfItContainsCode(theSystemString, nextChild, theCode, theListToPopulate);
3088                }
3089
3090                if (theCode.equals(theNext.getCode()) || foundCodeInChild) {
3091                        theListToPopulate.add(new FhirVersionIndependentConcept(theSystemString, theNext.getCode()));
3092                        return true;
3093                }
3094
3095                return false;
3096        }
3097
3098        @Nonnull
3099        private FhirVersionIndependentConcept toConcept(
3100                        IPrimitiveType<String> theCodeType,
3101                        IPrimitiveType<String> theCodeSystemIdentifierType,
3102                        IBaseCoding theCodingType) {
3103                String code = theCodeType != null ? theCodeType.getValueAsString() : null;
3104                String system = theCodeSystemIdentifierType != null
3105                                ? getUrlFromIdentifier(theCodeSystemIdentifierType.getValueAsString())
3106                                : null;
3107                String systemVersion = theCodeSystemIdentifierType != null
3108                                ? getVersionFromIdentifier(theCodeSystemIdentifierType.getValueAsString())
3109                                : null;
3110                if (theCodingType != null) {
3111                        Coding canonicalizedCoding = myVersionCanonicalizer.codingToCanonical(theCodingType);
3112                        assert canonicalizedCoding != null; // Shouldn't be null, since theCodingType isn't
3113                        code = canonicalizedCoding.getCode();
3114                        system = canonicalizedCoding.getSystem();
3115                        systemVersion = canonicalizedCoding.getVersion();
3116                }
3117                return new FhirVersionIndependentConcept(system, code, null, systemVersion);
3118        }
3119
3120        /**
3121         * When the search is for unversioned loinc system it uses the forcedId to obtain the current
3122         * version, as it is not necessarily the last  one anymore.
3123         * For other cases it keeps on considering the last uploaded as the current
3124         */
3125        @Override
3126        public Optional<TermValueSet> findCurrentTermValueSet(String theUrl) {
3127                if (TermReadSvcUtil.isLoincUnversionedValueSet(theUrl)) {
3128                        Optional<String> vsIdOpt = TermReadSvcUtil.getValueSetId(theUrl);
3129                        if (vsIdOpt.isEmpty()) {
3130                                return Optional.empty();
3131                        }
3132
3133                        return myTermValueSetDao.findTermValueSetByForcedId(vsIdOpt.get());
3134                }
3135
3136                List<TermValueSet> termValueSetList = myTermValueSetDao.findTermValueSetByUrl(Pageable.ofSize(1), theUrl);
3137                if (termValueSetList.isEmpty()) {
3138                        return Optional.empty();
3139                }
3140
3141                return Optional.of(termValueSetList.get(0));
3142        }
3143
3144        @Override
3145        public Optional<IBaseResource> readCodeSystemByForcedId(String theForcedId) {
3146                @SuppressWarnings("unchecked")
3147                List<ResourceTable> resultList = (List<ResourceTable>) myEntityManager
3148                                .createQuery("select r from ResourceTable r "
3149                                                + "where r.myResourceType = 'CodeSystem' and r.myFhirId = :fhirId")
3150                                .setParameter("fhirId", theForcedId)
3151                                .getResultList();
3152                if (resultList.isEmpty()) return Optional.empty();
3153
3154                if (resultList.size() > 1)
3155                        throw new NonUniqueResultException(Msg.code(911) + "More than one CodeSystem is pointed by forcedId: "
3156                                        + theForcedId + ". Was constraint " + ResourceTable.IDX_RES_TYPE_FHIR_ID + " removed?");
3157
3158                IFhirResourceDao<CodeSystem> csDao = myDaoRegistry.getResourceDao("CodeSystem");
3159                IBaseResource cs = myJpaStorageResourceParser.toResource(resultList.get(0), false);
3160                return Optional.of(cs);
3161        }
3162
3163        @Transactional
3164        @Override
3165        public ReindexTerminologyResult reindexTerminology() throws InterruptedException {
3166                if (myFulltextSearchSvc == null) {
3167                        return ReindexTerminologyResult.SEARCH_SVC_DISABLED;
3168                }
3169
3170                if (isBatchTerminologyTasksRunning()) {
3171                        return ReindexTerminologyResult.OTHER_BATCH_TERMINOLOGY_TASKS_RUNNING;
3172                }
3173
3174                // disallow pre-expanding ValueSets while reindexing
3175                myDeferredStorageSvc.setProcessDeferred(false);
3176
3177                int objectLoadingThreadNumber = calculateObjectLoadingThreadNumber();
3178                ourLog.info("Using {} threads to load objects", objectLoadingThreadNumber);
3179
3180                try {
3181                        SearchSession searchSession = getSearchSession();
3182                        searchSession
3183                                        .massIndexer(TermConcept.class)
3184                                        .dropAndCreateSchemaOnStart(true)
3185                                        .purgeAllOnStart(false)
3186                                        .batchSizeToLoadObjects(100)
3187                                        .cacheMode(CacheMode.IGNORE)
3188                                        .threadsToLoadObjects(6)
3189                                        .transactionTimeout(60 * SECONDS_IN_MINUTE)
3190                                        .monitor(new PojoMassIndexingLoggingMonitor(INDEXED_ROOTS_LOGGING_COUNT))
3191                                        .startAndWait();
3192                } finally {
3193                        myDeferredStorageSvc.setProcessDeferred(true);
3194                }
3195
3196                return ReindexTerminologyResult.SUCCESS;
3197        }
3198
3199        @VisibleForTesting
3200        boolean isBatchTerminologyTasksRunning() {
3201                return isNotSafeToPreExpandValueSets() || isPreExpandingValueSets();
3202        }
3203
3204        @VisibleForTesting
3205        int calculateObjectLoadingThreadNumber() {
3206                IConnectionPoolInfoProvider connectionPoolInfoProvider =
3207                                new ConnectionPoolInfoProvider(myHibernatePropertiesProvider.getDataSource());
3208                Optional<Integer> maxConnectionsOpt = connectionPoolInfoProvider.getTotalConnectionSize();
3209                if (maxConnectionsOpt.isEmpty()) {
3210                        return DEFAULT_MASS_INDEXER_OBJECT_LOADING_THREADS;
3211                }
3212
3213                int maxConnections = maxConnectionsOpt.get();
3214                int usableThreads = maxConnections < 6 ? 1 : maxConnections - 5;
3215                int objectThreads = Math.min(usableThreads, MAX_MASS_INDEXER_OBJECT_LOADING_THREADS);
3216                ourLog.debug(
3217                                "Data source connection pool has {} connections allocated, so reindexing will use {} object "
3218                                                + "loading threads (each using a connection)",
3219                                maxConnections,
3220                                objectThreads);
3221                return objectThreads;
3222        }
3223
3224        @VisibleForTesting
3225        SearchSession getSearchSession() {
3226                return Search.session(myEntityManager);
3227        }
3228
3229        @Override
3230        public ValueSetExpansionOutcome expandValueSet(
3231                        ValidationSupportContext theValidationSupportContext,
3232                        ValueSetExpansionOptions theExpansionOptions,
3233                        @Nonnull IBaseResource theValueSetToExpand) {
3234                ValueSet canonicalInput = myVersionCanonicalizer.valueSetToCanonical(theValueSetToExpand);
3235                org.hl7.fhir.r4.model.ValueSet expandedR4 = expandValueSet(theExpansionOptions, canonicalInput);
3236                return new ValueSetExpansionOutcome(myVersionCanonicalizer.valueSetFromCanonical(expandedR4));
3237        }
3238
3239        @Override
3240        public IBaseResource expandValueSet(ValueSetExpansionOptions theExpansionOptions, IBaseResource theInput) {
3241                org.hl7.fhir.r4.model.ValueSet valueSetToExpand = myVersionCanonicalizer.valueSetToCanonical(theInput);
3242                org.hl7.fhir.r4.model.ValueSet valueSetR4 = expandValueSet(theExpansionOptions, valueSetToExpand);
3243                return myVersionCanonicalizer.valueSetFromCanonical(valueSetR4);
3244        }
3245
3246        @Override
3247        public void expandValueSet(
3248                        ValueSetExpansionOptions theExpansionOptions,
3249                        IBaseResource theValueSetToExpand,
3250                        IValueSetConceptAccumulator theValueSetCodeAccumulator) {
3251                org.hl7.fhir.r4.model.ValueSet valueSetToExpand =
3252                                myVersionCanonicalizer.valueSetToCanonical(theValueSetToExpand);
3253                expandValueSet(theExpansionOptions, valueSetToExpand, theValueSetCodeAccumulator);
3254        }
3255
3256        private org.hl7.fhir.r4.model.ValueSet getValueSetFromResourceTable(ResourceTable theResourceTable) {
3257                Class<? extends IBaseResource> type =
3258                                getFhirContext().getResourceDefinition("ValueSet").getImplementingClass();
3259                IBaseResource valueSet = myJpaStorageResourceParser.toResource(type, theResourceTable, null, false);
3260                return myVersionCanonicalizer.valueSetToCanonical(valueSet);
3261        }
3262
3263        @Override
3264        public CodeValidationResult validateCodeIsInPreExpandedValueSet(
3265                        ValidationSupportContext theValidationSupportContext,
3266                        ConceptValidationOptions theOptions,
3267                        IBaseResource theValueSet,
3268                        String theSystem,
3269                        String theCode,
3270                        String theDisplay,
3271                        IBaseDatatype theCoding,
3272                        IBaseDatatype theCodeableConcept) {
3273                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null");
3274                org.hl7.fhir.r4.model.ValueSet valueSetR4 = myVersionCanonicalizer.valueSetToCanonical(theValueSet);
3275                org.hl7.fhir.r4.model.Coding codingR4 = myVersionCanonicalizer.codingToCanonical((IBaseCoding) theCoding);
3276                org.hl7.fhir.r4.model.CodeableConcept codeableConcept =
3277                                myVersionCanonicalizer.codeableConceptToCanonical(theCodeableConcept);
3278
3279                return validateCodeIsInPreExpandedValueSet(
3280                                theValidationSupportContext,
3281                                theOptions,
3282                                valueSetR4,
3283                                theSystem,
3284                                theCode,
3285                                theDisplay,
3286                                codingR4,
3287                                codeableConcept);
3288        }
3289
3290        @Override
3291        public boolean isValueSetPreExpandedForCodeValidation(IBaseResource theValueSet) {
3292                ValidateUtil.isNotNullOrThrowUnprocessableEntity(theValueSet, "ValueSet must not be null");
3293                org.hl7.fhir.r4.model.ValueSet valueSetR4 = myVersionCanonicalizer.valueSetToCanonical(theValueSet);
3294                return isValueSetPreExpandedForCodeValidation(valueSetR4);
3295        }
3296
3297        private static class TermCodeSystemVersionDetails {
3298
3299                private final long myPid;
3300                private final String myCodeSystemVersionId;
3301
3302                public TermCodeSystemVersionDetails(long thePid, String theCodeSystemVersionId) {
3303                        myPid = thePid;
3304                        myCodeSystemVersionId = theCodeSystemVersionId;
3305                }
3306        }
3307
3308        public static class Job implements HapiJob {
3309                @Autowired
3310                private ITermReadSvc myTerminologySvc;
3311
3312                @Override
3313                public void execute(JobExecutionContext theContext) {
3314                        myTerminologySvc.preExpandDeferredValueSetsToTerminologyTables();
3315                }
3316        }
3317
3318        /**
3319         * Properties returned from method buildSearchScroll
3320         */
3321        private static final class SearchProperties {
3322                private final List<Supplier<SearchScroll<EntityReference>>> mySearchScroll = new ArrayList<>();
3323                private List<String> myIncludeOrExcludeCodes;
3324
3325                public List<Supplier<SearchScroll<EntityReference>>> getSearchScroll() {
3326                        return mySearchScroll;
3327                }
3328
3329                public void addSearchScroll(Supplier<SearchScroll<EntityReference>> theSearchScrollSupplier) {
3330                        mySearchScroll.add(theSearchScrollSupplier);
3331                }
3332
3333                public List<String> getIncludeOrExcludeCodes() {
3334                        return myIncludeOrExcludeCodes;
3335                }
3336
3337                public void setIncludeOrExcludeCodes(List<String> theIncludeOrExcludeCodes) {
3338                        myIncludeOrExcludeCodes = theIncludeOrExcludeCodes;
3339                }
3340
3341                public boolean hasIncludeOrExcludeCodes() {
3342                        return !myIncludeOrExcludeCodes.isEmpty();
3343                }
3344        }
3345
3346        static boolean isValueSetDisplayLanguageMatch(ValueSetExpansionOptions theExpansionOptions, String theStoredLang) {
3347                if (theExpansionOptions == null) {
3348                        return true;
3349                }
3350
3351                if (theExpansionOptions.getTheDisplayLanguage() == null || theStoredLang == null) {
3352                        return true;
3353                }
3354
3355                return theExpansionOptions.getTheDisplayLanguage().equalsIgnoreCase(theStoredLang);
3356        }
3357
3358        @Nonnull
3359        private static String createMessageAppendForCodeNotFoundInCodeSystem(String theCodeSystemUrl) {
3360                return " - Code is not found in CodeSystem: " + theCodeSystemUrl;
3361        }
3362
3363        @VisibleForTesting
3364        public static void setForceDisableHibernateSearchForUnitTest(boolean theForceDisableHibernateSearchForUnitTest) {
3365                ourForceDisableHibernateSearchForUnitTest = theForceDisableHibernateSearchForUnitTest;
3366        }
3367
3368        static boolean isPlaceholder(DomainResource theResource) {
3369                boolean retVal = false;
3370                Extension extension = theResource.getExtensionByUrl(HapiExtensions.EXT_RESOURCE_PLACEHOLDER);
3371                if (extension != null && extension.hasValue() && extension.getValue() instanceof BooleanType) {
3372                        retVal = ((BooleanType) extension.getValue()).booleanValue();
3373                }
3374                return retVal;
3375        }
3376
3377        /**
3378         * This is only used for unit tests to test failure conditions
3379         */
3380        static void invokeRunnableForUnitTest() {
3381                if (myInvokeOnNextCallForUnitTest != null) {
3382                        Runnable invokeOnNextCallForUnitTest = myInvokeOnNextCallForUnitTest;
3383                        myInvokeOnNextCallForUnitTest = null;
3384                        invokeOnNextCallForUnitTest.run();
3385                }
3386        }
3387
3388        @VisibleForTesting
3389        public static void setInvokeOnNextCallForUnitTest(Runnable theInvokeOnNextCallForUnitTest) {
3390                myInvokeOnNextCallForUnitTest = theInvokeOnNextCallForUnitTest;
3391        }
3392
3393        static List<TermConcept> toPersistedConcepts(
3394                        List<CodeSystem.ConceptDefinitionComponent> theConcept, TermCodeSystemVersion theCodeSystemVersion) {
3395                ArrayList<TermConcept> retVal = new ArrayList<>();
3396
3397                for (CodeSystem.ConceptDefinitionComponent next : theConcept) {
3398                        if (isNotBlank(next.getCode())) {
3399                                TermConcept termConcept = toTermConcept(next, theCodeSystemVersion);
3400                                retVal.add(termConcept);
3401                        }
3402                }
3403
3404                return retVal;
3405        }
3406
3407        @Nonnull
3408        static TermConcept toTermConcept(
3409                        CodeSystem.ConceptDefinitionComponent theConceptDefinition, TermCodeSystemVersion theCodeSystemVersion) {
3410                TermConcept termConcept = new TermConcept();
3411                termConcept.setCode(theConceptDefinition.getCode());
3412                termConcept.setCodeSystemVersion(theCodeSystemVersion);
3413                termConcept.setDisplay(theConceptDefinition.getDisplay());
3414                termConcept.addChildren(
3415                                toPersistedConcepts(theConceptDefinition.getConcept(), theCodeSystemVersion), RelationshipTypeEnum.ISA);
3416
3417                for (CodeSystem.ConceptDefinitionDesignationComponent designationComponent :
3418                                theConceptDefinition.getDesignation()) {
3419                        if (isNotBlank(designationComponent.getValue())) {
3420                                TermConceptDesignation designation = termConcept.addDesignation();
3421                                designation.setLanguage(designationComponent.hasLanguage() ? designationComponent.getLanguage() : null);
3422                                if (designationComponent.hasUse()) {
3423                                        designation.setUseSystem(
3424                                                        designationComponent.getUse().hasSystem()
3425                                                                        ? designationComponent.getUse().getSystem()
3426                                                                        : null);
3427                                        designation.setUseCode(
3428                                                        designationComponent.getUse().hasCode()
3429                                                                        ? designationComponent.getUse().getCode()
3430                                                                        : null);
3431                                        designation.setUseDisplay(
3432                                                        designationComponent.getUse().hasDisplay()
3433                                                                        ? designationComponent.getUse().getDisplay()
3434                                                                        : null);
3435                                }
3436                                designation.setValue(designationComponent.getValue());
3437                        }
3438                }
3439
3440                for (CodeSystem.ConceptPropertyComponent next : theConceptDefinition.getProperty()) {
3441                        TermConceptProperty property = new TermConceptProperty();
3442
3443                        property.setKey(next.getCode());
3444                        property.setConcept(termConcept);
3445                        property.setCodeSystemVersion(theCodeSystemVersion);
3446
3447                        if (next.getValue() instanceof StringType) {
3448                                property.setType(TermConceptPropertyTypeEnum.STRING);
3449                                property.setValue(next.getValueStringType().getValue());
3450                        } else if (next.getValue() instanceof BooleanType) {
3451                                property.setType(TermConceptPropertyTypeEnum.BOOLEAN);
3452                                property.setValue(((BooleanType) next.getValue()).getValueAsString());
3453                        } else if (next.getValue() instanceof IntegerType) {
3454                                property.setType(TermConceptPropertyTypeEnum.INTEGER);
3455                                property.setValue(((IntegerType) next.getValue()).getValueAsString());
3456                        } else if (next.getValue() instanceof DecimalType) {
3457                                property.setType(TermConceptPropertyTypeEnum.DECIMAL);
3458                                property.setValue(((DecimalType) next.getValue()).getValueAsString());
3459                        } else if (next.getValue() instanceof DateTimeType) {
3460                                // DateType is not supported because it's not
3461                                // supported in CodeSystem.setValue
3462                                property.setType(TermConceptPropertyTypeEnum.DATETIME);
3463                                property.setValue(((DateTimeType) next.getValue()).getValueAsString());
3464                        } else if (next.getValue() instanceof Coding) {
3465                                Coding nextCoding = next.getValueCoding();
3466                                property.setType(TermConceptPropertyTypeEnum.CODING);
3467                                property.setCodeSystem(nextCoding.getSystem());
3468                                property.setValue(nextCoding.getCode());
3469                                property.setDisplay(nextCoding.getDisplay());
3470                        } else if (next.getValue() != null) {
3471                                ourLog.warn("Don't know how to handle properties of type: "
3472                                                + next.getValue().getClass());
3473                                continue;
3474                        }
3475
3476                        termConcept.getProperties().add(property);
3477                }
3478                return termConcept;
3479        }
3480
3481        static boolean isDisplayLanguageMatch(String theReqLang, String theStoredLang) {
3482                // NOTE: return the designation when one of then is not specified.
3483                if (theReqLang == null || theStoredLang == null) return true;
3484
3485                return theReqLang.equalsIgnoreCase(theStoredLang);
3486        }
3487}