001package org.hl7.fhir.r5.terminologies.validation;
002
003import java.io.IOException;
004
005/*
006  Copyright (c) 2011+, HL7, Inc.
007  All rights reserved.
008
009  Redistribution and use in source and binary forms, with or without modification, 
010  are permitted provided that the following conditions are met:
011
012 * Redistributions of source code must retain the above copyright notice, this 
013     list of conditions and the following disclaimer.
014 * Redistributions in binary form must reproduce the above copyright notice, 
015     this list of conditions and the following disclaimer in the documentation 
016     and/or other materials provided with the distribution.
017 * Neither the name of HL7 nor the names of its contributors may be used to 
018     endorse or promote products derived from this software without specific 
019     prior written permission.
020
021  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
022  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
023  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
024  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
025  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
026  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
027  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
028  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
029  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
030  POSSIBILITY OF SUCH DAMAGE.
031
032 */
033
034import java.util.ArrayList;
035import java.util.Arrays;
036import java.util.Calendar;
037import java.util.GregorianCalendar;
038import java.util.HashMap;
039import java.util.HashSet;
040import java.util.List;
041import java.util.Map;
042import java.util.Set;
043
044
045import lombok.extern.slf4j.Slf4j;
046import org.hl7.fhir.exceptions.FHIRException;
047import org.hl7.fhir.exceptions.NoTerminologyServiceException;
048import org.hl7.fhir.r5.context.ContextUtilities;
049import org.hl7.fhir.r5.context.IWorkerContext;
050import org.hl7.fhir.r5.elementmodel.LanguageUtils;
051import org.hl7.fhir.r5.extensions.ExtensionConstants;
052import org.hl7.fhir.r5.model.CanonicalType;
053import org.hl7.fhir.r5.model.CodeSystem;
054import org.hl7.fhir.r5.model.Enumerations.CodeSystemContentMode;
055import org.hl7.fhir.r5.model.Enumerations.FilterOperator;
056import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
057import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionDesignationComponent;
058import org.hl7.fhir.r5.model.CodeSystem.ConceptPropertyComponent;
059import org.hl7.fhir.r5.model.CodeableConcept;
060import org.hl7.fhir.r5.model.Coding;
061import org.hl7.fhir.r5.model.DataType;
062import org.hl7.fhir.r5.model.Extension;
063import org.hl7.fhir.r5.model.Enumerations.PublicationStatus;
064import org.hl7.fhir.r5.model.OperationOutcome.IssueType;
065import org.hl7.fhir.r5.model.OperationOutcome.OperationOutcomeIssueComponent;
066import org.hl7.fhir.r5.model.PackageInformation;
067import org.hl7.fhir.r5.model.Parameters;
068import org.hl7.fhir.r5.model.UriType;
069import org.hl7.fhir.r5.model.ValueSet;
070import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceComponent;
071import org.hl7.fhir.r5.model.ValueSet.ConceptReferenceDesignationComponent;
072import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
073import org.hl7.fhir.r5.model.ValueSet.ConceptSetFilterComponent;
074import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
075import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionParameterComponent;
076import org.hl7.fhir.r5.terminologies.CodeSystemUtilities;
077import org.hl7.fhir.r5.terminologies.client.TerminologyClientManager;
078import org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome;
079import org.hl7.fhir.r5.terminologies.providers.CodeSystemProvider;
080import org.hl7.fhir.r5.terminologies.providers.SpecialCodeSystem;
081import org.hl7.fhir.r5.terminologies.providers.URICodeSystem;
082import org.hl7.fhir.r5.terminologies.utilities.TerminologyOperationContext;
083import org.hl7.fhir.r5.terminologies.utilities.TerminologyOperationContext.TerminologyServiceProtectionException;
084import org.hl7.fhir.r5.terminologies.utilities.TerminologyServiceErrorClass;
085import org.hl7.fhir.r5.terminologies.utilities.ValidationResult;
086import org.hl7.fhir.r5.terminologies.utilities.ValueSetProcessBase;
087import org.hl7.fhir.r5.utils.OperationOutcomeUtilities;
088import org.hl7.fhir.r5.utils.ToolingExtensions;
089import org.hl7.fhir.r5.utils.UserDataNames;
090import org.hl7.fhir.r5.utils.validation.ValidationContextCarrier;
091import org.hl7.fhir.r5.utils.validation.ValidationContextCarrier.ValidationContextResourceProxy;
092import org.hl7.fhir.utilities.*;
093import org.hl7.fhir.utilities.i18n.AcceptLanguageHeader.LanguagePreference;
094import org.hl7.fhir.utilities.i18n.subtag.LanguageSubtagRegistry;
095import org.hl7.fhir.utilities.i18n.I18nConstants;
096import org.hl7.fhir.utilities.i18n.LanguageTag;
097import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
098import org.hl7.fhir.utilities.validation.ValidationOptions;
099
100@MarkedToMoveToAdjunctPackage
101@Slf4j
102public class ValueSetValidator extends ValueSetProcessBase {
103
104  public static final String NO_TRY_THE_SERVER = "The local terminology server cannot handle this request";
105
106
107  public class StringWithCode {
108    private OpIssueCode code;
109    private String message;
110    protected StringWithCode(OpIssueCode code, String message) {
111      super();
112      this.code = code;
113      this.message = message;
114    }
115    public OpIssueCode getCode() {
116      return code;
117    }
118    public String getMessage() {
119      return message;
120    } 
121  }
122
123  private ValueSet valueset;
124  private Map<String, ValueSetValidator> inner = new HashMap<>();
125  private ValidationOptions options;
126  private ValidationContextCarrier localContext;
127  private List<CodeSystem> localSystems = new ArrayList<>();
128  protected Parameters expansionParameters;
129  private TerminologyClientManager tcm;
130  private Set<String> unknownSystems;
131  private Set<String> unknownValueSets = new HashSet<>();
132  private boolean throwToServer;
133  private LanguageSubtagRegistry registry;
134  
135
136  public ValueSetValidator(IWorkerContext context, TerminologyOperationContext opContext, ValidationOptions options, ValueSet source, Parameters expansionProfile, TerminologyClientManager tcm, LanguageSubtagRegistry registry) {
137    super(context, opContext);
138    this.valueset = source;
139    this.options = options;
140    this.expansionParameters = expansionProfile;
141    this.tcm = tcm;
142    this.registry = registry;
143    analyseValueSet();
144  }
145  
146  public ValueSetValidator(IWorkerContext context, TerminologyOperationContext opContext, ValidationOptions options, ValueSet source, ValidationContextCarrier ctxt, Parameters expansionProfile, TerminologyClientManager tcm, LanguageSubtagRegistry registry) {
147    super(context, opContext);
148    this.valueset = source;
149    this.options = options.copy();
150    this.options.setEnglishOk(true);
151    this.localContext = ctxt;
152    this.expansionParameters = expansionProfile;
153    this.tcm = tcm;
154    this.registry = registry;
155    analyseValueSet();
156  }
157
158  public Set<String> getUnknownSystems() {
159    return unknownSystems;
160  }
161
162  public void setUnknownSystems(Set<String> unknownSystems) {
163    this.unknownSystems = unknownSystems;
164  }
165
166  public boolean isThrowToServer() {
167    return throwToServer;
168  }
169
170  public void setThrowToServer(boolean throwToServer) {
171    this.throwToServer = throwToServer;
172  }
173
174  private void analyseValueSet() {
175    opContext.note("analyse");
176    if (valueset != null) {
177      opContext.note("vs = "+valueset.getVersionedUrl());
178      opContext.seeContext(valueset.getVersionedUrl());
179      for (Extension s : valueset.getExtensionsByUrl(ExtensionConstants.EXT_VSSUPPLEMENT)) {
180        requiredSupplements.add(s.getValue().primitiveValue());
181      }
182
183      if (!requiredSupplements.isEmpty()) {
184        for (ConceptSetComponent inc : valueset.getCompose().getInclude()) {
185          if (inc.hasSystem()) {
186            checkCodeSystemResolves(inc);
187          }
188        }
189        for (ConceptSetComponent inc : valueset.getCompose().getExclude()) {
190          if (inc.hasSystem()) {
191            checkCodeSystemResolves(inc);
192          }
193        }
194      }
195    } else {
196      opContext.note("vs = null");
197    }
198
199    altCodeParams.seeParameters(expansionParameters);
200    altCodeParams.seeValueSet(valueset);
201    if (localContext != null) {
202      if (valueset != null) {
203        for (ConceptSetComponent i : valueset.getCompose().getInclude()) {
204          analyseComponent(i, "inc"+i);
205        }
206        for (ConceptSetComponent i : valueset.getCompose().getExclude()) {
207          analyseComponent(i, "exc"+i);
208        }
209      }
210    }
211    opContext.note("analysed");
212  }
213
214  private void checkCodeSystemResolves(ConceptSetComponent c) {
215    VersionInfo vi = new VersionInfo(this);
216    CodeSystem cs = resolveCodeSystem(c.getSystem(), vi.getVersion(c.getSystem(), c.getVersion()));
217    if (cs == null) {
218      // well, it doesn't really matter at this point. Mainly we're triggering the supplement analysis to happen 
219      opContext.note("Unable to resolve "+c.getSystem()+"#"+vi.getVersion(c.getSystem(), c.getVersion()));
220    }
221  }
222
223  private void analyseComponent(ConceptSetComponent i, String name) {
224    opContext.deadCheck("analyse Component "+name);
225    if (i.getSystemElement().hasExtension(ToolingExtensions.EXT_VALUESET_SYSTEM)) {
226      String ref = i.getSystemElement().getExtensionString(ToolingExtensions.EXT_VALUESET_SYSTEM);
227      if (ref.startsWith("#")) {
228        String id = ref.substring(1);
229        for (ValidationContextResourceProxy t : localContext.getResources()) {
230          CodeSystem cs = (CodeSystem) t.loadContainedResource(id, CodeSystem.class);
231          if (cs != null) {
232            localSystems.add(cs);
233          }
234        }
235      } else {        
236        throw new Error("Not done yet #2: "+ref);
237      }
238    }    
239  }
240
241  public ValidationResult validateCode(CodeableConcept code) throws FHIRException {
242    return validateCode("CodeableConcept", code);
243  }
244  
245  public ValidationResult validateCode(String path, CodeableConcept code) throws FHIRException {
246    opContext.deadCheck("validate "+code.toString());
247    checkValueSetOptions();
248
249    // first, we validate the codings themselves
250    ValidationProcessInfo info = new ValidationProcessInfo();
251    
252    if (throwToServer) {
253      checkValueSetLoad(info);
254    }
255
256    CodeableConcept vcc = new CodeableConcept();
257    List<ValidationResult> resList = new ArrayList<>();
258    
259    if (!options.isMembershipOnly()) {
260      int i = 0;
261      for (Coding c : code.getCoding()) {
262        if (!c.hasSystem() && !c.hasUserData(UserDataNames.tx_val_sys_error)) {
263          c.setUserData(UserDataNames.tx_val_sys_error, true);
264          info.addIssue(makeIssue(IssueSeverity.WARNING, IssueType.INVALID, path+".coding["+i+"]", context.formatMessage(I18nConstants.CODING_HAS_NO_SYSTEM__CANNOT_VALIDATE), OpIssueCode.InvalidData, null));
265        } else {
266          VersionInfo vi = new VersionInfo(this);
267          checkExpansion(c, vi);
268          checkInclude(c, vi);
269          CodeSystem cs = resolveCodeSystem(c.getSystem(), vi.getVersion(c.getSystem(), c.getVersion()));
270          ValidationResult res = null;
271          if (cs == null || (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.SUPPLEMENT)) {
272            if (context.isNoTerminologyServer()) {
273              if (c.hasVersion()) {
274                String msg = context.formatMessage(I18nConstants.UNKNOWN_CODESYSTEM_VERSION, c.getSystem(), c.getVersion() , resolveCodeSystemVersions(c.getSystem()).toString());
275                unknownSystems.add(c.getSystem()+"|"+c.getVersion());
276                res = new ValidationResult(IssueSeverity.ERROR, msg, makeIssue(IssueSeverity.ERROR, IssueType.NOTFOUND, path+".coding["+i+"].system", msg, OpIssueCode.NotFound, null)).setUnknownSystems(unknownSystems);
277              } else {
278                String msg = context.formatMessage(I18nConstants.UNKNOWN_CODESYSTEM, c.getSystem(), c.getVersion());
279                unknownSystems.add(c.getSystem());
280                res = new ValidationResult(IssueSeverity.ERROR, msg, makeIssue(IssueSeverity.ERROR, IssueType.NOTFOUND, path+".coding["+i+"].system", msg, OpIssueCode.NotFound, null)).setUnknownSystems(unknownSystems);
281              }
282            } else {
283              res = context.validateCode(options.withNoClient(), c, null);
284              if (res.isOk()) {
285                vcc.addCoding(new Coding().setCode(res.getCode()).setVersion(res.getVersion()).setSystem(res.getSystem()).setDisplay(res.getDisplay()));
286              }
287              for (OperationOutcomeIssueComponent iss : res.getIssues()) {
288                if (iss.getSeverity() == org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.ERROR && iss.getDetails().hasCoding("http://hl7.org/fhir/tools/CodeSystem/tx-issue-type", "not-found")) {
289                  iss.setSeverity(org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.WARNING);
290                  res.setSeverity(IssueSeverity.WARNING);
291                }
292                iss.resetPath("Coding", path+".coding["+i+"]");
293              }
294              if (res.isInactive()) {
295                String msg = context.formatMessage(I18nConstants.STATUS_CODE_WARNING_CODE, "not active", c.getCode());
296                res.getIssues().addAll(makeIssue(IssueSeverity.INFORMATION, IssueType.INVALID, path+".coding["+i+"].code", msg, OpIssueCode.CodeRule, res.getServer()));
297              }
298            }
299          } else if (cs.getContent() == CodeSystemContentMode.SUPPLEMENT || cs.hasSupplements()) {
300            String msg = context.formatMessage(I18nConstants.CODESYSTEM_CS_NO_SUPPLEMENT, cs.getVersionedUrl());
301            res = new ValidationResult(IssueSeverity.ERROR, msg, makeIssue(IssueSeverity.ERROR, IssueType.NOTFOUND, path+".coding["+i+"].system", msg, OpIssueCode.InvalidData, null));
302          } else {
303            c.setUserData(UserDataNames.TX_ASSOCIATED_CODESYSTEM, cs);
304
305            checkCanonical(info.getIssues(), path, cs, valueset);
306            res = validateCode(path+".coding["+i+"]", c, cs, vcc, info);
307          }
308          info.getIssues().addAll(res.getIssues());
309          if (res != null) {
310            resList.add(res);
311            if (!res.isOk() && !res.messageIsInIssues()) {
312              if (res.getErrorClass() == TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED) {                
313                info.getIssues().addAll(makeIssue(IssueSeverity.WARNING, IssueType.NOTFOUND, path+".coding["+i+"]", res.getMessage(), OpIssueCode.NotFound, res.getServer()));
314              } else {
315                info.getIssues().addAll(makeIssue(res.getSeverity(), IssueType.CODEINVALID, path+".coding["+i+"]", res.getMessage(), OpIssueCode.InvalidCode, res.getServer()));
316              }
317            }
318          } 
319        }
320        i++;
321      }
322    }
323    Coding foundCoding = null;
324    String msg = null;
325    Boolean result = false;
326    if (valueset != null) {
327      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(", ");
328      List<String> cpath = new ArrayList<>();
329      
330      int i = 0;
331      for (Coding c : code.getCoding()) {
332        String cs = "'"+c.getSystem()+(c.hasVersion() ? "|"+c.getVersion() : "")+"#"+c.getCode()+(c.hasDisplay() ? " ('"+c.getDisplay()+"')" : "")+"'";
333        String cs2 = c.getSystem()+(c.hasVersion() ? "|"+c.getVersion() : "");
334        cpath.add(path+".coding["+i+"]");
335        b.append(cs2);
336        Boolean ok = codeInValueSet(path, c.getSystem(), c.getVersion(), c.getCode(), info);
337        if (ok == null && result != null && result == false) {
338          result = null;
339        } else if (ok != null && ok) {
340          result = true;
341          foundCoding = c;
342          if (!options.isMembershipOnly()) {
343            vcc.addCoding().setSystem(c.getSystem()).setVersion(c.getVersion()).setCode(c.getCode());
344          }
345        }
346        if (ok == null || !ok) {
347          vcc.removeCoding(c.getSystem(), c.getVersion(), c.getCode());          
348        }
349        if (ok != null && !ok) {
350          msg = context.formatMessage(I18nConstants.NONE_OF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_ONE, null, valueset.getVersionedUrl(), cs);
351          info.getIssues().addAll(makeIssue(IssueSeverity.INFORMATION, IssueType.CODEINVALID, path+".coding["+i+"].code", msg, OpIssueCode.ThisNotInVS, null));
352        }
353        i++;
354      }
355      if (result == null) {
356        if (!unknownValueSets.isEmpty()) {
357          msg = context.formatMessage(I18nConstants.UNABLE_TO_CHECK_IF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_VS, valueset.getVersionedUrl(), CommaSeparatedStringBuilder.join(", ", unknownValueSets));
358        } else {
359          msg = context.formatMessage(I18nConstants.UNABLE_TO_CHECK_IF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_CS, valueset.getVersionedUrl(), b.toString());
360        }
361        info.getIssues().addAll(makeIssue(IssueSeverity.WARNING, unknownSystems.isEmpty() && unknownValueSets.isEmpty() ? IssueType.CODEINVALID : IssueType.NOTFOUND, null, msg, OpIssueCode.VSProcessing, null));
362      } else if (!result) {
363        // to match Ontoserver
364        OperationOutcomeIssueComponent iss = new OperationOutcomeIssueComponent(org.hl7.fhir.r5.model.OperationOutcome.IssueSeverity.ERROR, org.hl7.fhir.r5.model.OperationOutcome.IssueType.CODEINVALID);
365        iss.getDetails().setText(context.formatMessage(I18nConstants.TX_GENERAL_CC_ERROR_MESSAGE, valueset.getVersionedUrl()));
366        iss.getDetails().addCoding("http://hl7.org/fhir/tools/CodeSystem/tx-issue-type", OpIssueCode.NotInVS.toCode(), null);
367        info.getIssues().add(iss);
368
369//        msg = context.formatMessagePlural(code.getCoding().size(), I18nConstants.NONE_OF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_, valueset.getVersionedUrl(), b.toString());
370//        info.getIssues().addAll(makeIssue(IssueSeverity.ERROR, IssueType.CODEINVALID, code.getCoding().size() == 1 ? path+".coding[0].code" : path, msg));
371      }
372    }
373
374    if (vcc.hasCoding() && code.hasText()) {
375      vcc.setText(code.getText());
376    }
377    if (!checkRequiredSupplements(info)) {
378      return new ValidationResult(IssueSeverity.ERROR, info.getIssues().get(info.getIssues().size()-1).getDetails().getText(), info.getIssues());
379    } else if (info.hasErrors()) {
380      ValidationResult res = new ValidationResult(IssueSeverity.ERROR, info.summary(), info.getIssues());
381      if (foundCoding != null) {
382        ConceptDefinitionComponent cd = new ConceptDefinitionComponent(foundCoding.getCode());
383        cd.setDisplay(lookupDisplay(foundCoding));
384        res.setDefinition(cd);
385        res.setSystem(foundCoding.getSystem());
386        res.setVersion(foundCoding.hasVersion() ? foundCoding.getVersion() : foundCoding.hasUserData(UserDataNames.TX_ASSOCIATED_CODESYSTEM) ? ((CodeSystem) foundCoding.getUserData(UserDataNames.TX_ASSOCIATED_CODESYSTEM)).getVersion() : null);
387        res.setDisplay(cd.getDisplay());
388      }
389      if (info.getErr() != null) {
390        res.setErrorClass(info.getErr());
391      }
392      res.setUnknownSystems(unknownSystems);
393      res.addCodeableConcept(vcc);
394      return res;
395    } else if (result == null) {
396      return new ValidationResult(IssueSeverity.WARNING, info.summary(), info.getIssues());
397    } else if (foundCoding == null && valueset != null) {
398      return new ValidationResult(IssueSeverity.ERROR, "Internal Error that should not happen", makeIssue(IssueSeverity.FATAL, IssueType.EXCEPTION, path, "Internal Error that should not happen", OpIssueCode.VSProcessing, null));
399    } else if (info.getIssues().size() > 0) {
400      if (foundCoding == null) {
401        IssueSeverity lvl = IssueSeverity.INFORMATION; 
402        for (OperationOutcomeIssueComponent iss : info.getIssues()) {
403          lvl = IssueSeverity.max(lvl, OperationOutcomeUtilities.convert(iss.getSeverity()));
404        }
405        return new ValidationResult(lvl, info.summary(), info.getIssues());        
406      } else {
407        String disp = lookupDisplay(foundCoding);
408        ConceptDefinitionComponent cd = new ConceptDefinitionComponent(foundCoding.getCode());
409        cd.setDisplay(disp);
410        return new ValidationResult(IssueSeverity.WARNING, info.summaryList(), foundCoding.getSystem(), getVersion(foundCoding), cd, disp, info.getIssues()).addCodeableConcept(vcc);
411      }
412    } else if (!result) {
413      if (valueset != null) {
414        throw new Error("This should never happen: no result but is value set");
415      } else if (vcc.hasCoding()) {
416        return new ValidationResult(vcc.getCodingFirstRep().getSystem(), getVersion(vcc.getCodingFirstRep()), new ConceptDefinitionComponent(vcc.getCodingFirstRep().getCode()).setDisplay(vcc.getCodingFirstRep().getDisplay()), vcc.getCodingFirstRep().getDisplay()).addCodeableConcept(vcc);
417      } else {
418        throw new Error("This should never happen: no result, no value set, no coding");
419      }
420    } else if (foundCoding != null) {
421      ConceptDefinitionComponent cd = new ConceptDefinitionComponent(foundCoding.getCode());
422      cd.setDisplay(lookupDisplay(foundCoding));
423      return new ValidationResult(foundCoding.getSystem(), getVersion(foundCoding), cd, getPreferredDisplay(cd, null)).addCodeableConcept(vcc);
424    } else {
425      throw new Error("This should never happen - ther response from the server could not be understood");
426    }
427  }
428
429  private void checkValueSetLoad(ValidationProcessInfo info) {
430    int serverCount = getServerLoad(info);
431    // There's a trade off here: if we're going to hit the server inside the components, then
432    // the amount of value set collateral we send is limited, but we pay the price of hitting 
433    // the server multiple times. If, on the other hand, we give up on that, and hit the server 
434    // directly, we have to send value set collateral (though we cache at the higher level)
435    //
436    // the cutoff value is chosen experimentally
437    if (serverCount > 2) {
438      throw new VSCheckerException("This value set is better processed on the server for performance reasons", null, true);
439    }
440  }
441
442  private int getServerLoad(ValidationProcessInfo info) {
443    int serverCount = 0;
444    if (valueset != null) {
445      for (ConceptSetComponent inc : valueset.getCompose().getInclude()) {
446        serverCount = serverCount + checkValueSetLoad(inc, info);
447      }
448      for (ConceptSetComponent inc : valueset.getCompose().getExclude()) {
449        serverCount = serverCount + checkValueSetLoad(inc, info);
450      }
451    }
452    return serverCount;
453  }
454  
455  private int checkValueSetLoad(ConceptSetComponent inc, ValidationProcessInfo info) {
456    int serverCount = 0;
457    for (UriType uri : inc.getValueSet()) {
458      String url = getCu().pinValueSet(uri.getValue(), expansionParameters);
459      ValueSetValidator vsv = getVs(url, info);
460      serverCount += vsv.getServerLoad(info);
461    }
462    if (inc.hasSystem()) {
463      CodeSystem cs = resolveCodeSystem(inc.getSystem(), inc.getVersion());
464      if (cs == null || (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT)) {
465        serverCount++;
466      }
467    }
468    return serverCount;
469  }
470
471  private boolean checkRequiredSupplements(ValidationProcessInfo info) {
472    if (!requiredSupplements.isEmpty()) {
473      String msg = context.formatMessagePlural(requiredSupplements.size(), I18nConstants.VALUESET_SUPPLEMENT_MISSING, CommaSeparatedStringBuilder.build(requiredSupplements));
474      throw new TerminologyServiceProtectionException(msg, TerminologyServiceErrorClass.BUSINESS_RULE, IssueType.NOTFOUND);
475    }
476    return requiredSupplements.isEmpty();
477  }
478
479  private boolean valueSetDependsOn(String system, String version) {
480    for (ConceptSetComponent inc : valueset.getCompose().getInclude()) {
481      if (system.equals(inc.getSystem()) && (version == null || inc.getVersion() == null || version.equals(inc.getVersion()))) {
482        return true;
483      }
484    }
485    return false;
486  }
487
488  private String getVersion(Coding c) {
489    if (c.hasVersion()) {
490      return c.getVersion();
491    } else if (c.hasUserData(UserDataNames.TX_ASSOCIATED_CODESYSTEM)) {
492      return ((CodeSystem) c.getUserData(UserDataNames.TX_ASSOCIATED_CODESYSTEM)).getVersion();
493    } else {
494      return null;
495    }
496  }
497
498  private String lookupDisplay(Coding c) {
499    CodeSystem cs = resolveCodeSystem(c.getSystem(), c.getVersion());
500    if (cs != null) {
501      ConceptDefinitionComponent cd = CodeSystemUtilities.findCodeOrAltCode(cs.getConcept(), c.getCode(), null);
502      if (cd != null) {
503        return getPreferredDisplay(cd, cs); 
504      }
505    }
506    return null;
507  }
508
509  public CodeSystem resolveCodeSystem(String system, String version) {
510    for (CodeSystem t : localSystems) {
511      if (t.getUrl().equals(system) && versionsMatch(version, t.getVersion())) {
512        return t;
513      }
514    }
515    CodeSystem cs = context.fetchSupplementedCodeSystem(system, version);
516    if (cs == null) {
517      cs = findSpecialCodeSystem(system, version);
518    }
519    if (cs == null) {
520      cs = context.findTxResource(CodeSystem.class, system, version);
521    }
522    if (cs != null) {
523      if (cs.hasUserData("supplements.installed")) {
524        for (String s : cs.getUserString("supplements.installed").split("\\,")) {
525          s = removeSupplement(s);
526        }
527      }
528    }
529    return cs;
530  }
531
532  public List<String> resolveCodeSystemVersions(String system) {
533    List<String> res = new ArrayList<>();
534    for (CodeSystem t : localSystems) {
535      if (t.getUrl().equals(system) && t.hasVersion()) {
536        res.add(t.getVersion());
537      }
538    }
539    res.addAll(new ContextUtilities(context).fetchCodeSystemVersions(system));
540    return res;
541  }
542
543  private boolean versionsMatch(String versionTest, String versionActual) {
544    return versionTest == null && VersionUtilities.versionsMatch(versionTest, versionActual);
545  }
546
547  public ValidationResult validateCode(Coding code) throws FHIRException {
548    return validateCode("Coding", code); 
549  }
550  
551  public ValidationResult validateCode(String path, Coding code) throws FHIRException {
552    opContext.deadCheck("validate "+code.toString());
553    checkValueSetOptions();
554    
555    String warningMessage = null;
556    // first, we validate the concept itself
557
558    ValidationResult res = null;
559    boolean inExpansion = false;
560    boolean inInclude = false;
561    List<OperationOutcomeIssueComponent> issues = new ArrayList<>();
562    ValidationProcessInfo info = new ValidationProcessInfo(issues);
563    VersionInfo vi = new VersionInfo(this);
564    checkCanonical(issues, path, valueset, valueset);
565
566    String system = code.getSystem();
567    if (!options.isMembershipOnly()) {
568      if (system == null && !code.hasDisplay() && options.isGuessSystem()) { // dealing with just a plain code (enum)
569        List<StringWithCode> problems = new ArrayList<>();
570        system = systemForCodeInValueSet(code.getCode(), problems);
571        if (system == null) {
572          if (problems.size() == 0) {
573            throw new Error("Unable to resolve systems but no reason why"); // this is an error in the java code
574          } else if (problems.size() == 1) {
575            String msg = context.formatMessagePlural(1, I18nConstants.NONE_OF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_, valueset.getVersionedUrl(), "'"+code.toString()+"'");
576            issues.addAll(makeIssue(IssueSeverity.ERROR, IssueType.CODEINVALID, "code", msg, OpIssueCode.NotInVS, null));
577            issues.addAll(makeIssue(IssueSeverity.ERROR, IssueType.NOTFOUND, "code", problems.get(0).getMessage(), problems.get(0).getCode(), null));
578            return new ValidationResult(IssueSeverity.ERROR, problems.get(0).getMessage()+"; "+msg, issues);
579          } else {
580            CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder("; ");
581            for (StringWithCode s : problems) {
582              b.append(s.getMessage());
583            }            
584            ValidationResult vr = new ValidationResult(IssueSeverity.ERROR, b.toString(), null);
585            for (StringWithCode s : problems) {
586              vr.getIssues().addAll(makeIssue(IssueSeverity.ERROR, IssueType.UNKNOWN, path, s.getMessage(), s.getCode(), vr.getServer()));
587            }
588            return vr;
589          }
590        }
591      }
592      if (!code.hasSystem()) {
593        if (options.isGuessSystem() && system == null && Utilities.isAbsoluteUrl(code.getCode())) {
594          system = "urn:ietf:rfc:3986"; // this arises when using URIs bound to value sets
595        }
596        code.setSystem(system);
597      }
598      if (!code.hasSystem()) {
599        res = new ValidationResult(IssueSeverity.WARNING, context.formatMessage(I18nConstants.CODING_HAS_NO_SYSTEM__CANNOT_VALIDATE), null);
600        if (!code.hasUserData(UserDataNames.tx_val_sys_error)) {
601          code.setUserData(UserDataNames.tx_val_sys_error, true);
602          res.getIssues().addAll(makeIssue(IssueSeverity.WARNING, IssueType.INVALID, path, context.formatMessage(I18nConstants.CODING_HAS_NO_SYSTEM__CANNOT_VALIDATE), OpIssueCode.InvalidData, null));
603        }
604      } else {
605        if (!Utilities.isAbsoluteUrl(system)) {
606          String msg = context.formatMessage(I18nConstants.TERMINOLOGY_TX_SYSTEM_RELATIVE);
607          issues.addAll(makeIssue(IssueSeverity.ERROR, IssueType.INVALID, path+".system", msg, OpIssueCode.InvalidData, null));                
608        }
609        inExpansion = checkExpansion(code, vi);
610        inInclude = checkInclude(code, vi);
611        String wv = vi.getVersion(system, code.getVersion());
612        CodeSystem cs = resolveCodeSystem(system, wv);
613        if (cs == null) {
614          if (!VersionUtilities.isR6Plus(context.getVersion()) && "urn:ietf:bcp:13".equals(system) && Utilities.existsInList(code.getCode(), "xml", "json", "ttl") && "http://hl7.org/fhir/ValueSet/mimetypes".equals(valueset.getUrl())) {
615            return new ValidationResult(system, null, new ConceptDefinitionComponent(code.getCode()), "application/fhir+"+code.getCode());        
616          } else {
617            OpIssueCode oic = OpIssueCode.NotFound;
618            IssueType itype = IssueType.NOTFOUND;
619            ValueSet vs = context.fetchResource(ValueSet.class, system);
620            if (vs != null) {
621              warningMessage = context.formatMessage(I18nConstants.TERMINOLOGY_TX_SYSTEM_VALUESET2, system);  
622              oic = OpIssueCode.InvalidData;
623              itype = IssueType.INVALID;
624            } else if (wv == null) {
625              warningMessage = context.formatMessage(I18nConstants.UNKNOWN_CODESYSTEM, system);
626              unknownSystems.add(system);
627            } else {
628              warningMessage = context.formatMessage(I18nConstants.UNKNOWN_CODESYSTEM_VERSION, system, wv, resolveCodeSystemVersions(system).toString());
629              unknownSystems.add(system+"|"+wv);
630            }
631            if (!inExpansion) {
632              if (valueset != null && valueset.hasExpansion()) {
633                String msg = context.formatMessage(I18nConstants.CODESYSTEM_CS_UNK_EXPANSION,
634                    valueset.getUrl(), 
635                    code.getSystem(), 
636                    code.getCode().toString());
637                issues.addAll(makeIssue(IssueSeverity.ERROR, itype, path, msg, OpIssueCode.VSProcessing, null));
638                throw new VSCheckerException(msg, issues, TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED);
639              } else {
640                issues.addAll(makeIssue(IssueSeverity.ERROR, itype, path+".system", warningMessage, oic, null));
641                res = new ValidationResult(IssueSeverity.WARNING, warningMessage, issues);              
642                if (valueset == null) {
643                  throw new VSCheckerException(warningMessage, issues, TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED);
644                } else {
645                  //              String msg = context.formatMessagePlural(1, I18nConstants.NONE_OF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_, valueset.getUrl(), code.toString());
646                  //              issues.addAll(makeIssue(IssueSeverity.ERROR, IssueType.INVALID, path, msg));
647                  // we don't do this yet
648                  // throw new VSCheckerException(warningMessage, issues); 
649                }
650              }
651            }
652          }
653        } else {
654          checkCanonical(issues, path, cs, valueset);
655        }
656        if (cs != null && (cs.hasSupplements() || cs.getContent() == CodeSystemContentMode.SUPPLEMENT)) {
657          String msg = context.formatMessage(I18nConstants.CODESYSTEM_CS_NO_SUPPLEMENT, cs.getVersionedUrl());
658          return new ValidationResult(IssueSeverity.ERROR, msg, makeIssue(IssueSeverity.ERROR, IssueType.INVALID, path+".system", msg, OpIssueCode.InvalidData, null, I18nConstants.CODESYSTEM_CS_NO_SUPPLEMENT));        
659        }
660        if (cs!=null && cs.getContent() != CodeSystemContentMode.COMPLETE) {
661          warningMessage = "Resolved system "+system+(cs.hasVersion() ? " (v"+cs.getVersion()+")" : "")+", but the definition ";
662          switch (cs.getContent()) {
663          case EXAMPLE:
664            warningMessage = warningMessage +"only has example content";
665            break;
666          case FRAGMENT:
667            warningMessage = warningMessage + "is only a fragment";
668            break;
669          case NOTPRESENT:
670            warningMessage = warningMessage + "doesn't include any codes";
671            break;
672          case SUPPLEMENT:
673            warningMessage = warningMessage + " is for a supplement to "+cs.getSupplements();
674            break;
675          default:
676            break;
677          }
678          warningMessage = warningMessage + ", so the code has not been validated";
679          if (cs.getContent() == CodeSystemContentMode.NOTPRESENT) {
680            throw new VSCheckerException(warningMessage, null, TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED);
681          }
682          if (!options.isExampleOK() && !inExpansion && cs.getContent() != CodeSystemContentMode.FRAGMENT) { // we're going to give it a go if it's a fragment
683            throw new VSCheckerException(warningMessage, null, true);
684          }
685        }
686
687        if (cs != null /*&& (cs.getContent() == CodeSystemContentMode.COMPLETE || cs.getContent() == CodeSystemContentMode.FRAGMENT)*/) {
688          if (!(cs.getContent() == CodeSystemContentMode.COMPLETE || cs.getContent() == CodeSystemContentMode.FRAGMENT ||
689              (options.isExampleOK() && cs.getContent() == CodeSystemContentMode.EXAMPLE))) {
690            if (inInclude) {
691              ConceptReferenceComponent cc = findInInclude(code);
692              if (cc != null) {
693                // we'll take it on faith
694                String disp = getPreferredDisplay(cc);
695                res = new ValidationResult(system, cs.getVersion(), new ConceptDefinitionComponent().setCode(cc.getCode()).setDisplay(disp), disp);
696                res.addMessage("Resolved system "+system+", but the definition is not complete, so assuming value set include is correct");
697                return res;
698              }
699            }
700            // we can't validate that here. 
701            throw new FHIRException("Unable to evaluate based on code system with status = "+cs.getContent().toCode());
702          }
703          res = validateCode(path, code, cs, null, info);
704          res.setIssues(issues);
705        } else if (cs == null && valueset.hasExpansion() && inExpansion) {
706          for (ValueSetExpansionParameterComponent p : valueset.getExpansion().getParameter()) {
707            if ("used-supplement".equals(p.getName())) {
708              removeSupplement(p.getValue().primitiveValue());
709            }
710          }
711          // we just take the value set as face value then
712          res = new ValidationResult(system, wv, new ConceptDefinitionComponent().setCode(code.getCode()).setDisplay(code.getDisplay()), code.getDisplay());
713          if (!preferServerSide(system)) {
714            res.addMessage("Code System unknown, so assuming value set expansion is correct ("+warningMessage+")");
715          }
716        } else {
717          // well, we didn't find a code system - try the expansion? 
718          // disabled waiting for discussion
719          if (throwToServer) {
720            throw new FHIRException(NO_TRY_THE_SERVER);
721          }
722        }
723      }
724    } else {
725      inExpansion = checkExpansion(code, vi);
726      inInclude = checkInclude(code, vi);
727    }
728    String wv = vi.getVersion(system, code.getVersion());
729    if (!checkRequiredSupplements(info)) {
730      return new ValidationResult(IssueSeverity.ERROR, issues.get(issues.size()-1).getDetails().getText(), issues);
731    }
732
733    
734    // then, if we have a value set, we check it's in the value set
735    if (valueset != null) {
736      if ((res==null || res.isOk())) { 
737        Boolean ok = codeInValueSet(path, system, wv, code.getCode(), info);
738        if (ok == null || !ok) {
739          if (res == null) {
740            res = new ValidationResult((IssueSeverity) null, null, info.getIssues());
741          }
742          if (info.getErr() != null) {
743            res.setErrorClass(info.getErr());
744          }
745          if (ok == null) {
746            String m = null;
747            if (!unknownSystems.isEmpty()) {
748              m = context.formatMessage(I18nConstants.UNABLE_TO_CHECK_IF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_CS, valueset.getVersionedUrl(), CommaSeparatedStringBuilder.join(",", unknownSystems));
749            } else if (!unknownValueSets.isEmpty()) {
750              res.addMessage(info.getIssues().get(0).getDetails().getText());
751              m = context.formatMessage(I18nConstants.UNABLE_TO_CHECK_IF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_VS, valueset.getVersionedUrl(), CommaSeparatedStringBuilder.join(",", unknownValueSets));
752            } else {
753              // not sure why we'd get to here?
754              m = context.formatMessage(I18nConstants.UNABLE_TO_CHECK_IF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_, valueset.getVersionedUrl());
755            }
756            res.addMessage(m);
757            res.getIssues().addAll(makeIssue(IssueSeverity.WARNING, IssueType.NOTFOUND, null, m, OpIssueCode.VSProcessing, null));
758            res.setUnknownSystems(unknownSystems);
759            res.setSeverity(IssueSeverity.ERROR); // back patching for display logic issue
760            res.setErrorClass(TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED);
761          } else if (!inExpansion && !inInclude) {
762//            if (!info.getIssues().isEmpty()) {
763//              res.setMessage("Not in value set "+valueset.getUrl()+": "+info.summary()).setSeverity(IssueSeverity.ERROR);              
764//              res.getIssues().addAll(makeIssue(IssueSeverity.ERROR, IssueType.INVALID, path, res.getMessage()));
765//            } else
766//            {
767              String msg = context.formatMessagePlural(1, I18nConstants.NONE_OF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_, valueset.getVersionedUrl(), "'"+code.toString()+"'");
768              res.addMessage(msg).setSeverity(IssueSeverity.ERROR);
769              res.getIssues().addAll(makeIssue(IssueSeverity.ERROR, IssueType.CODEINVALID, path+".code", msg, OpIssueCode.NotInVS, null));
770              res.setDefinition(null);
771              res.setSystem(null);
772              res.setDisplay(null);
773              res.setUnknownSystems(unknownSystems);              
774//            }
775          } else if (warningMessage!=null) {
776            String msg = context.formatMessage(I18nConstants.CODE_FOUND_IN_EXPANSION_HOWEVER_, warningMessage);
777            res = new ValidationResult(IssueSeverity.WARNING, msg, makeIssue(IssueSeverity.WARNING, IssueType.EXCEPTION, path, msg, OpIssueCode.VSProcessing, null));
778          } else if (inExpansion) {
779            res.setMessage("Code found in expansion, however: " + res.getMessage());
780            res.getIssues().addAll(makeIssue(IssueSeverity.WARNING, IssueType.EXCEPTION, path, res.getMessage(), OpIssueCode.VSProcessing, null));
781          } else if (inInclude) {
782            res.setMessage("Code found in include, however: " + res.getMessage());
783            res.getIssues().addAll(makeIssue(IssueSeverity.WARNING, IssueType.EXCEPTION, path, res.getMessage(), OpIssueCode.VSProcessing, null));
784          }
785        } else if (res == null) {
786          res = new ValidationResult(system, wv, null, null);
787        }
788      } else if ((res != null && !res.isOk())) {
789        String msg = context.formatMessagePlural(1, I18nConstants.NONE_OF_THE_PROVIDED_CODES_ARE_IN_THE_VALUE_SET_, valueset.getVersionedUrl(), "'"+code.toString()+"'");
790        res.addMessage(msg);
791        res.getIssues().addAll(makeIssue(IssueSeverity.ERROR, IssueType.CODEINVALID, path+".code", msg, OpIssueCode.NotInVS, null));
792      }
793    }
794    if (res != null && res.getSeverity() == IssueSeverity.INFORMATION && res.getMessage() != null) {
795      res.setSeverity(IssueSeverity.ERROR); // back patching for display logic issue
796    }
797    return res;
798  }
799
800  private void checkValueSetOptions() {
801    if (valueset != null) {
802      for (Extension ext : valueset.getCompose().getExtensionsByUrl("http://hl7.org/fhir/tools/StructureDefinition/valueset-expansion-parameter")) {
803        var name = ext.getExtensionString("name");
804        var value = ext.getExtensionByUrl("value").getValue();
805        if ("displayLanguage".equals(name)) {
806          options.setLanguages(value.primitiveValue());
807        }
808      }
809      if (!options.hasLanguages() && valueset.hasLanguage()) {
810        options.addLanguage(valueset.getLanguage());
811      }
812    }
813
814    if (options.getLanguages() != null) {
815      for (LanguagePreference t : options.getLanguages().getLangs()) {
816        try {
817          LanguageTag tag = new LanguageTag(registry, t.getLang());
818        } catch (Exception e) {
819          throw new TerminologyServiceProtectionException(context.formatMessage(I18nConstants.INVALID_DISPLAY_NAME, options.getLanguages().getSource()), TerminologyServiceErrorClass.PROCESSING, IssueType.PROCESSING, e.getMessage());
820        }
821      }
822    }
823  }
824
825  private static final Set<String> SERVER_SIDE_LIST = new HashSet<>(Arrays.asList("http://fdasis.nlm.nih.gov", "http://hl7.org/fhir/sid/ndc", "http://loinc.org", "http://snomed.info/sct", "http://unitsofmeasure.org", 
826      "http://unstats.un.org/unsd/methods/m49/m49.htm", "http://varnomen.hgvs.org", "http://www.nlm.nih.gov/research/umls/rxnorm", "https://www.usps.com/",
827      "urn:ietf:bcp:13","urn:ietf:bcp:47","urn:ietf:rfc:3986", "urn:iso:std:iso:3166","urn:iso:std:iso:4217", "urn:oid:1.2.36.1.2001.1005.17"));
828  
829  private boolean preferServerSide(String system) {
830    if (SERVER_SIDE_LIST.contains(system)) {
831      return true;
832    }
833    
834    try {
835      if (tcm.supportsSystem(system)) {
836        return true;
837      }
838    } catch (IOException e) {
839      e.printStackTrace();
840    }
841  
842    return false;    
843  }
844
845  private boolean checkInclude(Coding code, VersionInfo vi) {
846    if (valueset == null || code.getSystem() == null || code.getCode() == null) {
847      return false;
848    }
849    for (ConceptSetComponent inc : valueset.getCompose().getExclude()) {
850      if (inc.hasSystem() && inc.getSystem().equals(code.getSystem())) {
851        for (ConceptReferenceComponent cc : inc.getConcept()) {
852          if (cc.hasCode() && cc.getCode().equals(code.getCode())) {
853            return false;
854          }
855        }
856      }
857    }
858    for (ConceptSetComponent inc : valueset.getCompose().getInclude()) {
859      if (inc.hasSystem() && inc.getSystem().equals(code.getSystem())) {
860        vi.setComposeVersion(inc.getVersion());
861        for (ConceptReferenceComponent cc : inc.getConcept()) {
862          if (cc.hasCode() && cc.getCode().equals(code.getCode())) {
863            return true;
864          }
865        }
866      }
867    }
868    return false;
869  }
870
871  private ConceptReferenceComponent findInInclude(Coding code) {
872    if (valueset == null || code.getSystem() == null || code.getCode() == null) {
873      return null;
874    }
875    for (ConceptSetComponent inc : valueset.getCompose().getInclude()) {
876      if (inc.hasSystem() && inc.getSystem().equals(code.getSystem())) {
877        for (ConceptReferenceComponent cc : inc.getConcept()) {
878          if (cc.hasCode() && cc.getCode().equals(code.getCode())) {
879            return cc;
880          }
881        }
882      }
883    }
884    return null;
885  }
886
887  private CodeSystem findSpecialCodeSystem(String system, String version) {
888    if ("urn:ietf:rfc:3986".equals(system)) {
889      CodeSystem cs = new CodeSystem();
890      cs.setUrl(system);
891      cs.setUserData(UserDataNames.tx_cs_special, new URICodeSystem());
892      cs.setContent(CodeSystemContentMode.COMPLETE);
893      return cs; 
894    }
895    return null;
896  }
897
898  private ValidationResult findCodeInExpansion(Coding code) {
899    if (valueset==null || !valueset.hasExpansion())
900      return null;
901    return findCodeInExpansion(code, valueset.getExpansion().getContains());
902  }
903
904  private ValidationResult findCodeInExpansion(Coding code, List<ValueSetExpansionContainsComponent> contains) {
905    for (ValueSetExpansionContainsComponent containsComponent: contains) {
906      opContext.deadCheck("findCodeInExpansion");
907      if (containsComponent.getSystem().equals(code.getSystem()) && containsComponent.getCode().equals(code.getCode())) {
908        ConceptDefinitionComponent ccd = new ConceptDefinitionComponent();
909        ccd.setCode(containsComponent.getCode());
910        ccd.setDisplay(containsComponent.getDisplay());
911        ValidationResult res = new ValidationResult(code.getSystem(), code.hasVersion() ? code.getVersion() : containsComponent.getVersion(), ccd, getPreferredDisplay(ccd, null));
912        return res;
913      }
914      if (containsComponent.hasContains()) {
915        ValidationResult res = findCodeInExpansion(code, containsComponent.getContains());
916        if (res != null) {
917          return res;
918        }
919      }
920    }
921    return null;
922  }
923
924  private boolean checkExpansion(Coding code, VersionInfo vi) {
925    if (valueset==null || !valueset.hasExpansion()) {
926      return false;
927    }
928    return checkExpansion(code, valueset.getExpansion().getContains(), vi);
929  }
930
931  private boolean checkExpansion(Coding code, List<ValueSetExpansionContainsComponent> contains, VersionInfo vi) {
932    for (ValueSetExpansionContainsComponent containsComponent: contains) {
933      opContext.deadCheck("checkExpansion: "+code.toString());
934      if (containsComponent.hasSystem() && containsComponent.hasCode() && containsComponent.getSystem().equals(code.getSystem()) && containsComponent.getCode().equals(code.getCode())) {
935        vi.setExpansionVersion(containsComponent.getVersion());
936        return true;
937      }
938      if (containsComponent.hasContains() && checkExpansion(code, containsComponent.getContains(), vi)) {
939        return true;
940      }
941    }
942    return false;
943  }
944
945  private ValidationResult validateCode(String path, Coding code, CodeSystem cs, CodeableConcept vcc, ValidationProcessInfo info) {
946    ConceptDefinitionComponent cc = cs.hasUserData(UserDataNames.tx_cs_special) ? ((SpecialCodeSystem) cs.getUserData(UserDataNames.tx_cs_special)).findConcept(code) : findCodeInConcept(cs.getConcept(), code.getCode(), cs.getCaseSensitive(), allAltCodes);
947    if (cc == null) {
948      cc = findSpecialConcept(code, cs);
949    }
950    if (cc == null) {
951      if (cs.getContent() == CodeSystemContentMode.FRAGMENT) {
952        String msg = context.formatMessage(I18nConstants.UNKNOWN_CODE_IN_FRAGMENT, code.getCode(), cs.getUrl(),  cs.getVersion());
953        return new ValidationResult(IssueSeverity.WARNING, msg, makeIssue(IssueSeverity.WARNING, IssueType.CODEINVALID, path+".code", msg, OpIssueCode.InvalidCode, null));        
954      } else {
955        String msg = context.formatMessage(I18nConstants.UNKNOWN_CODE_IN_VERSION, code.getCode(), cs.getUrl(), cs.getVersion());
956        return new ValidationResult(IssueSeverity.ERROR, msg, makeIssue(IssueSeverity.ERROR, IssueType.CODEINVALID, path+".code", msg, OpIssueCode.InvalidCode, null));
957      }
958    } else {
959      if (!cc.getCode().equals(code.getCode())) {
960        String msg = context.formatMessage(I18nConstants.CODE_CASE_DIFFERENCE, code.getCode(), cc.getCode(), cs.getVersionedUrl());
961        info.addIssue(makeIssue(IssueSeverity.INFORMATION, IssueType.BUSINESSRULE, path+".code", msg, OpIssueCode.CodeRule, null));
962      }
963    }
964    Coding vc = new Coding().setCode(cc.getCode()).setSystem(cs.getUrl()).setVersion(cs.getVersion()).setDisplay(getPreferredDisplay(cc, cs));
965    if (vcc != null) {
966      vcc.addCoding(vc);
967    }
968
969    boolean inactive = (CodeSystemUtilities.isInactive(cs, cc));
970    String status = inactive ? (CodeSystemUtilities.getStatus(cs, cc)) : null;
971
972    boolean isDefaultLang = false;
973    boolean ws = false;     
974    if (code.getDisplay() == null) {
975      return new ValidationResult(code.getSystem(), cs.getVersion(), cc, vc.getDisplay()).setStatus(inactive, status);
976    }
977    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(", ", " or ");
978    if (cc.hasDisplay() && isOkLanguage(cs.getLanguage())) {
979      b.append("'"+cc.getDisplay()+"'"+(cs.hasLanguage() ? " ("+cs.getLanguage()+")" : ""));
980      if (code.getDisplay().equalsIgnoreCase(cc.getDisplay())) {
981        return new ValidationResult(code.getSystem(), cs.getVersion(), cc, getPreferredDisplay(cc, cs)).setStatus(inactive, status);
982      } else if (Utilities.normalize(code.getDisplay()).equals(Utilities.normalize(cc.getDisplay()))) {
983        ws = true;
984      }
985    } else if (cc.hasDisplay() && code.getDisplay().equalsIgnoreCase(cc.getDisplay())) {
986      isDefaultLang = true;
987    }
988    
989    for (ConceptDefinitionDesignationComponent ds : cc.getDesignation()) {
990      opContext.deadCheck("validateCode1 "+ds.toString());
991      if (isOkLanguage(ds.getLanguage())) {
992        b.append("'"+ds.getValue()+"' ("+ds.getLanguage()+")");
993        if (code.getDisplay().equalsIgnoreCase(ds.getValue())) {
994          return new ValidationResult(code.getSystem(),cs.getVersion(),  cc, getPreferredDisplay(cc, cs)).setStatus(inactive, status);
995        }
996        if (Utilities.normalize(code.getDisplay()).equalsIgnoreCase(Utilities.normalize(ds.getValue()))) {
997          ws = true;
998        }
999      }
1000    }
1001    // also check to see if the value set has another display
1002    if (options.isUseValueSetDisplays()) {
1003      ConceptReferencePair vs = findValueSetRef(code.getSystem(), code.getCode());
1004      if (vs != null && (vs.getCc().hasDisplay() ||vs.getCc().hasDesignation())) {
1005        if (vs.getCc().hasDisplay() && isOkLanguage(vs.getValueset().getLanguage())) {
1006          b.append("'"+vs.getCc().getDisplay()+"'");
1007          if (code.getDisplay().equalsIgnoreCase(vs.getCc().getDisplay())) {
1008            return new ValidationResult(code.getSystem(), cs.getVersion(), cc, getPreferredDisplay(cc, cs)).setStatus(inactive, status);
1009          }
1010        }
1011        for (ConceptReferenceDesignationComponent ds : vs.getCc().getDesignation()) {
1012          opContext.deadCheck("validateCode2 "+ds.toString());
1013          if (isOkLanguage(ds.getLanguage())) {
1014            b.append("'"+ds.getValue()+"'");
1015            if (code.getDisplay().equalsIgnoreCase(ds.getValue())) {
1016              return new ValidationResult(code.getSystem(), cs.getVersion(), cc, getPreferredDisplay(cc, cs)).setStatus(inactive, status);
1017            }
1018          }
1019        }
1020      }
1021    }
1022    if (b.count() > 0) {
1023      String msg = context.formatMessagePlural(b.count(), ws ? I18nConstants.DISPLAY_NAME_WS_FOR__SHOULD_BE_ONE_OF__INSTEAD_OF : I18nConstants.DISPLAY_NAME_FOR__SHOULD_BE_ONE_OF__INSTEAD_OF, code.getSystem(), code.getCode(), b.toString(), code.getDisplay(), options.langSummary());
1024      return new ValidationResult(dispWarningStatus(), msg, code.getSystem(), cs.getVersion(), cc, getPreferredDisplay(cc, cs), makeIssue(dispWarning(), IssueType.INVALID, path+".display", msg, OpIssueCode.Display, null)).setStatus(inactive, status);
1025    } else if (isDefaultLang) {
1026      // we didn't find any valid displays because there aren't any, so the default language is acceptable, but we'll still add a hint about that
1027      boolean none = options.getLanguages().getLangs().size() == 1 && !hasLanguage(cs, options.getLanguages().getLangs().get(0));
1028      String msg = context.formatMessagePlural(options.getLanguages().getLangs().size(), none ? I18nConstants.NO_VALID_DISPLAY_FOUND_LANG_NONE : I18nConstants.NO_VALID_DISPLAY_FOUND_LANG_SOME, code.getSystem(), code.getCode(), code.getDisplay(), options.langSummary(), code.getDisplay());
1029      String n = null;
1030      return new ValidationResult(IssueSeverity.INFORMATION, n, code.getSystem(), cs.getVersion(), cc, getPreferredDisplay(cc, cs), makeIssue(IssueSeverity.INFORMATION, IssueType.INVALID, path+".display", msg, OpIssueCode.DisplayComment, null)).setStatus(inactive, status);      
1031    } else if (!code.getDisplay().equals(vc.getDisplay())) {
1032      String msg = context.formatMessage(I18nConstants.NO_VALID_DISPLAY_FOUND_NONE_FOR_LANG_ERR, code.getDisplay(), code.getSystem(), code.getCode(), options.langSummary(), vc.getDisplay());
1033      return new ValidationResult(IssueSeverity.ERROR, msg, code.getSystem(), cs.getVersion(), cc, cc.getDisplay(), makeIssue(dispWarning(), IssueType.INVALID, path+".display", msg, OpIssueCode.Display, null)).setStatus(inactive, status).setErrorIsDisplayIssue(true);      
1034    } else {
1035      String msg = context.formatMessagePlural(options.getLanguages().getLangs().size(), I18nConstants.NO_VALID_DISPLAY_FOUND, code.getSystem(), code.getCode(), code.getDisplay(), options.langSummary());
1036      return new ValidationResult(IssueSeverity.WARNING, msg, code.getSystem(), cs.getVersion(), cc, cc.getDisplay(), makeIssue(IssueSeverity.WARNING, IssueType.INVALID, path+".display", msg, OpIssueCode.Display, null)).setStatus(inactive, status);      
1037    }
1038  }
1039
1040  private boolean hasLanguage(CodeSystem cs, LanguagePreference languagePreference) {
1041    String lang = languagePreference.getLang();
1042    if (lang == null) {
1043      return false;
1044    }
1045    for (ConceptDefinitionComponent cc : cs.getConcept()) {
1046      boolean hl = hasLanguage(cs, cc, lang);
1047      if (hl) {
1048        return true;
1049      }
1050    }
1051    return false;
1052  }
1053
1054  private boolean hasLanguage(CodeSystem cs, ConceptDefinitionComponent cc, String lang) {
1055    if (lang.equals(cs.getLanguage()) && cc.hasDisplay()) {
1056      return true;
1057    }
1058    for (ConceptDefinitionDesignationComponent d : cc.getDesignation()) {
1059      if (lang.equals(d.getLanguage())) {
1060        return true;
1061      }
1062    }
1063    for (ConceptDefinitionComponent cc1 : cc.getConcept()) {
1064      boolean hl = hasLanguage(cs, cc1, lang);
1065      if (hl) {
1066        return true;
1067      }
1068    }
1069    return false;
1070  }
1071
1072  private ConceptDefinitionComponent findSpecialConcept(Coding c, CodeSystem cs) {
1073    // handling weird special cases in v2 code systems 
1074    if ("http://terminology.hl7.org/CodeSystem/v2-0203".equals(cs.getUrl())) {
1075      String code = c.getCode();
1076      if (code != null && code.startsWith("NN") && code.length() > 3) {
1077        ConceptDefinitionComponent cd = findCountryCode(code.substring(2));
1078        if (cd != null) {
1079          return new ConceptDefinitionComponent(code).setDisplay("National Identifier for "+cd.getDisplay());
1080        }
1081      }      
1082    }
1083//    0396: HL7nnnn, IBTnnnn, ISOnnnn, X12Dennnn, 99zzz
1084//    0335: PRNxxx
1085    return null;
1086  }
1087
1088  
1089  private ConceptDefinitionComponent findCountryCode(String code) {
1090    ValidationResult vr = context.validateCode(new ValidationOptions(FhirPublication.R5), "urn:iso:std:iso:3166", null, code, null);
1091    return vr == null || !vr.isOk() ? null : new ConceptDefinitionComponent(code).setDisplay(vr.getDisplay()).setDefinition(vr.getDefinition());
1092  }
1093
1094  private IssueSeverity dispWarning() {
1095    return options.isDisplayWarningMode() ? IssueSeverity.WARNING : IssueSeverity.ERROR; 
1096  }
1097  
1098  private IssueSeverity dispWarningStatus() {
1099    return options.isDisplayWarningMode() ? IssueSeverity.WARNING : IssueSeverity.INFORMATION; // information -> error later
1100  }
1101
1102  private boolean isOkLanguage(String language) {
1103    if (!options.hasLanguages()) {
1104      return true;
1105    }
1106    if (LanguageUtils.langsMatch(options.getLanguages(), language)) {
1107      return true;
1108    }
1109    if (language == null && (options.langSummary().contains("en") || options.langSummary().contains("en-US") || options.isEnglishOk())) {
1110      return true;
1111    }
1112    return false;
1113  }
1114
1115  private ConceptReferencePair findValueSetRef(String system, String code) {
1116    if (valueset == null)
1117      return null;
1118    // if it has an expansion
1119    for (ValueSetExpansionContainsComponent exp : valueset.getExpansion().getContains()) {
1120      opContext.deadCheck("findValueSetRef "+exp.toString());
1121      if (system.equals(exp.getSystem()) && code.equals(exp.getCode())) {
1122        ConceptReferenceComponent cc = new ConceptReferenceComponent();
1123        cc.setDisplay(exp.getDisplay());
1124        cc.setDesignation(exp.getDesignation());
1125        return new ConceptReferencePair(valueset, cc);
1126      }
1127    }
1128    for (ConceptSetComponent inc : valueset.getCompose().getInclude()) {
1129      if (system.equals(inc.getSystem())) {
1130        for (ConceptReferenceComponent cc : inc.getConcept()) {
1131          if (cc.getCode().equals(code)) {
1132            return new ConceptReferencePair(valueset, cc);
1133          }
1134        }
1135      }
1136      for (CanonicalType url : inc.getValueSet()) {
1137        ConceptReferencePair cc = getVs(getCu().pinValueSet(url.asStringValue(), expansionParameters), null).findValueSetRef(system, code);
1138        if (cc != null) {
1139          return cc;
1140        }
1141      }
1142    }
1143    return null;
1144  }
1145
1146  /*
1147   * Check that all system values within an expansion correspond to the specified system value
1148   */
1149  private boolean checkSystem(List<ValueSetExpansionContainsComponent> containsList, String system) {
1150    for (ValueSetExpansionContainsComponent contains : containsList) {
1151      if (!contains.getSystem().equals(system) || (contains.hasContains() && !checkSystem(contains.getContains(), system))) {
1152        return false;
1153      }
1154    }
1155    return true;
1156  }
1157
1158  private ConceptDefinitionComponent findCodeInConcept(ConceptDefinitionComponent concept, String code, boolean caseSensitive, AlternateCodesProcessingRules altCodeRules) {
1159    opContext.deadCheck("findCodeInConcept: "+code.toString()+", "+concept.toString());
1160    if (code.equals(concept.getCode())) {
1161      return concept;
1162    }
1163    ConceptDefinitionComponent cc = findCodeInConcept(concept.getConcept(), code, caseSensitive, altCodeRules);
1164    if (cc != null) {
1165      return cc;
1166    }
1167    if (concept.hasUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK)) {
1168      List<ConceptDefinitionComponent> children = (List<ConceptDefinitionComponent>) concept.getUserData(CodeSystemUtilities.USER_DATA_CROSS_LINK);
1169      for (ConceptDefinitionComponent c : children) {
1170        cc = findCodeInConcept(c, code, caseSensitive, altCodeRules);
1171        if (cc != null) {
1172          return cc;
1173        }
1174      }
1175    }
1176    return null;
1177  }
1178  
1179  private ConceptDefinitionComponent findCodeInConcept(List<ConceptDefinitionComponent> concept, String code, boolean caseSensitive, AlternateCodesProcessingRules altCodeRules) {
1180    for (ConceptDefinitionComponent cc : concept) {
1181      if (code.equals(cc.getCode()) || (!caseSensitive && (code.equalsIgnoreCase(cc.getCode())))) {
1182        return cc;
1183      }
1184      if (Utilities.existsInList(code, alternateCodes(cc, altCodeRules))) {
1185        return cc;
1186      }
1187      ConceptDefinitionComponent c = findCodeInConcept(cc, code, caseSensitive, altCodeRules);
1188      if (c != null) {
1189        return c;
1190      }
1191    }
1192    return null;
1193  }
1194
1195
1196  private List<String> alternateCodes(ConceptDefinitionComponent focus, AlternateCodesProcessingRules altCodeRules) {
1197    List<String> codes = new ArrayList<>();
1198    for (ConceptPropertyComponent p : focus.getProperty()) {
1199      if ("alternateCode".equals(p.getCode()) && (altCodeRules.passes(p.getExtension())) && p.getValue().isPrimitive()) {
1200        codes.add(p.getValue().primitiveValue());        
1201      }
1202    }
1203    return codes;
1204  }
1205
1206  
1207  private String systemForCodeInValueSet(String code, List<StringWithCode> problems) {
1208    Set<String> sys = new HashSet<>();
1209    if (!scanForCodeInValueSet(code, sys, problems)) {
1210      return null;
1211    }
1212    if (sys.size() == 0) {
1213      problems.add(new StringWithCode(OpIssueCode.InferFailed, context.formatMessage(I18nConstants.UNABLE_TO_INFER_CODESYSTEM, code, valueset.getVersionedUrl())));
1214      return null;
1215    } else if (sys.size() > 1) {
1216      problems.add(new StringWithCode(OpIssueCode.InferFailed, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_SYSTEM__VALUE_SET_HAS_MULTIPLE_MATCHES, code, valueset.getVersionedUrl(), sys.toString())));
1217      return null;
1218    } else {
1219      return sys.iterator().next();
1220    }
1221  }
1222  
1223  private boolean scanForCodeInValueSet(String code, Set<String> sys, List<StringWithCode> problems) {
1224    if (valueset.hasCompose()) {
1225      //  ignore excludes - they can't make any difference
1226      if (!valueset.getCompose().hasInclude() && !valueset.getExpansion().hasContains()) {
1227        problems.add(new StringWithCode(OpIssueCode.InferFailed, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_SYSTEM__VALUE_SET_HAS_NO_INCLUDES_OR_EXPANSION, code, valueset.getVersionedUrl())));
1228      }
1229
1230      int i = 0;
1231      for (ConceptSetComponent vsi : valueset.getCompose().getInclude()) {
1232        opContext.deadCheck("scanForCodeInValueSet: "+code.toString());
1233        if (scanForCodeInValueSetInclude(code, sys, problems, i, vsi)) {
1234          return true;
1235        }
1236        i++;
1237      }
1238    } else if (valueset.hasExpansion()) {
1239      // Retrieve a list of all systems associated with this code in the expansion
1240      if (!checkSystems(valueset.getExpansion().getContains(), code, sys, problems)) {
1241        return false;
1242      }
1243    }
1244    return true;
1245  }
1246
1247  private boolean scanForCodeInValueSetInclude(String code, Set<String> sys, List<StringWithCode> problems, int i, ConceptSetComponent vsi) {
1248    if (vsi.hasValueSet()) {
1249      for (CanonicalType u : vsi.getValueSet()) {
1250        if (!checkForCodeInValueSet(code, getCu().pinValueSet(u.getValue(), expansionParameters), sys, problems)) {
1251          return false;
1252        }
1253      }
1254    } else if (!vsi.hasSystem()) { 
1255      problems.add(new StringWithCode(OpIssueCode.InferFailed, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_SYSTEM__VALUE_SET_HAS_INCLUDE_WITH_NO_SYSTEM, code, valueset.getVersionedUrl(), i)));
1256      return false;
1257    }
1258    if (vsi.hasSystem()) {
1259      if (vsi.hasFilter()) {
1260        ValueSet vsDummy = new ValueSet();
1261        vsDummy.setUrl(UUIDUtilities.makeUuidUrn());
1262        vsDummy.setStatus(PublicationStatus.ACTIVE);
1263        vsDummy.getCompose().addInclude(vsi);
1264        Coding c = new Coding().setCode(code).setSystem(vsi.getSystem());
1265        ValidationResult vr = context.validateCode(options.withGuessSystem(false), c, vsDummy);
1266        if (vr.isOk()) {
1267          sys.add(vsi.getSystem());
1268        } else {
1269          // problems.add(new StringWithCode(OpIssueCode.InferFailed, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_SYSTEM__VALUE_SET_HAS_INCLUDE_WITH_FILTER, code, valueset.getVersionedUrl(), i, vsi.getSystem(), filterSummary(vsi))));
1270          return false;
1271        }
1272      }
1273      CodeSystemProvider csp = CodeSystemProvider.factory(vsi.getSystem());
1274      if (csp != null) {
1275        Boolean ok = csp.checkCode(code);
1276        if (ok == null) {
1277          problems.add(new StringWithCode(OpIssueCode.InferFailed, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_SYSTEM_SYSTEM_IS_INDETERMINATE, code, valueset.getVersionedUrl(), vsi.getSystem())));
1278          sys.add(vsi.getSystem());
1279        } else if (ok) {
1280          sys.add(vsi.getSystem());
1281        }
1282      } else {
1283        CodeSystem cs = resolveCodeSystem(vsi.getSystem(), vsi.getVersion());
1284        if (cs != null && cs.getContent() == CodeSystemContentMode.COMPLETE) {
1285
1286          if (vsi.hasConcept()) {
1287            for (ConceptReferenceComponent cc : vsi.getConcept()) {
1288              boolean match = cs.getCaseSensitive() ? cc.getCode().equals(code) : cc.getCode().equalsIgnoreCase(code);
1289              if (match) {
1290                sys.add(vsi.getSystem());
1291              }
1292            }
1293          } else {
1294            ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), code, cs.getCaseSensitive(), allAltCodes);
1295            if (cc != null) {
1296              sys.add(vsi.getSystem());
1297            }
1298          }
1299        } else if (vsi.hasConcept()) {
1300          for (ConceptReferenceComponent cc : vsi.getConcept()) {
1301            boolean match = cc.getCode().equals(code);
1302            if (match) {
1303              sys.add(vsi.getSystem());
1304            }
1305          }
1306        } else if (!VersionUtilities.isR6Plus(context.getVersion()) && Utilities.existsInList(code, "xml", "json", "ttl") && "urn:ietf:bcp:13".equals(vsi.getSystem())) {
1307          sys.add(vsi.getSystem());
1308          return true;
1309        } else {
1310          ValueSet vsDummy = new ValueSet();
1311          vsDummy.setUrl(UUIDUtilities.makeUuidUrn());
1312          vsDummy.setStatus(PublicationStatus.ACTIVE);
1313          vsDummy.getCompose().addInclude(vsi);
1314          ValidationResult vr = context.validateCode(options.withNoClient(), code, vsDummy);
1315          if (vr.isOk()) {
1316            sys.add(vsi.getSystem());
1317          } else {
1318            // ok, we'll try to expand this one then 
1319            ValueSetExpansionOutcome vse = context.expandVS(new TerminologyOperationDetails(requiredSupplements), vsi, false, false);
1320            if (vse.isOk()) {
1321              if (!checkSystems(vse.getValueset().getExpansion().getContains(), code, sys, problems)) {
1322                return false;
1323              }
1324            } else {
1325              problems.add(new StringWithCode(OpIssueCode.NotFound, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_SYSTEM__VALUE_SET_HAS_INCLUDE_WITH_UNKNOWN_SYSTEM, code, valueset.getVersionedUrl(), i, vsi.getSystem(), vse.getAllErrors().toString())));              
1326              return false;
1327            }
1328
1329          }
1330        }
1331      }
1332    }
1333    return false;
1334  }
1335
1336  private String filterSummary(ConceptSetComponent vsi) {
1337    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
1338    for (ConceptSetFilterComponent f : vsi.getFilter()) {
1339      b.append(f.getProperty()+f.getOp().toCode()+f.getValue());
1340    }
1341    return b.toString();
1342  }
1343
1344  private boolean checkForCodeInValueSet(String code, String uri, Set<String> sys, List<StringWithCode> problems) {
1345    ValueSetValidator vs = getVs(uri, null);
1346    return vs.scanForCodeInValueSet(code, sys, problems);
1347  }
1348
1349  /*
1350   * Recursively go through all codes in the expansion and for any coding that matches the specified code, add the system for that coding
1351   * to the passed list. 
1352   */
1353  private boolean checkSystems(List<ValueSetExpansionContainsComponent> contains, String code, Set<String> systems, List<StringWithCode> problems) {
1354    for (ValueSetExpansionContainsComponent c: contains) {
1355      opContext.deadCheck("checkSystems "+code.toString());
1356      if (c.getCode().equals(code)) {
1357        systems.add(c.getSystem());
1358      }
1359      if (c.hasContains())
1360        checkSystems(c.getContains(), code, systems, problems);
1361    }
1362    return true;
1363  }
1364  
1365  public Boolean codeInValueSet(String path, String system, String version, String code, ValidationProcessInfo info) throws FHIRException {
1366    if (valueset == null) {
1367      return null;
1368    }
1369    opContext.deadCheck("codeInValueSet: "+system+"#"+code);
1370    checkCanonical(info.getIssues(), path, valueset, valueset);
1371    Boolean result = false;
1372    VersionInfo vi = new VersionInfo(this);
1373    String vspath = "ValueSet['"+valueset.getVersionedUrl()+"].compose"; 
1374      
1375    if (valueset.hasExpansion()) {
1376      return checkExpansion(new Coding(system, code, null), vi);
1377    } else if (valueset.hasCompose()) {
1378      int i = 0;
1379      int c = 0;
1380      for (ConceptSetComponent vsi : valueset.getCompose().getInclude()) {
1381        Boolean ok = inComponent(path, vsi, i, system, version, code, valueset.getCompose().getInclude().size() == 1, info, vspath+".include["+c+"]");
1382        i++;
1383        c++;
1384        if (ok == null && result != null && result == false) {
1385          result = null;
1386        } else if (ok != null && ok) {
1387          result = true;
1388          break;
1389        }
1390      }
1391      i = valueset.getCompose().getInclude().size();
1392      c = 0;
1393      for (ConceptSetComponent vsi : valueset.getCompose().getExclude()) {
1394        Boolean nok = inComponent(path, vsi, i, system, version, code, valueset.getCompose().getInclude().size() == 1, info, vspath+".exclude["+c+"]");
1395        i++;
1396        c++;
1397        if (nok == null && result != null && result == false) {
1398          result = null;
1399        } else if (nok != null && nok) {
1400          result = false;
1401        }
1402      }
1403    } 
1404
1405    return result;
1406  }
1407
1408  private Boolean inComponent(String path, ConceptSetComponent vsi, int vsiIndex, String system, String version, String code, boolean only, ValidationProcessInfo info, String vspath) throws FHIRException {
1409    opContext.deadCheck("inComponent "+vsiIndex);
1410    boolean ok = true;
1411    
1412    if (vsi.hasValueSet()) {
1413      if (isValueSetUnionImports()) {
1414        ok = false;
1415        for (UriType uri : vsi.getValueSet()) {
1416          if (inImport(path, getCu().pinValueSet(uri.getValue(), expansionParameters), system, version, code, info)) {
1417            return true;
1418          }
1419        }
1420      } else {
1421        Boolean bok = inImport(path, getCu().pinValueSet(vsi.getValueSet().get(0).getValue(), expansionParameters), system, version, code, info);
1422        if (bok == null) {
1423          return bok;
1424        }
1425        ok = bok;
1426        for (int i = 1; i < vsi.getValueSet().size(); i++) {
1427          UriType uri = vsi.getValueSet().get(i);
1428          ok = ok && inImport(path, getCu().pinValueSet(uri.getValue(), expansionParameters), system, version, code, info); 
1429        }
1430      }
1431    }
1432
1433    if (!vsi.hasSystem() || !ok) {
1434      return ok;
1435    }
1436    
1437    if (only && system == null) {
1438      // whether we know the system or not, we'll accept the stated codes at face value
1439      for (ConceptReferenceComponent cc : vsi.getConcept()) {
1440        if (cc.getCode().equals(code)) {
1441          return true;
1442        }
1443      }
1444    }
1445
1446    if (system == null || !system.equals(vsi.getSystem()))
1447      return false;
1448    // ok, we need the code system
1449    CodeSystem cs = resolveCodeSystem(system, version);
1450    if (cs == null || (cs.getContent() != CodeSystemContentMode.COMPLETE && cs.getContent() != CodeSystemContentMode.FRAGMENT)) {
1451      if (throwToServer) {
1452        // make up a transient value set with
1453        ValueSet vs = new ValueSet();
1454        vs.setStatus(PublicationStatus.ACTIVE);
1455        vs.setUrl(valueset.getUrl()+"--"+vsiIndex);
1456        vs.setVersion(valueset.getVersion());
1457        vs.getCompose().addInclude(vsi);
1458        opContext.deadCheck("hit server "+vs.getVersionedUrl());
1459        ValidationResult res = context.validateCode(options.withNoClient(), new Coding(system, code, null), vs);
1460        if (res.getErrorClass() == TerminologyServiceErrorClass.UNKNOWN || res.getErrorClass() == TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED || res.getErrorClass() == TerminologyServiceErrorClass.VALUESET_UNSUPPORTED) {
1461          if (info != null && res.getErrorClass() == TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED) {
1462            // server didn't know the code system either - we'll take it face value
1463            if (!info.hasNotFound(system)) {
1464              String msg = context.formatMessage(I18nConstants.UNKNOWN_CODESYSTEM, system);
1465              info.addIssue(makeIssue(IssueSeverity.WARNING, IssueType.UNKNOWN, path, msg, OpIssueCode.NotFound, null));
1466              for (ConceptReferenceComponent cc : vsi.getConcept()) {
1467                if (cc.getCode().equals(code)) {
1468                  opContext.deadCheck("server true");
1469                  return true;
1470                }
1471              }
1472            }
1473            info.setErr(TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED);
1474            opContext.deadCheck("server codesystem unsupported");
1475            return null;
1476          }
1477          opContext.deadCheck("server not found");
1478          return false;
1479        }
1480        if (res.getErrorClass() == TerminologyServiceErrorClass.NOSERVICE) {
1481          opContext.deadCheck("server no server");
1482          throw new NoTerminologyServiceException();
1483        }
1484        return res.isOk();
1485      } else {
1486        info.setErr(TerminologyServiceErrorClass.CODESYSTEM_UNSUPPORTED);
1487        if (unknownSystems != null) {
1488          if (version == null) {
1489            unknownSystems.add(system);
1490          } else {
1491            unknownSystems.add(system+"|"+version);          
1492          }
1493        }
1494        return null;
1495      }
1496    } else {
1497      checkCanonical(info.getIssues(), path, cs, valueset);
1498      if ((valueset.getCompose().hasInactive() && !valueset.getCompose().getInactive()) || options.isActiveOnly()) {
1499        if (CodeSystemUtilities.isInactive(cs, code)) {
1500          info.addIssue(makeIssue(IssueSeverity.ERROR, IssueType.BUSINESSRULE, path+".code", context.formatMessage(I18nConstants.STATUS_CODE_WARNING_CODE, "not active", code), OpIssueCode.CodeRule, null));        
1501          return false;
1502        }
1503      }
1504      
1505      if (vsi.hasFilter()) {
1506        ok = true;
1507        int i = 0;
1508        for (ConceptSetFilterComponent f : vsi.getFilter()) {
1509          if (!codeInFilter(cs, vspath+".filter["+i+"]", system, f, code)) {
1510            return false;
1511          }
1512          i++;
1513        }
1514      }
1515
1516      List<ConceptDefinitionComponent> list = cs.getConcept();
1517      ok = validateCodeInConceptList(code, cs, list, allAltCodes);
1518      if (ok && vsi.hasConcept()) {
1519        for (ConceptReferenceComponent cc : vsi.getConcept()) {
1520          if (cc.getCode().equals(code)) { 
1521            return true;
1522          }
1523        }
1524        return false;
1525      } else {
1526        // recheck that this is a valid alternate code
1527        ok = validateCodeInConceptList(code, cs, list, altCodeParams);
1528        return ok;
1529      }
1530    }
1531  }
1532
1533  protected boolean isValueSetUnionImports() {
1534    PackageInformation p = (PackageInformation) valueset.getSourcePackage();
1535    if (p != null) {
1536      return p.getDate().before(new GregorianCalendar(2022, Calendar.MARCH, 31).getTime());
1537    } else {
1538      return false;
1539    }
1540  }
1541
1542  private boolean codeInFilter(CodeSystem cs, String path, String system, ConceptSetFilterComponent f, String code) throws FHIRException {
1543    String v = f.getValue();
1544    if (v == null) {
1545      List<OperationOutcomeIssueComponent> issues = new ArrayList<>();
1546      issues.addAll(makeIssue(IssueSeverity.ERROR, IssueType.INVALID, path+".value", context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE, cs.getUrl(), f.getProperty(), f.getOp().toCode()), OpIssueCode.VSProcessing, null)); 
1547      throw new VSCheckerException(context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM_FILTER_WITH_NO_VALUE, cs.getUrl(), f.getProperty(), f.getOp().toCode()), issues, TerminologyServiceErrorClass.INTERNAL_ERROR);
1548      
1549    }
1550    if ("concept".equals(f.getProperty()))
1551      return codeInConceptFilter(cs, f, code);
1552    else if ("code".equals(f.getProperty()) && f.getOp() == FilterOperator.REGEX)
1553      return codeInRegexFilter(cs, f, code);
1554    else if (CodeSystemUtilities.isDefinedProperty(cs, f.getProperty())) {
1555      return codeInPropertyFilter(cs, f, code);
1556    } else if (isKnownProperty(f.getProperty())) {
1557      return codeInKnownPropertyFilter(cs, f, code);
1558    } else {
1559      log.error("todo: handle filters with property = "+f.getProperty()+" "+f.getOp().toCode());
1560      throw new FHIRException(context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM__FILTER_WITH_PROPERTY__, cs.getUrl(), f.getProperty(), f.getOp().toCode()));
1561    }
1562  }
1563
1564  private boolean isKnownProperty(String code) {
1565    return Utilities.existsInList(code, "notSelectable");
1566  }
1567
1568  private boolean codeInPropertyFilter(CodeSystem cs, ConceptSetFilterComponent f, String code) {
1569    switch (f.getOp()) {
1570    case EQUAL:
1571      if (f.getValue() == null) {
1572        return false;
1573      }
1574      DataType d = CodeSystemUtilities.getProperty(cs, code, f.getProperty());
1575      return d != null && f.getValue().equals(d.primitiveValue());
1576    case EXISTS: 
1577      return CodeSystemUtilities.getProperty(cs, code, f.getProperty()) != null;
1578    case REGEX:
1579      if (f.getValue() == null) {
1580        return false;
1581      }
1582      d = CodeSystemUtilities.getProperty(cs, code, f.getProperty());
1583      return d != null && d.primitiveValue() != null && d.primitiveValue().matches(f.getValue());
1584    case IN:
1585      if (f.getValue() == null) {
1586        return false;
1587      }
1588      String[] values = f.getValue().split("\\,");
1589      d = CodeSystemUtilities.getProperty(cs, code, f.getProperty());
1590      if (d != null) {
1591        String v = d.primitiveValue();
1592        for (String value : values) {
1593          if (v != null && v.equals(value.trim())) {
1594            return true;
1595          }
1596        }
1597      }
1598      return false;
1599    case NOTIN:
1600      if (f.getValue() == null) {
1601        return true;
1602      }
1603      values = f.getValue().split("\\,");
1604      d = CodeSystemUtilities.getProperty(cs, code, f.getProperty());
1605      if (d != null) {
1606        String v = d.primitiveValue();
1607        for (String value : values) {
1608          if (v != null && v.equals(value.trim())) {
1609            return false;
1610          }
1611        }
1612      }
1613      return true;
1614    default:
1615      log.error("todo: handle property filters with op = "+f.getOp());
1616      throw new FHIRException(context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM__PROPERTY_FILTER_WITH_OP__, cs.getUrl(), f.getOp()));
1617    }
1618  }
1619  
1620  private boolean codeInKnownPropertyFilter(CodeSystem cs, ConceptSetFilterComponent f, String code) {
1621
1622    switch (f.getOp()) {
1623    case EQUAL:
1624      if (f.getValue() == null) {
1625        return false;
1626      }
1627      DataType d = CodeSystemUtilities.getProperty(cs, code, f.getProperty());
1628      return d != null && f.getValue().equals(d.primitiveValue());
1629    case EXISTS: 
1630      return CodeSystemUtilities.getProperty(cs, code, f.getProperty()) != null;
1631    case REGEX:
1632      if (f.getValue() == null) {
1633        return false;
1634      }
1635      d = CodeSystemUtilities.getProperty(cs, code, f.getProperty());
1636      return d != null && d.primitiveValue() != null && d.primitiveValue().matches(f.getValue());
1637    default:
1638      log.error("todo: handle known property filters with op = "+f.getOp());
1639      throw new FHIRException(context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM__PROPERTY_FILTER_WITH_OP__, cs.getUrl(), f.getOp()));
1640    }
1641  }
1642
1643  private boolean codeInRegexFilter(CodeSystem cs, ConceptSetFilterComponent f, String code) {
1644    return code.matches(f.getValue());
1645  }
1646
1647  private boolean codeInConceptFilter(CodeSystem cs, ConceptSetFilterComponent f, String code) throws FHIRException {
1648    switch (f.getOp()) {
1649    case ISA: return codeInConceptIsAFilter(cs, f, code, false);
1650    case ISNOTA: return !codeInConceptIsAFilter(cs, f, code, false);
1651    case DESCENDENTOF: return codeInConceptIsAFilter(cs, f, code, true); 
1652    default:
1653      log.error("todo: handle concept filters with op = "+f.getOp());
1654      throw new FHIRException(context.formatMessage(I18nConstants.UNABLE_TO_HANDLE_SYSTEM__CONCEPT_FILTER_WITH_OP__, cs.getUrl(), f.getOp()));
1655    }
1656  }
1657
1658  private boolean codeInConceptIsAFilter(CodeSystem cs, ConceptSetFilterComponent f, String code, boolean excludeRoot) {
1659    if (!excludeRoot && code.equals(f.getValue())) {
1660      return true;
1661    }
1662    ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), f.getValue(), cs.getCaseSensitive(), altCodeParams);
1663    if (cc == null) {
1664      return false;
1665    }
1666    ConceptDefinitionComponent cc2 = findCodeInConcept(cc, code, cs.getCaseSensitive(), altCodeParams);
1667    return cc2 != null && cc2 != cc;
1668  }
1669
1670  public boolean validateCodeInConceptList(String code, CodeSystem def, List<ConceptDefinitionComponent> list, AlternateCodesProcessingRules altCodeRules) {
1671    opContext.deadCheck("validateCodeInConceptList");
1672    if (def.hasUserData(UserDataNames.tx_cs_special)) {
1673      return ((SpecialCodeSystem) def.getUserData(UserDataNames.tx_cs_special)).findConcept(new Coding().setCode(code)) != null; 
1674    } else if (def.getCaseSensitive()) {
1675      for (ConceptDefinitionComponent cc : list) {
1676        if (cc.getCode().equals(code)) { 
1677          return true;
1678        }
1679        if (Utilities.existsInList(code, alternateCodes(cc, altCodeRules))) {
1680          return true;
1681        }
1682        if (cc.hasConcept() && validateCodeInConceptList(code, def, cc.getConcept(), altCodeRules)) {
1683          return true;
1684        }
1685      }
1686    } else {
1687      for (ConceptDefinitionComponent cc : list) {
1688        if (cc.getCode().equalsIgnoreCase(code)) { 
1689          return true;
1690        }
1691        if (cc.hasConcept() && validateCodeInConceptList(code, def, cc.getConcept(), altCodeRules)) {
1692          return true;
1693        }
1694      }
1695    }
1696    return false;
1697  }
1698
1699  private ValueSetValidator getVs(String url, ValidationProcessInfo info) {
1700    if (inner.containsKey(url)) {
1701      return inner.get(url);
1702    }
1703    ValueSet vs = context.findTxResource(ValueSet.class, url, valueset);
1704    if (vs == null && info != null) {
1705      unknownValueSets.add(url);
1706      info.addIssue(makeIssue(IssueSeverity.ERROR, IssueType.NOTFOUND, null, context.formatMessage(I18nConstants.UNABLE_TO_RESOLVE_VALUE_SET_, url), OpIssueCode.NotFound, null));
1707    }
1708    ValueSetValidator vsc = new ValueSetValidator(context, opContext.copy(), options, vs, localContext, expansionParameters, tcm, registry);
1709    vsc.setThrowToServer(throwToServer);
1710    inner.put(url, vsc);
1711    return vsc;
1712  }
1713  
1714  private Boolean inImport(String path, String uri, String system, String version, String code, ValidationProcessInfo info) throws FHIRException {
1715    ValueSetValidator vs = getVs(uri, info);
1716    if (vs == null) {
1717      return false;
1718    } else {
1719      Boolean ok = vs.codeInValueSet(path, system, version, code, info);
1720      return ok;
1721    }
1722  }
1723
1724
1725  private String getPreferredDisplay(ConceptReferenceComponent cc) {
1726    if (!options.hasLanguages()) {
1727      return cc.getDisplay();
1728    }
1729    if (LanguageUtils.langsMatch(options.getLanguages(), valueset.getLanguage())) {
1730      return cc.getDisplay();
1731    }
1732    // if there's no language, we default to accepting the displays as (US) English
1733    if (valueset.getLanguage() == null && (options.langSummary().contains("en") || options.langSummary().contains("en-US"))) {
1734      return cc.getDisplay();
1735    }
1736    for (ConceptReferenceDesignationComponent d : cc.getDesignation()) {
1737      if (!d.hasUse() && LanguageUtils.langsMatch(options.getLanguages(), d.getLanguage())) {
1738        return d.getValue();
1739      }
1740    }
1741    for (ConceptReferenceDesignationComponent d : cc.getDesignation()) {
1742      if (LanguageUtils.langsMatch(options.getLanguages(), d.getLanguage())) {
1743        return d.getValue();
1744      }
1745    }
1746    return cc.getDisplay();
1747  }
1748
1749
1750  private String getPreferredDisplay(ConceptDefinitionComponent cc, CodeSystem cs) {
1751    if (!options.hasLanguages()) {
1752      return cc.getDisplay();
1753    }
1754    if (cs != null && LanguageUtils.langsMatch(options.getLanguages(), cs.getLanguage())) {
1755      return cc.getDisplay();
1756    }
1757    // if there's no language, we default to accepting the displays as (US) English
1758    if ((cs == null || cs.getLanguage() == null) && (options.langSummary().contains("en") || options.langSummary().contains("en-US"))) {
1759      return cc.getDisplay();
1760    }
1761    for (ConceptDefinitionDesignationComponent d : cc.getDesignation()) {
1762      if (!d.hasUse() && LanguageUtils.langsMatch(options.getLanguages(), d.getLanguage())) {
1763        return d.getValue();
1764      }
1765    }
1766    for (ConceptDefinitionDesignationComponent d : cc.getDesignation()) {
1767      if (LanguageUtils.langsMatch(options.getLanguages(), d.getLanguage())) {
1768        return d.getValue();
1769      }
1770    }
1771    return cc.getDisplay();
1772  }
1773
1774}