Source code

001package org.hl7.fhir.dstu3.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.ByteArrayOutputStream;
035import java.io.FileNotFoundException;
036import java.io.FileOutputStream;
037import java.io.IOException;
038import java.util.ArrayList;
039import java.util.HashMap;
040import java.util.HashSet;
041import java.util.List;
042import java.util.Locale;
043import java.util.Map;
044import java.util.ResourceBundle;
045import java.util.Set;
046
047import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
048import org.hl7.fhir.dstu3.formats.JsonParser;
049import org.hl7.fhir.dstu3.model.BooleanType;
050import org.hl7.fhir.dstu3.model.Bundle;
051import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
052import org.hl7.fhir.dstu3.model.CodeSystem;
053import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemHierarchyMeaning;
054import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
055import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
056import org.hl7.fhir.dstu3.model.CodeableConcept;
057import org.hl7.fhir.dstu3.model.Coding;
058import org.hl7.fhir.dstu3.model.ConceptMap;
059import org.hl7.fhir.dstu3.model.DataElement;
060import org.hl7.fhir.dstu3.model.ExpansionProfile;
061import org.hl7.fhir.dstu3.model.IntegerType;
062import org.hl7.fhir.dstu3.model.OperationDefinition;
063import org.hl7.fhir.dstu3.model.OperationOutcome;
064import org.hl7.fhir.dstu3.model.Parameters;
065import org.hl7.fhir.dstu3.model.Parameters.ParametersParameterComponent;
066import org.hl7.fhir.dstu3.model.PrimitiveType;
067import org.hl7.fhir.dstu3.model.Questionnaire;
068import org.hl7.fhir.dstu3.model.Reference;
069import org.hl7.fhir.dstu3.model.Resource;
070import org.hl7.fhir.dstu3.model.SearchParameter;
071import org.hl7.fhir.dstu3.model.StringType;
072import org.hl7.fhir.dstu3.model.StructureDefinition;
073import org.hl7.fhir.dstu3.model.StructureDefinition.TypeDerivationRule;
074import org.hl7.fhir.dstu3.model.StructureMap;
075import org.hl7.fhir.dstu3.model.UriType;
076import org.hl7.fhir.dstu3.model.ValueSet;
077import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
078import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
079import org.hl7.fhir.dstu3.model.ValueSet.ValueSetComposeComponent;
080import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
081import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
082import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ETooCostly;
083import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
084import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
085import org.hl7.fhir.dstu3.terminologies.ValueSetExpanderFactory;
086import org.hl7.fhir.dstu3.terminologies.ValueSetExpansionCache;
087import org.hl7.fhir.dstu3.utils.ToolingExtensions;
088import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
089import org.hl7.fhir.exceptions.FHIRException;
090import org.hl7.fhir.exceptions.NoTerminologyServiceException;
091import org.hl7.fhir.exceptions.TerminologyServiceException;
092import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
093import org.hl7.fhir.utilities.FileUtilities;
094import org.hl7.fhir.utilities.UUIDUtilities;
095import org.hl7.fhir.utilities.Utilities;
096import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
097import org.hl7.fhir.utilities.i18n.I18nBase;
098import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
099import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
100
101import com.google.gson.JsonObject;
102import com.google.gson.JsonSyntaxException;
103
104import ca.uhn.fhir.rest.api.Constants;
105
106
107@Deprecated
108public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext {
109
110  // all maps are to the full URI
111  protected Map<String, CodeSystem> codeSystems = new HashMap<String, CodeSystem>();
112  protected Set<String> nonSupportedCodeSystems = new HashSet<String>();
113  protected Map<String, ValueSet> valueSets = new HashMap<String, ValueSet>();
114  protected Map<String, ConceptMap> maps = new HashMap<String, ConceptMap>();
115  protected Map<String, StructureMap> transforms = new HashMap<String, StructureMap>();
116  protected Map<String, DataElement> dataElements = new HashMap<String, DataElement>();
117  protected Map<String, StructureDefinition> profiles = new HashMap<String, StructureDefinition>();
118  protected Map<String, SearchParameter> searchParameters = new HashMap<String, SearchParameter>();
119  protected Map<String, StructureDefinition> extensionDefinitions = new HashMap<String, StructureDefinition>();
120  protected Map<String, Questionnaire> questionnaires = new HashMap<String, Questionnaire>();
121  protected Map<String, OperationDefinition> operations = new HashMap<String, OperationDefinition>();
122
123  protected ValueSetExpanderFactory expansionCache = new ValueSetExpansionCache(this);
124  protected boolean cacheValidation; // if true, do an expansion and cache the expansion
125  private Set<String> failed = new HashSet<String>(); // value sets for which we don't try to do expansion, since the first attempt to get a comprehensive expansion was not successful
126  protected Map<String, Map<String, ValidationResult>> validationCache = new HashMap<String, Map<String, ValidationResult>>();
127  protected String tsServer;
128  protected String validationCachePath;
129  protected String name;
130
131  // private ValueSetExpansionCache expansionCache; //   
132
133  protected FHIRToolingClient txServer;
134  private Bundle bndCodeSystems;
135  private boolean canRunWithoutTerminology;
136  protected boolean allowLoadingDuplicates;
137  protected boolean noTerminologyServer;
138  protected String cache;
139  private int expandCodesLimit = 1000;
140  protected ILoggingService logger;
141  protected ExpansionProfile expProfile;
142  private Locale locale;
143  private ResourceBundle i18Nmessages;
144
145  public Map<String, CodeSystem> getCodeSystems() {
146    return codeSystems;
147  }
148
149  public Map<String, DataElement> getDataElements() {
150    return dataElements;
151  }
152
153  public Map<String, ValueSet> getValueSets() {
154    return valueSets;
155  }
156
157  public Map<String, ConceptMap> getMaps() {
158    return maps;
159  }
160
161  public Map<String, StructureDefinition> getProfiles() {
162    return profiles;
163  }
164
165  public Map<String, StructureDefinition> getExtensionDefinitions() {
166    return extensionDefinitions;
167  }
168
169  public Map<String, Questionnaire> getQuestionnaires() {
170    return questionnaires;
171  }
172
173  public Map<String, OperationDefinition> getOperations() {
174    return operations;
175  }
176
177  public void seeExtensionDefinition(String url, StructureDefinition ed) throws Exception {
178    if (extensionDefinitions.get(ed.getUrl()) != null) {
179      throw new Exception("duplicate extension definition: " + ed.getUrl());
180    }
181    extensionDefinitions.put(ed.getId(), ed);
182    extensionDefinitions.put(url, ed);
183    extensionDefinitions.put(ed.getUrl(), ed);
184  }
185
186  public void dropExtensionDefinition(String id) {
187    StructureDefinition sd = extensionDefinitions.get(id);
188    extensionDefinitions.remove(id);
189    if (sd != null) {
190      extensionDefinitions.remove(sd.getUrl());
191    }
192  }
193
194  public void seeQuestionnaire(String url, Questionnaire theQuestionnaire) throws Exception {
195    if (questionnaires.get(theQuestionnaire.getId()) != null) {
196      throw new Exception("duplicate extension definition: " + theQuestionnaire.getId());
197    }
198    questionnaires.put(theQuestionnaire.getId(), theQuestionnaire);
199    questionnaires.put(url, theQuestionnaire);
200  }
201
202  public void seeOperation(OperationDefinition opd) throws Exception {
203    if (operations.get(opd.getUrl()) != null) {
204      throw new Exception("duplicate extension definition: " + opd.getUrl());
205    }
206    operations.put(opd.getUrl(), opd);
207    operations.put(opd.getId(), opd);
208  }
209
210  public void seeValueSet(String url, ValueSet vs) throws Exception {
211    if (valueSets.containsKey(vs.getUrl()) && !allowLoadingDuplicates) {
212      throw new Exception("Duplicate value set " + vs.getUrl());
213    }
214    valueSets.put(vs.getId(), vs);
215    valueSets.put(url, vs);
216    valueSets.put(vs.getUrl(), vs);
217  }
218
219  public void dropValueSet(String id) {
220    ValueSet vs = valueSets.get(id);
221    valueSets.remove(id);
222    if (vs != null) {
223      valueSets.remove(vs.getUrl());
224    }
225  }
226
227  public void seeCodeSystem(String url, CodeSystem cs) throws FHIRException {
228    if (codeSystems.containsKey(cs.getUrl()) && !allowLoadingDuplicates) {
229      throw new FHIRException("Duplicate code system " + cs.getUrl());
230    }
231    codeSystems.put(cs.getId(), cs);
232    codeSystems.put(url, cs);
233    codeSystems.put(cs.getUrl(), cs);
234  }
235
236  public void dropCodeSystem(String id) {
237    CodeSystem cs = codeSystems.get(id);
238    codeSystems.remove(id);
239    if (cs != null) {
240      codeSystems.remove(cs.getUrl());
241    }
242  }
243
244  public void seeProfile(String url, StructureDefinition p) throws Exception {
245    if (profiles.containsKey(p.getUrl())) {
246      throw new Exception("Duplicate Profile " + p.getUrl());
247    }
248    profiles.put(p.getId(), p);
249    profiles.put(url, p);
250    profiles.put(p.getUrl(), p);
251  }
252
253  public void dropProfile(String id) {
254    StructureDefinition sd = profiles.get(id);
255    profiles.remove(id);
256    if (sd != null) {
257      profiles.remove(sd.getUrl());
258    }
259  }
260
261  @Override
262  public CodeSystem fetchCodeSystem(String system) {
263    return codeSystems.get(system);
264  }
265
266  @Override
267  public boolean supportsSystem(String system) throws TerminologyServiceException {
268    if (codeSystems.containsKey(system)) {
269      return true;
270    } else if (nonSupportedCodeSystems.contains(system)) {
271      return false;
272    } else if (system.startsWith("http://example.org") || system.startsWith("http://acme.com")
273      || system.startsWith("http://hl7.org/fhir/valueset-") || system.startsWith("urn:oid:")) {
274      return false;
275    } else {
276      if (noTerminologyServer) {
277        return false;
278      }
279      if (bndCodeSystems == null) {
280        try {
281          tlog("Terminology server: Check for supported code systems for " + system);
282          bndCodeSystems = txServer.fetchFeed(txServer.getAddress()
283            + "/CodeSystem?content-mode=not-present&_summary=true&_count=1000");
284        } catch (Exception e) {
285          if (canRunWithoutTerminology) {
286            noTerminologyServer = true;
287            log("==============!! Running without terminology server !!============== (" + e
288              .getMessage() + ")");
289            return false;
290          } else {
291            throw new TerminologyServiceException(e);
292          }
293        }
294      }
295      if (bndCodeSystems != null) {
296        for (BundleEntryComponent be : bndCodeSystems.getEntry()) {
297          CodeSystem cs = (CodeSystem) be.getResource();
298          if (!codeSystems.containsKey(cs.getUrl())) {
299            codeSystems.put(cs.getUrl(), null);
300          }
301        }
302      }
303      if (codeSystems.containsKey(system)) {
304        return true;
305      }
306    }
307    nonSupportedCodeSystems.add(system);
308    return false;
309  }
310
311  private void log(String message) {
312    if (logger != null) {
313      logger.logMessage(message);
314    } else {
315      System.out.println(message);
316    }
317  }
318
319  @Override
320  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical) {
321    try {
322      if (vs.hasExpansion()) {
323        return new ValueSetExpansionOutcome(vs.copy());
324      }
325      String cacheFn = null;
326      if (cache != null) {
327        cacheFn = Utilities.path(cache, determineCacheId(vs, heirarchical) + ".json");
328        if (ManagedFileAccess.file(cacheFn).exists()) {
329          return loadFromCache(vs.copy(), cacheFn);
330        }
331      }
332      if (cacheOk && vs.hasUrl()) {
333        if (expProfile == null) {
334          throw new Exception("No ExpansionProfile provided");
335        }
336        ValueSetExpansionOutcome vse = expansionCache.getExpander()
337          .expand(vs, expProfile.setExcludeNested(!heirarchical));
338        if (vse.getValueset() != null) {
339          if (cache != null) {
340            FileOutputStream s = ManagedFileAccess.outStream(cacheFn);
341            newJsonParser().compose(ManagedFileAccess.outStream(cacheFn), vse.getValueset());
342            s.close();
343          }
344        }
345        return vse;
346      } else {
347        ValueSetExpansionOutcome res = expandOnServer(vs, cacheFn);
348        if (cacheFn != null) {
349          if (res.getValueset() != null) {
350            saveToCache(res.getValueset(), cacheFn);
351          } else {
352            OperationOutcome oo = new OperationOutcome();
353            oo.addIssue().getDetails().setText(res.getError());
354            saveToCache(oo, cacheFn);
355          }
356        }
357        return res;
358      }
359    } catch (NoTerminologyServiceException e) {
360      return new ValueSetExpansionOutcome(
361        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
362        TerminologyServiceErrorClass.NOSERVICE);
363    } catch (Exception e) {
364      return new ValueSetExpansionOutcome(
365        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
366        TerminologyServiceErrorClass.UNKNOWN);
367    }
368  }
369
370  private ValueSetExpansionOutcome loadFromCache(ValueSet vs, String cacheFn)
371    throws FileNotFoundException, Exception {
372    JsonParser parser = new JsonParser();
373    Resource r = parser.parse(ManagedFileAccess.inStream(cacheFn));
374    if (r instanceof OperationOutcome) {
375      return new ValueSetExpansionOutcome(
376        ((OperationOutcome) r).getIssue().get(0).getDetails().getText(),
377        TerminologyServiceErrorClass.UNKNOWN);
378    } else {
379      vs.setExpansion(((ValueSet) r)
380        .getExpansion()); // because what is cached might be from a different value set
381      return new ValueSetExpansionOutcome(vs);
382    }
383  }
384
385  private void saveToCache(Resource res, String cacheFn) throws FileNotFoundException, Exception {
386    JsonParser parser = new JsonParser();
387    parser.compose(ManagedFileAccess.outStream(cacheFn), res);
388  }
389
390  private String determineCacheId(ValueSet vs, boolean heirarchical) throws Exception {
391    // just the content logical definition is hashed
392    ValueSet vsid = new ValueSet();
393    vsid.setCompose(vs.getCompose());
394    JsonParser parser = new JsonParser();
395    parser.setOutputStyle(OutputStyle.NORMAL);
396    ByteArrayOutputStream b = new ByteArrayOutputStream();
397    parser.compose(b, vsid);
398    b.close();
399    String s = new String(b.toByteArray(), Constants.CHARSET_UTF8);
400    // any code systems we can find, we add these too. 
401    for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
402      CodeSystem cs = fetchCodeSystem(inc.getSystem());
403      if (cs != null) {
404        String css = cacheValue(cs);
405        s = s + css;
406      }
407    }
408    s = s + "-" + Boolean.toString(heirarchical);
409    String r = Integer.toString(s.hashCode());
410    //    FileUtilities.stringToFile(s, Utilities.path(cache, r+".id.json"));
411    return r;
412  }
413
414
415  private String cacheValue(CodeSystem cs) throws IOException {
416    CodeSystem csid = new CodeSystem();
417    csid.setId(cs.getId());
418    csid.setVersion(cs.getVersion());
419    csid.setContent(cs.getContent());
420    csid.setHierarchyMeaning(CodeSystemHierarchyMeaning.GROUPEDBY);
421    for (ConceptDefinitionComponent cc : cs.getConcept()) {
422      csid.getConcept().add(processCSConcept(cc));
423    }
424    JsonParser parser = new JsonParser();
425    parser.setOutputStyle(OutputStyle.NORMAL);
426    ByteArrayOutputStream b = new ByteArrayOutputStream();
427    parser.compose(b, csid);
428    b.close();
429    return new String(b.toByteArray(), Constants.CHARSET_UTF8);
430  }
431
432
433  private ConceptDefinitionComponent processCSConcept(ConceptDefinitionComponent cc) {
434    ConceptDefinitionComponent ccid = new ConceptDefinitionComponent();
435    ccid.setCode(cc.getCode());
436    ccid.setDisplay(cc.getDisplay());
437    for (ConceptDefinitionComponent cci : cc.getConcept()) {
438      ccid.getConcept().add(processCSConcept(cci));
439    }
440    return ccid;
441  }
442
443  public ValueSetExpansionOutcome expandOnServer(ValueSet vs, String fn) throws Exception {
444    if (noTerminologyServer) {
445      return new ValueSetExpansionOutcome(
446        "Error expanding ValueSet: running without terminology services",
447        TerminologyServiceErrorClass.NOSERVICE);
448    }
449    if (expProfile == null) {
450      throw new Exception("No ExpansionProfile provided");
451    }
452
453    try {
454      Parameters params = new Parameters();
455      params.addParameter().setName("profile").setResource(expProfile.setIncludeDefinition(false));
456      params.addParameter().setName("_limit").setValue(new IntegerType(expandCodesLimit));
457      params.addParameter().setName("_incomplete").setValue(new BooleanType("true"));
458      tlog("Terminology Server: $expand on " + getVSSummary(vs));
459      ValueSet result = txServer.expandValueset(vs, params);
460      return new ValueSetExpansionOutcome(result);
461    } catch (Exception e) {
462      return new ValueSetExpansionOutcome(
463        "Error expanding ValueSet \"" + vs.getUrl() + ": " + e.getMessage(),
464        TerminologyServiceErrorClass.UNKNOWN);
465    }
466  }
467
468  private String getVSSummary(ValueSet vs) {
469    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
470    for (ConceptSetComponent cc : vs.getCompose().getInclude()) {
471      b.append("Include " + getIncSummary(cc));
472    }
473    for (ConceptSetComponent cc : vs.getCompose().getExclude()) {
474      b.append("Exclude " + getIncSummary(cc));
475    }
476    return b.toString();
477  }
478
479  private String getIncSummary(ConceptSetComponent cc) {
480    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
481    for (UriType vs : cc.getValueSet()) {
482      b.append(vs.asStringValue());
483    }
484    String vsd =
485      b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
486    String system = cc.getSystem();
487    if (cc.hasConcept()) {
488      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
489    }
490    if (cc.hasFilter()) {
491      String s = "";
492      for (ConceptSetFilterComponent f : cc.getFilter()) {
493        if (!Utilities.noString(s)) {
494          s = s + " & ";
495        }
496        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
497      }
498      return "from " + system + " where " + s + vsd;
499    }
500    return "All codes from " + system + vsd;
501  }
502
503  private ValidationResult handleByCache(ValueSet vs, Coding coding, boolean tryCache) {
504    String cacheId = cacheId(coding);
505    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
506    if (cache == null) {
507      cache = new HashMap<String, IWorkerContext.ValidationResult>();
508      validationCache.put(vs.getUrl(), cache);
509    }
510    if (cache.containsKey(cacheId)) {
511      return cache.get(cacheId);
512    }
513    if (!tryCache) {
514      return null;
515    }
516    if (!cacheValidation) {
517      return null;
518    }
519    if (failed.contains(vs.getUrl())) {
520      return null;
521    }
522    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
523    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
524      failed.add(vs.getUrl());
525      return null;
526    }
527
528    ValidationResult res = validateCode(coding, vse.getValueset());
529    cache.put(cacheId, res);
530    return res;
531  }
532
533  private boolean notcomplete(ValueSet vs) {
534    if (!vs.hasExpansion()) {
535      return true;
536    }
537    if (!vs.getExpansion()
538      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-unclosed").isEmpty()) {
539      return true;
540    }
541    if (!vs.getExpansion()
542      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-toocostly").isEmpty()) {
543      return true;
544    }
545    return false;
546  }
547
548  private ValidationResult handleByCache(ValueSet vs, CodeableConcept concept, boolean tryCache) {
549    String cacheId = cacheId(concept);
550    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
551    if (cache == null) {
552      cache = new HashMap<String, IWorkerContext.ValidationResult>();
553      validationCache.put(vs.getUrl(), cache);
554    }
555    if (cache.containsKey(cacheId)) {
556      return cache.get(cacheId);
557    }
558
559    if (validationCache.containsKey(vs.getUrl()) && validationCache.get(vs.getUrl())
560      .containsKey(cacheId)) {
561      return validationCache.get(vs.getUrl()).get(cacheId);
562    }
563    if (!tryCache) {
564      return null;
565    }
566    if (!cacheValidation) {
567      return null;
568    }
569    if (failed.contains(vs.getUrl())) {
570      return null;
571    }
572    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
573    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
574      failed.add(vs.getUrl());
575      return null;
576    }
577    ValidationResult res = validateCode(concept, vse.getValueset());
578    cache.put(cacheId, res);
579    return res;
580  }
581
582  private String cacheId(Coding coding) {
583    return "|" + coding.getSystem() + "|" + coding.getVersion() + "|" + coding.getCode() + "|"
584      + coding.getDisplay();
585  }
586
587  private String cacheId(CodeableConcept cc) {
588    StringBuilder b = new StringBuilder();
589    for (Coding c : cc.getCoding()) {
590      b.append("#");
591      b.append(cacheId(c));
592    }
593    return b.toString();
594  }
595
596  private ValidationResult verifyCodeExternal(ValueSet vs, Coding coding, boolean tryCache)
597    throws Exception {
598    ValidationResult res = vs == null ? null : handleByCache(vs, coding, tryCache);
599    if (res != null) {
600      return res;
601    }
602    Parameters pin = new Parameters();
603    pin.addParameter().setName("coding").setValue(coding);
604    if (vs != null) {
605      pin.addParameter().setName("valueSet").setResource(vs);
606    }
607    res = serverValidateCode(pin, vs == null);
608    if (vs != null) {
609      Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
610      cache.put(cacheId(coding), res);
611    }
612    return res;
613  }
614
615  private ValidationResult verifyCodeExternal(ValueSet vs, CodeableConcept cc, boolean tryCache)
616    throws Exception {
617    ValidationResult res = handleByCache(vs, cc, tryCache);
618    if (res != null) {
619      return res;
620    }
621    Parameters pin = new Parameters();
622    pin.addParameter().setName("codeableConcept").setValue(cc);
623    pin.addParameter().setName("valueSet").setResource(vs);
624    res = serverValidateCode(pin, false);
625    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
626    cache.put(cacheId(cc), res);
627    return res;
628  }
629
630  private ValidationResult serverValidateCode(Parameters pin, boolean doCache) throws Exception {
631    if (noTerminologyServer) {
632      return new ValidationResult(null, null, TerminologyServiceErrorClass.NOSERVICE);
633    }
634    String cacheName = doCache ? generateCacheName(pin) : null;
635    ValidationResult res = loadFromCache(cacheName);
636    if (res != null) {
637      return res;
638    }
639    tlog("Terminology Server: $validate-code " + describeValidationParameters(pin));
640    for (ParametersParameterComponent pp : pin.getParameter()) {
641      if (pp.getName().equals("profile")) {
642        throw new Error("Can only specify profile in the context");
643      }
644    }
645    if (expProfile == null) {
646      throw new Exception("No ExpansionProfile provided");
647    }
648    pin.addParameter().setName("profile").setResource(expProfile);
649
650    Parameters pout = txServer.operateType(ValueSet.class, "validate-code", pin);
651    boolean ok = false;
652    String message = "No Message returned";
653    String display = null;
654    TerminologyServiceErrorClass err = TerminologyServiceErrorClass.UNKNOWN;
655    for (ParametersParameterComponent p : pout.getParameter()) {
656      if (p.getName().equals("result")) {
657        ok = ((BooleanType) p.getValue()).getValue().booleanValue();
658      } else if (p.getName().equals("message")) {
659        message = ((StringType) p.getValue()).getValue();
660      } else if (p.getName().equals("display")) {
661        display = ((StringType) p.getValue()).getValue();
662      } else if (p.getName().equals("cause")) {
663        try {
664          IssueType it = IssueType.fromCode(((StringType) p.getValue()).getValue());
665          if (it == IssueType.UNKNOWN) {
666            err = TerminologyServiceErrorClass.UNKNOWN;
667          } else if (it == IssueType.NOTSUPPORTED) {
668            err = TerminologyServiceErrorClass.VALUESET_UNSUPPORTED;
669          }
670        } catch (FHIRException e) {
671        }
672      }
673    }
674    if (!ok) {
675      res = new ValidationResult(IssueSeverity.ERROR, message, err);
676    } else if (display != null) {
677      res = new ValidationResult(new ConceptDefinitionComponent().setDisplay(display));
678    } else {
679      res = new ValidationResult(new ConceptDefinitionComponent());
680    }
681    saveToCache(res, cacheName);
682    return res;
683  }
684
685
686  private void tlog(String msg) {
687    //    log(msg);
688  }
689
690  @SuppressWarnings("rawtypes")
691  private String describeValidationParameters(Parameters pin) {
692    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
693    for (ParametersParameterComponent p : pin.getParameter()) {
694      if (p.hasValue() && p.getValue() instanceof PrimitiveType) {
695        b.append(p.getName() + "=" + ((PrimitiveType) p.getValue()).asStringValue());
696      } else if (p.hasValue() && p.getValue() instanceof Coding) {
697        b.append("system=" + ((Coding) p.getValue()).getSystem());
698        b.append("code=" + ((Coding) p.getValue()).getCode());
699        b.append("display=" + ((Coding) p.getValue()).getDisplay());
700      } else if (p.hasValue() && p.getValue() instanceof CodeableConcept) {
701        if (((CodeableConcept) p.getValue()).hasCoding()) {
702          Coding c = ((CodeableConcept) p.getValue()).getCodingFirstRep();
703          b.append("system=" + c.getSystem());
704          b.append("code=" + c.getCode());
705          b.append("display=" + c.getDisplay());
706        } else if (((CodeableConcept) p.getValue()).hasText()) {
707          b.append("text=" + ((CodeableConcept) p.getValue()).getText());
708        }
709      } else if (p.hasResource() && (p.getResource() instanceof ValueSet)) {
710        b.append("valueset=" + getVSSummary((ValueSet) p.getResource()));
711      }
712    }
713    return b.toString();
714  }
715
716  private ValidationResult loadFromCache(String fn) throws FileNotFoundException, IOException {
717    if (fn == null) {
718      return null;
719    }
720    if (!(ManagedFileAccess.file(fn).exists())) {
721      return null;
722    }
723    String cnt = FileUtilities.fileToString(fn);
724    if (cnt.startsWith("!error: ")) {
725      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(8));
726    } else if (cnt.startsWith("!warning: ")) {
727      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(10));
728    } else {
729      return new ValidationResult(new ConceptDefinitionComponent().setDisplay(cnt));
730    }
731  }
732
733  private void saveToCache(ValidationResult res, String cacheName) throws IOException {
734    if (cacheName == null) {
735      return;
736    }
737    if (res.getDisplay() != null) {
738      FileUtilities.stringToFile(res.getDisplay(), cacheName);
739    } else if (res.getMessage() != null) {
740      if (res.getSeverity() == IssueSeverity.WARNING) {
741        FileUtilities.stringToFile("!warning: " + res.getMessage(), cacheName);
742      } else {
743        FileUtilities.stringToFile("!error: " + res.getMessage(), cacheName);
744      }
745    }
746  }
747
748  private String generateCacheName(Parameters pin) throws IOException {
749    if (cache == null) {
750      return null;
751    }
752    String json = new JsonParser().composeString(pin);
753    return Utilities.path(cache, "vc" + Integer.toString(json.hashCode()) + ".json");
754  }
755
756  @Override
757  public ValueSetExpansionComponent expandVS(ConceptSetComponent inc, boolean heirachical)
758    throws TerminologyServiceException {
759    ValueSet vs = new ValueSet();
760    vs.setCompose(new ValueSetComposeComponent());
761    vs.getCompose().getInclude().add(inc);
762    ValueSetExpansionOutcome vse = expandVS(vs, true, heirachical);
763    ValueSet valueset = vse.getValueset();
764    if (valueset == null) {
765      throw new TerminologyServiceException("Error Expanding ValueSet: " + vse.getError());
766    }
767    return valueset.getExpansion();
768  }
769
770  @Override
771  public ValidationResult validateCode(String system, String code, String display) {
772    try {
773      if (codeSystems.containsKey(system) && codeSystems.get(system) != null) {
774        return verifyCodeInCodeSystem(codeSystems.get(system), system, code, display);
775      } else {
776        return verifyCodeExternal(null,
777          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
778      }
779    } catch (Exception e) {
780      return new ValidationResult(IssueSeverity.FATAL,
781        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
782    }
783  }
784
785
786  @Override
787  public ValidationResult validateCode(Coding code, ValueSet vs) {
788    if (codeSystems.containsKey(code.getSystem()) && codeSystems.get(code.getSystem()) != null) {
789      try {
790        return verifyCodeInCodeSystem(codeSystems.get(code.getSystem()), code.getSystem(),
791          code.getCode(), code.getDisplay());
792      } catch (Exception e) {
793        return new ValidationResult(IssueSeverity.FATAL,
794          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
795            .getMessage());
796      }
797    } else if (vs.hasExpansion()) {
798      try {
799        return verifyCodeInternal(vs, code.getSystem(), code.getCode(), code.getDisplay());
800      } catch (Exception e) {
801        return new ValidationResult(IssueSeverity.FATAL,
802          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
803            .getMessage());
804      }
805    } else {
806      try {
807        return verifyCodeExternal(vs, code, true);
808      } catch (Exception e) {
809        return new ValidationResult(IssueSeverity.WARNING,
810          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
811            .getMessage());
812      }
813    }
814  }
815
816  @Override
817  public ValidationResult validateCode(CodeableConcept code, ValueSet vs) {
818    try {
819      if (vs.hasExpansion()) {
820        return verifyCodeInternal(vs, code);
821      } else {
822        // we'll try expanding first; if that doesn't work, then we'll just pass it to the server to validate 
823        // ... could be a problem if the server doesn't have the code systems we have locally, so we try not to depend on the server
824        try {
825          ValueSetExpansionOutcome vse = expandVS(vs, true, false);
826          if (vse.getValueset() != null && !hasTooCostlyExpansion(vse.getValueset())) {
827            return verifyCodeInternal(vse.getValueset(), code);
828          }
829        } catch (Exception e) {
830          // failed? we'll just try the server
831        }
832        return verifyCodeExternal(vs, code, true);
833      }
834    } catch (Exception e) {
835      return new ValidationResult(IssueSeverity.FATAL,
836        "Error validating code \"" + code.toString() + "\": " + e.getMessage(),
837        TerminologyServiceErrorClass.SERVER_ERROR);
838    }
839  }
840
841
842  private boolean hasTooCostlyExpansion(ValueSet valueset) {
843    return valueset != null && valueset.hasExpansion() && ToolingExtensions
844      .hasExtension(valueset.getExpansion(),
845        "http://hl7.org/fhir/StructureDefinition/valueset-toocostly");
846  }
847
848  @Override
849  public ValidationResult validateCode(String system, String code, String display, ValueSet vs) {
850    try {
851      if (system == null && display == null) {
852        return verifyCodeInternal(vs, code);
853      }
854      if ((codeSystems.containsKey(system) && codeSystems.get(system) != null) || vs
855        .hasExpansion()) {
856        return verifyCodeInternal(vs, system, code, display);
857      } else {
858        return verifyCodeExternal(vs,
859          new Coding().setSystem(system).setCode(code).setDisplay(display), true);
860      }
861    } catch (Exception e) {
862      return new ValidationResult(IssueSeverity.FATAL,
863        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage(),
864        TerminologyServiceErrorClass.SERVER_ERROR);
865    }
866  }
867
868  @Override
869  public ValidationResult validateCode(String system, String code, String display,
870    ConceptSetComponent vsi) {
871    try {
872      ValueSet vs = new ValueSet();
873      vs.setUrl(UUIDUtilities.makeUuidUrn());
874      vs.getCompose().addInclude(vsi);
875      return verifyCodeExternal(vs,
876        new Coding().setSystem(system).setCode(code).setDisplay(display), true);
877    } catch (Exception e) {
878      return new ValidationResult(IssueSeverity.FATAL,
879        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
880    }
881  }
882
883  public void initTS(String cachePath, String tsServer) throws Exception {
884    cache = cachePath;
885    this.tsServer = tsServer;
886    expansionCache = new ValueSetExpansionCache(this, null);
887    validationCachePath = Utilities.path(cachePath, "validation.cache");
888    try {
889      loadValidationCache();
890    } catch (Exception e) {
891      e.printStackTrace();
892    }
893  }
894
895  protected void loadValidationCache() throws JsonSyntaxException, Exception {
896  }
897
898  @Override
899  public List<ConceptMap> findMapsForSource(String url) {
900    List<ConceptMap> res = new ArrayList<ConceptMap>();
901    for (ConceptMap map : maps.values()) {
902      if (((Reference) map.getSource()).getReference().equals(url)) {
903        res.add(map);
904      }
905    }
906    return res;
907  }
908
909  private ValidationResult verifyCodeInternal(ValueSet vs, CodeableConcept code) throws Exception {
910    for (Coding c : code.getCoding()) {
911      ValidationResult res = verifyCodeInternal(vs, c.getSystem(), c.getCode(), c.getDisplay());
912      if (res.isOk()) {
913        return res;
914      }
915    }
916    if (code.getCoding().isEmpty()) {
917      return new ValidationResult(IssueSeverity.ERROR, "None code provided");
918    } else {
919      return new ValidationResult(IssueSeverity.ERROR,
920        "None of the codes are in the specified value set");
921    }
922  }
923
924  private ValidationResult verifyCodeInternal(ValueSet vs, String system, String code,
925    String display) throws Exception {
926    if (vs.hasExpansion()) {
927      return verifyCodeInExpansion(vs, system, code, display);
928    } else {
929      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
930      if (vse.getValueset() != null) {
931        return verifyCodeExternal(vs,
932          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
933      } else {
934        return verifyCodeInExpansion(vse.getValueset(), system, code, display);
935      }
936    }
937  }
938
939  private ValidationResult verifyCodeInternal(ValueSet vs, String code)
940    throws FileNotFoundException, ETooCostly, IOException, FHIRException {
941    if (vs.hasExpansion()) {
942      return verifyCodeInExpansion(vs, code);
943    } else {
944      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
945      if (vse.getValueset() == null) {
946        return new ValidationResult(IssueSeverity.ERROR, vse.getError(), vse.getErrorClass());
947      } else {
948        return verifyCodeInExpansion(vse.getValueset(), code);
949      }
950    }
951  }
952
953  private ValidationResult verifyCodeInCodeSystem(CodeSystem cs, String system, String code,
954    String display) throws Exception {
955    ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), code);
956    if (cc == null) {
957      if (cs.getContent().equals(CodeSystem.CodeSystemContentMode.COMPLETE)) {
958        return new ValidationResult(IssueSeverity.ERROR,
959          "Unknown Code " + code + " in " + cs.getUrl());
960      } else if (!cs.getContent().equals(CodeSystem.CodeSystemContentMode.NOTPRESENT)) {
961        return new ValidationResult(IssueSeverity.WARNING,
962          "Unknown Code " + code + " in partial code list of " + cs.getUrl());
963      } else {
964        return verifyCodeExternal(null,
965          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
966      }
967    }
968    //
969    //        return new ValidationResult(IssueSeverity.WARNING, "A definition was found for "+cs.getUrl()+", but it has no codes in the definition");
970    //      return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+cs.getUrl());
971    if (display == null) {
972      return new ValidationResult(cc);
973    }
974    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
975    if (cc.hasDisplay()) {
976      b.append(cc.getDisplay());
977      if (display.equalsIgnoreCase(cc.getDisplay())) {
978        return new ValidationResult(cc);
979      }
980    }
981    for (ConceptDefinitionDesignationComponent ds : cc.getDesignation()) {
982      b.append(ds.getValue());
983      if (display.equalsIgnoreCase(ds.getValue())) {
984        return new ValidationResult(cc);
985      }
986    }
987    return new ValidationResult(IssueSeverity.WARNING,
988      "Display Name for " + code + " must be one of '" + b.toString() + "'", cc);
989  }
990
991
992  private ValidationResult verifyCodeInExpansion(ValueSet vs, String system, String code,
993    String display) {
994    ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
995    if (cc == null) {
996      return new ValidationResult(IssueSeverity.ERROR,
997        "Unknown Code " + code + " in " + vs.getUrl());
998    }
999    if (display == null) {
1000      return new ValidationResult(
1001        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1002    }
1003    if (cc.hasDisplay()) {
1004      if (display.equalsIgnoreCase(cc.getDisplay())) {
1005        return new ValidationResult(
1006          new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1007      }
1008      return new ValidationResult(IssueSeverity.WARNING,
1009        "Display Name for " + code + " must be '" + cc.getDisplay() + "'",
1010        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1011    }
1012    return null;
1013  }
1014
1015  private ValidationResult verifyCodeInExpansion(ValueSet vs, String code) throws FHIRException {
1016    if (vs.getExpansion()
1017      .hasExtension("http://hl7.org/fhir/StructureDefinition/valueset-toocostly")) {
1018      throw new FHIRException("Unable to validate core - value set is too costly to expand");
1019    } else {
1020      ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
1021      if (cc == null) {
1022        return new ValidationResult(IssueSeverity.ERROR,
1023          "Unknown Code " + code + " in " + vs.getUrl());
1024      }
1025      return null;
1026    }
1027  }
1028
1029  private ValueSetExpansionContainsComponent findCode(
1030    List<ValueSetExpansionContainsComponent> contains, String code) {
1031    for (ValueSetExpansionContainsComponent cc : contains) {
1032      if (code.equals(cc.getCode())) {
1033        return cc;
1034      }
1035      ValueSetExpansionContainsComponent c = findCode(cc.getContains(), code);
1036      if (c != null) {
1037        return c;
1038      }
1039    }
1040    return null;
1041  }
1042
1043  private ConceptDefinitionComponent findCodeInConcept(List<ConceptDefinitionComponent> concept,
1044    String code) {
1045    for (ConceptDefinitionComponent cc : concept) {
1046      if (code.equals(cc.getCode())) {
1047        return cc;
1048      }
1049      ConceptDefinitionComponent c = findCodeInConcept(cc.getConcept(), code);
1050      if (c != null) {
1051        return c;
1052      }
1053    }
1054    return null;
1055  }
1056
1057  public Set<String> getNonSupportedCodeSystems() {
1058    return nonSupportedCodeSystems;
1059  }
1060
1061  public boolean isCanRunWithoutTerminology() {
1062    return canRunWithoutTerminology;
1063  }
1064
1065  public void setCanRunWithoutTerminology(boolean canRunWithoutTerminology) {
1066    this.canRunWithoutTerminology = canRunWithoutTerminology;
1067  }
1068
1069  public int getExpandCodesLimit() {
1070    return expandCodesLimit;
1071  }
1072
1073  public void setExpandCodesLimit(int expandCodesLimit) {
1074    this.expandCodesLimit = expandCodesLimit;
1075  }
1076
1077  public void setLogger(ILoggingService logger) {
1078    this.logger = logger;
1079  }
1080
1081  public ExpansionProfile getExpansionProfile() {
1082    return expProfile;
1083  }
1084
1085  public void setExpansionProfile(ExpansionProfile expProfile) {
1086    this.expProfile = expProfile;
1087  }
1088
1089  @Override
1090  public boolean isNoTerminologyServer() {
1091    return noTerminologyServer;
1092  }
1093
1094  public String getName() {
1095    return name;
1096  }
1097
1098  public void setName(String name) {
1099    this.name = name;
1100  }
1101
1102  @Override
1103  public Set<String> getResourceNamesAsSet() {
1104    Set<String> res = new HashSet<String>();
1105    res.addAll(getResourceNames());
1106    return res;
1107  }
1108
1109  public void reportStatus(JsonObject json) {
1110    json.addProperty("codeystem-count", codeSystems.size());
1111    json.addProperty("valueset-count", valueSets.size());
1112    json.addProperty("conceptmap-count", maps.size());
1113    json.addProperty("transforms-count", transforms.size());
1114    json.addProperty("structures-count", profiles.size());
1115  }
1116
1117  public void cacheResource(Resource r) throws Exception {
1118    if (r instanceof ValueSet) {
1119      seeValueSet(((ValueSet) r).getUrl(), (ValueSet) r);
1120    } else if (r instanceof CodeSystem) {
1121      seeCodeSystem(((CodeSystem) r).getUrl(), (CodeSystem) r);
1122    } else if (r instanceof StructureDefinition) {
1123      StructureDefinition sd = (StructureDefinition) r;
1124      if ("http://hl7.org/fhir/StructureDefinition/Extension".equals(sd.getBaseDefinition())) {
1125        seeExtensionDefinition(sd.getUrl(), sd);
1126      } else if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT) {
1127        seeProfile(sd.getUrl(), sd);
1128      }
1129    }
1130  }
1131
1132  public void dropResource(String type, String id) throws FHIRException {
1133    if (type.equals("ValueSet")) {
1134      dropValueSet(id);
1135    }
1136    if (type.equals("CodeSystem")) {
1137      dropCodeSystem(id);
1138    }
1139    if (type.equals("StructureDefinition")) {
1140      dropProfile(id);
1141      dropExtensionDefinition(id);
1142    }
1143  }
1144
1145  public boolean isAllowLoadingDuplicates() {
1146    return allowLoadingDuplicates;
1147  }
1148
1149  public void setAllowLoadingDuplicates(boolean allowLoadingDuplicates) {
1150    this.allowLoadingDuplicates = allowLoadingDuplicates;
1151  }
1152
1153  @Override
1154  public StructureDefinition fetchTypeDefinition(String typeName) {
1155    return fetchResource(StructureDefinition.class,
1156      "http://hl7.org/fhir/StructureDefinition/" + typeName);
1157  }
1158}