Source code

001package org.hl7.fhir.dstu3.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.ByteArrayOutputStream;
035import java.io.File;
036import java.io.FileInputStream;
037import java.io.FileNotFoundException;
038import java.io.FileOutputStream;
039import java.io.IOException;
040import java.text.MessageFormat;
041import java.util.ArrayList;
042import java.util.HashMap;
043import java.util.HashSet;
044import java.util.List;
045import java.util.Locale;
046import java.util.Map;
047import java.util.Objects;
048import java.util.ResourceBundle;
049import java.util.Set;
050
051import ca.uhn.fhir.rest.api.Constants;
052import org.hl7.fhir.dstu3.formats.IParser.OutputStyle;
053import org.hl7.fhir.dstu3.formats.JsonParser;
054import org.hl7.fhir.dstu3.model.BooleanType;
055import org.hl7.fhir.dstu3.model.Bundle;
056import org.hl7.fhir.dstu3.model.Bundle.BundleEntryComponent;
057import org.hl7.fhir.dstu3.model.CodeSystem;
058import org.hl7.fhir.dstu3.model.CodeSystem.CodeSystemHierarchyMeaning;
059import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionComponent;
060import org.hl7.fhir.dstu3.model.CodeSystem.ConceptDefinitionDesignationComponent;
061import org.hl7.fhir.dstu3.model.CodeableConcept;
062import org.hl7.fhir.dstu3.model.Coding;
063import org.hl7.fhir.dstu3.model.ConceptMap;
064import org.hl7.fhir.dstu3.model.DataElement;
065import org.hl7.fhir.dstu3.model.ExpansionProfile;
066import org.hl7.fhir.dstu3.model.OperationDefinition;
067import org.hl7.fhir.dstu3.model.OperationOutcome;
068import org.hl7.fhir.dstu3.model.Parameters;
069import org.hl7.fhir.dstu3.model.Parameters.ParametersParameterComponent;
070import org.hl7.fhir.dstu3.model.PrimitiveType;
071import org.hl7.fhir.dstu3.model.Questionnaire;
072import org.hl7.fhir.dstu3.model.Reference;
073import org.hl7.fhir.dstu3.model.Resource;
074import org.hl7.fhir.dstu3.model.SearchParameter;
075import org.hl7.fhir.dstu3.model.StringType;
076import org.hl7.fhir.dstu3.model.StructureDefinition;
077import org.hl7.fhir.dstu3.model.StructureDefinition.TypeDerivationRule;
078import org.hl7.fhir.dstu3.model.StructureMap;
079import org.hl7.fhir.dstu3.model.UriType;
080import org.hl7.fhir.dstu3.model.ValueSet;
081import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetComponent;
082import org.hl7.fhir.dstu3.model.ValueSet.ConceptSetFilterComponent;
083import org.hl7.fhir.dstu3.model.ValueSet.ValueSetComposeComponent;
084import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionComponent;
085import org.hl7.fhir.dstu3.model.ValueSet.ValueSetExpansionContainsComponent;
086import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ETooCostly;
087import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
088import org.hl7.fhir.dstu3.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
089import org.hl7.fhir.dstu3.terminologies.ValueSetExpanderFactory;
090import org.hl7.fhir.dstu3.terminologies.ValueSetExpansionCache;
091import org.hl7.fhir.dstu3.utils.ToolingExtensions;
092import org.hl7.fhir.dstu3.utils.client.FHIRToolingClient;
093import org.hl7.fhir.exceptions.FHIRException;
094import org.hl7.fhir.exceptions.NoTerminologyServiceException;
095import org.hl7.fhir.exceptions.TerminologyServiceException;
096import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
097import org.hl7.fhir.utilities.TextFile;
098import org.hl7.fhir.utilities.Utilities;
099import org.hl7.fhir.utilities.i18n.I18nBase;
100import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
101import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
102
103import com.google.gson.JsonObject;
104import com.google.gson.JsonSyntaxException;
105
106public abstract class BaseWorkerContext extends I18nBase implements IWorkerContext {
107
108  // all maps are to the full URI
109  protected Map<String, CodeSystem> codeSystems = new HashMap<String, CodeSystem>();
110  protected Set<String> nonSupportedCodeSystems = new HashSet<String>();
111  protected Map<String, ValueSet> valueSets = new HashMap<String, ValueSet>();
112  protected Map<String, ConceptMap> maps = new HashMap<String, ConceptMap>();
113  protected Map<String, StructureMap> transforms = new HashMap<String, StructureMap>();
114  protected Map<String, DataElement> dataElements = new HashMap<String, DataElement>();
115  protected Map<String, StructureDefinition> profiles = new HashMap<String, StructureDefinition>();
116  protected Map<String, SearchParameter> searchParameters = new HashMap<String, SearchParameter>();
117  protected Map<String, StructureDefinition> extensionDefinitions = new HashMap<String, StructureDefinition>();
118  protected Map<String, Questionnaire> questionnaires = new HashMap<String, Questionnaire>();
119  protected Map<String, OperationDefinition> operations = new HashMap<String, OperationDefinition>();
120
121  protected ValueSetExpanderFactory expansionCache = new ValueSetExpansionCache(this);
122  protected boolean cacheValidation; // if true, do an expansion and cache the expansion
123  private Set<String> failed = new HashSet<String>(); // value sets for which we don't try to do expansion, since the first attempt to get a comprehensive expansion was not successful
124  protected Map<String, Map<String, ValidationResult>> validationCache = new HashMap<String, Map<String, ValidationResult>>();
125  protected String tsServer;
126  protected String validationCachePath;
127  protected String name;
128
129  // private ValueSetExpansionCache expansionCache; //   
130
131  protected FHIRToolingClient txServer;
132  private Bundle bndCodeSystems;
133  private boolean canRunWithoutTerminology;
134  protected boolean allowLoadingDuplicates;
135  protected boolean noTerminologyServer;
136  protected String cache;
137  private int expandCodesLimit = 1000;
138  protected ILoggingService logger;
139  protected ExpansionProfile expProfile;
140  private Locale locale;
141  private ResourceBundle i18Nmessages;
142
143  public Map<String, CodeSystem> getCodeSystems() {
144    return codeSystems;
145  }
146
147  public Map<String, DataElement> getDataElements() {
148    return dataElements;
149  }
150
151  public Map<String, ValueSet> getValueSets() {
152    return valueSets;
153  }
154
155  public Map<String, ConceptMap> getMaps() {
156    return maps;
157  }
158
159  public Map<String, StructureDefinition> getProfiles() {
160    return profiles;
161  }
162
163  public Map<String, StructureDefinition> getExtensionDefinitions() {
164    return extensionDefinitions;
165  }
166
167  public Map<String, Questionnaire> getQuestionnaires() {
168    return questionnaires;
169  }
170
171  public Map<String, OperationDefinition> getOperations() {
172    return operations;
173  }
174
175  public void seeExtensionDefinition(String url, StructureDefinition ed) throws Exception {
176    if (extensionDefinitions.get(ed.getUrl()) != null) {
177      throw new Exception("duplicate extension definition: " + ed.getUrl());
178    }
179    extensionDefinitions.put(ed.getId(), ed);
180    extensionDefinitions.put(url, ed);
181    extensionDefinitions.put(ed.getUrl(), ed);
182  }
183
184  public void dropExtensionDefinition(String id) {
185    StructureDefinition sd = extensionDefinitions.get(id);
186    extensionDefinitions.remove(id);
187    if (sd != null) {
188      extensionDefinitions.remove(sd.getUrl());
189    }
190  }
191
192  public void seeQuestionnaire(String url, Questionnaire theQuestionnaire) throws Exception {
193    if (questionnaires.get(theQuestionnaire.getId()) != null) {
194      throw new Exception("duplicate extension definition: " + theQuestionnaire.getId());
195    }
196    questionnaires.put(theQuestionnaire.getId(), theQuestionnaire);
197    questionnaires.put(url, theQuestionnaire);
198  }
199
200  public void seeOperation(OperationDefinition opd) throws Exception {
201    if (operations.get(opd.getUrl()) != null) {
202      throw new Exception("duplicate extension definition: " + opd.getUrl());
203    }
204    operations.put(opd.getUrl(), opd);
205    operations.put(opd.getId(), opd);
206  }
207
208  public void seeValueSet(String url, ValueSet vs) throws Exception {
209    if (valueSets.containsKey(vs.getUrl()) && !allowLoadingDuplicates) {
210      throw new Exception("Duplicate value set " + vs.getUrl());
211    }
212    valueSets.put(vs.getId(), vs);
213    valueSets.put(url, vs);
214    valueSets.put(vs.getUrl(), vs);
215  }
216
217  public void dropValueSet(String id) {
218    ValueSet vs = valueSets.get(id);
219    valueSets.remove(id);
220    if (vs != null) {
221      valueSets.remove(vs.getUrl());
222    }
223  }
224
225  public void seeCodeSystem(String url, CodeSystem cs) throws FHIRException {
226    if (codeSystems.containsKey(cs.getUrl()) && !allowLoadingDuplicates) {
227      throw new FHIRException("Duplicate code system " + cs.getUrl());
228    }
229    codeSystems.put(cs.getId(), cs);
230    codeSystems.put(url, cs);
231    codeSystems.put(cs.getUrl(), cs);
232  }
233
234  public void dropCodeSystem(String id) {
235    CodeSystem cs = codeSystems.get(id);
236    codeSystems.remove(id);
237    if (cs != null) {
238      codeSystems.remove(cs.getUrl());
239    }
240  }
241
242  public void seeProfile(String url, StructureDefinition p) throws Exception {
243    if (profiles.containsKey(p.getUrl())) {
244      throw new Exception("Duplicate Profile " + p.getUrl());
245    }
246    profiles.put(p.getId(), p);
247    profiles.put(url, p);
248    profiles.put(p.getUrl(), p);
249  }
250
251  public void dropProfile(String id) {
252    StructureDefinition sd = profiles.get(id);
253    profiles.remove(id);
254    if (sd != null) {
255      profiles.remove(sd.getUrl());
256    }
257  }
258
259  @Override
260  public CodeSystem fetchCodeSystem(String system) {
261    return codeSystems.get(system);
262  }
263
264  @Override
265  public boolean supportsSystem(String system) throws TerminologyServiceException {
266    if (codeSystems.containsKey(system)) {
267      return true;
268    } else if (nonSupportedCodeSystems.contains(system)) {
269      return false;
270    } else if (system.startsWith("http://example.org") || system.startsWith("http://acme.com")
271      || system.startsWith("http://hl7.org/fhir/valueset-") || system.startsWith("urn:oid:")) {
272      return false;
273    } else {
274      if (noTerminologyServer) {
275        return false;
276      }
277      if (bndCodeSystems == null) {
278        try {
279          tlog("Terminology server: Check for supported code systems for " + system);
280          bndCodeSystems = txServer.fetchFeed(txServer.getAddress()
281            + "/CodeSystem?content-mode=not-present&_summary=true&_count=1000");
282        } catch (Exception e) {
283          if (canRunWithoutTerminology) {
284            noTerminologyServer = true;
285            log("==============!! Running without terminology server !!============== (" + e
286              .getMessage() + ")");
287            return false;
288          } else {
289            throw new TerminologyServiceException(e);
290          }
291        }
292      }
293      if (bndCodeSystems != null) {
294        for (BundleEntryComponent be : bndCodeSystems.getEntry()) {
295          CodeSystem cs = (CodeSystem) be.getResource();
296          if (!codeSystems.containsKey(cs.getUrl())) {
297            codeSystems.put(cs.getUrl(), null);
298          }
299        }
300      }
301      if (codeSystems.containsKey(system)) {
302        return true;
303      }
304    }
305    nonSupportedCodeSystems.add(system);
306    return false;
307  }
308
309  private void log(String message) {
310    if (logger != null) {
311      logger.logMessage(message);
312    } else {
313      System.out.println(message);
314    }
315  }
316
317  @Override
318  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical) {
319    try {
320      if (vs.hasExpansion()) {
321        return new ValueSetExpansionOutcome(vs.copy());
322      }
323      String cacheFn = null;
324      if (cache != null) {
325        cacheFn = Utilities.path(cache, determineCacheId(vs, heirarchical) + ".json");
326        if (new File(cacheFn).exists()) {
327          return loadFromCache(vs.copy(), cacheFn);
328        }
329      }
330      if (cacheOk && vs.hasUrl()) {
331        if (expProfile == null) {
332          throw new Exception("No ExpansionProfile provided");
333        }
334        ValueSetExpansionOutcome vse = expansionCache.getExpander()
335          .expand(vs, expProfile.setExcludeNested(!heirarchical));
336        if (vse.getValueset() != null) {
337          if (cache != null) {
338            FileOutputStream s = new FileOutputStream(cacheFn);
339            newJsonParser().compose(new FileOutputStream(cacheFn), vse.getValueset());
340            s.close();
341          }
342        }
343        return vse;
344      } else {
345        ValueSetExpansionOutcome res = expandOnServer(vs, cacheFn);
346        if (cacheFn != null) {
347          if (res.getValueset() != null) {
348            saveToCache(res.getValueset(), cacheFn);
349          } else {
350            OperationOutcome oo = new OperationOutcome();
351            oo.addIssue().getDetails().setText(res.getError());
352            saveToCache(oo, cacheFn);
353          }
354        }
355        return res;
356      }
357    } catch (NoTerminologyServiceException e) {
358      return new ValueSetExpansionOutcome(
359        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
360        TerminologyServiceErrorClass.NOSERVICE);
361    } catch (Exception e) {
362      return new ValueSetExpansionOutcome(
363        e.getMessage() == null ? e.getClass().getName() : e.getMessage(),
364        TerminologyServiceErrorClass.UNKNOWN);
365    }
366  }
367
368  private ValueSetExpansionOutcome loadFromCache(ValueSet vs, String cacheFn)
369    throws FileNotFoundException, Exception {
370    JsonParser parser = new JsonParser();
371    Resource r = parser.parse(new FileInputStream(cacheFn));
372    if (r instanceof OperationOutcome) {
373      return new ValueSetExpansionOutcome(
374        ((OperationOutcome) r).getIssue().get(0).getDetails().getText(),
375        TerminologyServiceErrorClass.UNKNOWN);
376    } else {
377      vs.setExpansion(((ValueSet) r)
378        .getExpansion()); // because what is cached might be from a different value set
379      return new ValueSetExpansionOutcome(vs);
380    }
381  }
382
383  private void saveToCache(Resource res, String cacheFn) throws FileNotFoundException, Exception {
384    JsonParser parser = new JsonParser();
385    parser.compose(new FileOutputStream(cacheFn), res);
386  }
387
388  private String determineCacheId(ValueSet vs, boolean heirarchical) throws Exception {
389    // just the content logical definition is hashed
390    ValueSet vsid = new ValueSet();
391    vsid.setCompose(vs.getCompose());
392    JsonParser parser = new JsonParser();
393    parser.setOutputStyle(OutputStyle.NORMAL);
394    ByteArrayOutputStream b = new ByteArrayOutputStream();
395    parser.compose(b, vsid);
396    b.close();
397    String s = new String(b.toByteArray(), Constants.CHARSET_UTF8);
398    // any code systems we can find, we add these too. 
399    for (ConceptSetComponent inc : vs.getCompose().getInclude()) {
400      CodeSystem cs = fetchCodeSystem(inc.getSystem());
401      if (cs != null) {
402        String css = cacheValue(cs);
403        s = s + css;
404      }
405    }
406    s = s + "-" + Boolean.toString(heirarchical);
407    String r = Integer.toString(s.hashCode());
408    //    TextFile.stringToFile(s, Utilities.path(cache, r+".id.json"));
409    return r;
410  }
411
412
413  private String cacheValue(CodeSystem cs) throws IOException {
414    CodeSystem csid = new CodeSystem();
415    csid.setId(cs.getId());
416    csid.setVersion(cs.getVersion());
417    csid.setContent(cs.getContent());
418    csid.setHierarchyMeaning(CodeSystemHierarchyMeaning.GROUPEDBY);
419    for (ConceptDefinitionComponent cc : cs.getConcept()) {
420      csid.getConcept().add(processCSConcept(cc));
421    }
422    JsonParser parser = new JsonParser();
423    parser.setOutputStyle(OutputStyle.NORMAL);
424    ByteArrayOutputStream b = new ByteArrayOutputStream();
425    parser.compose(b, csid);
426    b.close();
427    return new String(b.toByteArray(), Constants.CHARSET_UTF8);
428  }
429
430
431  private ConceptDefinitionComponent processCSConcept(ConceptDefinitionComponent cc) {
432    ConceptDefinitionComponent ccid = new ConceptDefinitionComponent();
433    ccid.setCode(cc.getCode());
434    ccid.setDisplay(cc.getDisplay());
435    for (ConceptDefinitionComponent cci : cc.getConcept()) {
436      ccid.getConcept().add(processCSConcept(cci));
437    }
438    return ccid;
439  }
440
441  public ValueSetExpansionOutcome expandOnServer(ValueSet vs, String fn) throws Exception {
442    if (noTerminologyServer) {
443      return new ValueSetExpansionOutcome(
444        "Error expanding ValueSet: running without terminology services",
445        TerminologyServiceErrorClass.NOSERVICE);
446    }
447    if (expProfile == null) {
448      throw new Exception("No ExpansionProfile provided");
449    }
450
451    try {
452      Map<String, String> params = new HashMap<String, String>();
453      params.put("_limit", Integer.toString(expandCodesLimit));
454      params.put("_incomplete", "true");
455      tlog("Terminology Server: $expand on " + getVSSummary(vs));
456      ValueSet result = txServer.expandValueset(vs, expProfile.setIncludeDefinition(false), params);
457      return new ValueSetExpansionOutcome(result);
458    } catch (Exception e) {
459      return new ValueSetExpansionOutcome(
460        "Error expanding ValueSet \"" + vs.getUrl() + ": " + e.getMessage(),
461        TerminologyServiceErrorClass.UNKNOWN);
462    }
463  }
464
465  private String getVSSummary(ValueSet vs) {
466    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
467    for (ConceptSetComponent cc : vs.getCompose().getInclude()) {
468      b.append("Include " + getIncSummary(cc));
469    }
470    for (ConceptSetComponent cc : vs.getCompose().getExclude()) {
471      b.append("Exclude " + getIncSummary(cc));
472    }
473    return b.toString();
474  }
475
476  private String getIncSummary(ConceptSetComponent cc) {
477    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
478    for (UriType vs : cc.getValueSet()) {
479      b.append(vs.asStringValue());
480    }
481    String vsd =
482      b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
483    String system = cc.getSystem();
484    if (cc.hasConcept()) {
485      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
486    }
487    if (cc.hasFilter()) {
488      String s = "";
489      for (ConceptSetFilterComponent f : cc.getFilter()) {
490        if (!Utilities.noString(s)) {
491          s = s + " & ";
492        }
493        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
494      }
495      return "from " + system + " where " + s + vsd;
496    }
497    return "All codes from " + system + vsd;
498  }
499
500  private ValidationResult handleByCache(ValueSet vs, Coding coding, boolean tryCache) {
501    String cacheId = cacheId(coding);
502    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
503    if (cache == null) {
504      cache = new HashMap<String, IWorkerContext.ValidationResult>();
505      validationCache.put(vs.getUrl(), cache);
506    }
507    if (cache.containsKey(cacheId)) {
508      return cache.get(cacheId);
509    }
510    if (!tryCache) {
511      return null;
512    }
513    if (!cacheValidation) {
514      return null;
515    }
516    if (failed.contains(vs.getUrl())) {
517      return null;
518    }
519    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
520    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
521      failed.add(vs.getUrl());
522      return null;
523    }
524
525    ValidationResult res = validateCode(coding, vse.getValueset());
526    cache.put(cacheId, res);
527    return res;
528  }
529
530  private boolean notcomplete(ValueSet vs) {
531    if (!vs.hasExpansion()) {
532      return true;
533    }
534    if (!vs.getExpansion()
535      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-unclosed").isEmpty()) {
536      return true;
537    }
538    if (!vs.getExpansion()
539      .getExtensionsByUrl("http://hl7.org/fhir/StructureDefinition/valueset-toocostly").isEmpty()) {
540      return true;
541    }
542    return false;
543  }
544
545  private ValidationResult handleByCache(ValueSet vs, CodeableConcept concept, boolean tryCache) {
546    String cacheId = cacheId(concept);
547    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
548    if (cache == null) {
549      cache = new HashMap<String, IWorkerContext.ValidationResult>();
550      validationCache.put(vs.getUrl(), cache);
551    }
552    if (cache.containsKey(cacheId)) {
553      return cache.get(cacheId);
554    }
555
556    if (validationCache.containsKey(vs.getUrl()) && validationCache.get(vs.getUrl())
557      .containsKey(cacheId)) {
558      return validationCache.get(vs.getUrl()).get(cacheId);
559    }
560    if (!tryCache) {
561      return null;
562    }
563    if (!cacheValidation) {
564      return null;
565    }
566    if (failed.contains(vs.getUrl())) {
567      return null;
568    }
569    ValueSetExpansionOutcome vse = expandVS(vs, true, false);
570    if (vse.getValueset() == null || notcomplete(vse.getValueset())) {
571      failed.add(vs.getUrl());
572      return null;
573    }
574    ValidationResult res = validateCode(concept, vse.getValueset());
575    cache.put(cacheId, res);
576    return res;
577  }
578
579  private String cacheId(Coding coding) {
580    return "|" + coding.getSystem() + "|" + coding.getVersion() + "|" + coding.getCode() + "|"
581      + coding.getDisplay();
582  }
583
584  private String cacheId(CodeableConcept cc) {
585    StringBuilder b = new StringBuilder();
586    for (Coding c : cc.getCoding()) {
587      b.append("#");
588      b.append(cacheId(c));
589    }
590    return b.toString();
591  }
592
593  private ValidationResult verifyCodeExternal(ValueSet vs, Coding coding, boolean tryCache)
594    throws Exception {
595    ValidationResult res = vs == null ? null : handleByCache(vs, coding, tryCache);
596    if (res != null) {
597      return res;
598    }
599    Parameters pin = new Parameters();
600    pin.addParameter().setName("coding").setValue(coding);
601    if (vs != null) {
602      pin.addParameter().setName("valueSet").setResource(vs);
603    }
604    res = serverValidateCode(pin, vs == null);
605    if (vs != null) {
606      Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
607      cache.put(cacheId(coding), res);
608    }
609    return res;
610  }
611
612  private ValidationResult verifyCodeExternal(ValueSet vs, CodeableConcept cc, boolean tryCache)
613    throws Exception {
614    ValidationResult res = handleByCache(vs, cc, tryCache);
615    if (res != null) {
616      return res;
617    }
618    Parameters pin = new Parameters();
619    pin.addParameter().setName("codeableConcept").setValue(cc);
620    pin.addParameter().setName("valueSet").setResource(vs);
621    res = serverValidateCode(pin, false);
622    Map<String, ValidationResult> cache = validationCache.get(vs.getUrl());
623    cache.put(cacheId(cc), res);
624    return res;
625  }
626
627  private ValidationResult serverValidateCode(Parameters pin, boolean doCache) throws Exception {
628    if (noTerminologyServer) {
629      return new ValidationResult(null, null, TerminologyServiceErrorClass.NOSERVICE);
630    }
631    String cacheName = doCache ? generateCacheName(pin) : null;
632    ValidationResult res = loadFromCache(cacheName);
633    if (res != null) {
634      return res;
635    }
636    tlog("Terminology Server: $validate-code " + describeValidationParameters(pin));
637    for (ParametersParameterComponent pp : pin.getParameter()) {
638      if (pp.getName().equals("profile")) {
639        throw new Error("Can only specify profile in the context");
640      }
641    }
642    if (expProfile == null) {
643      throw new Exception("No ExpansionProfile provided");
644    }
645    pin.addParameter().setName("profile").setResource(expProfile);
646
647    Parameters pout = txServer.operateType(ValueSet.class, "validate-code", pin);
648    boolean ok = false;
649    String message = "No Message returned";
650    String display = null;
651    TerminologyServiceErrorClass err = TerminologyServiceErrorClass.UNKNOWN;
652    for (ParametersParameterComponent p : pout.getParameter()) {
653      if (p.getName().equals("result")) {
654        ok = ((BooleanType) p.getValue()).getValue().booleanValue();
655      } else if (p.getName().equals("message")) {
656        message = ((StringType) p.getValue()).getValue();
657      } else if (p.getName().equals("display")) {
658        display = ((StringType) p.getValue()).getValue();
659      } else if (p.getName().equals("cause")) {
660        try {
661          IssueType it = IssueType.fromCode(((StringType) p.getValue()).getValue());
662          if (it == IssueType.UNKNOWN) {
663            err = TerminologyServiceErrorClass.UNKNOWN;
664          } else if (it == IssueType.NOTSUPPORTED) {
665            err = TerminologyServiceErrorClass.VALUESET_UNSUPPORTED;
666          }
667        } catch (FHIRException e) {
668        }
669      }
670    }
671    if (!ok) {
672      res = new ValidationResult(IssueSeverity.ERROR, message, err);
673    } else if (display != null) {
674      res = new ValidationResult(new ConceptDefinitionComponent().setDisplay(display));
675    } else {
676      res = new ValidationResult(new ConceptDefinitionComponent());
677    }
678    saveToCache(res, cacheName);
679    return res;
680  }
681
682
683  private void tlog(String msg) {
684    //    log(msg);
685  }
686
687  @SuppressWarnings("rawtypes")
688  private String describeValidationParameters(Parameters pin) {
689    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
690    for (ParametersParameterComponent p : pin.getParameter()) {
691      if (p.hasValue() && p.getValue() instanceof PrimitiveType) {
692        b.append(p.getName() + "=" + ((PrimitiveType) p.getValue()).asStringValue());
693      } else if (p.hasValue() && p.getValue() instanceof Coding) {
694        b.append("system=" + ((Coding) p.getValue()).getSystem());
695        b.append("code=" + ((Coding) p.getValue()).getCode());
696        b.append("display=" + ((Coding) p.getValue()).getDisplay());
697      } else if (p.hasValue() && p.getValue() instanceof CodeableConcept) {
698        if (((CodeableConcept) p.getValue()).hasCoding()) {
699          Coding c = ((CodeableConcept) p.getValue()).getCodingFirstRep();
700          b.append("system=" + c.getSystem());
701          b.append("code=" + c.getCode());
702          b.append("display=" + c.getDisplay());
703        } else if (((CodeableConcept) p.getValue()).hasText()) {
704          b.append("text=" + ((CodeableConcept) p.getValue()).getText());
705        }
706      } else if (p.hasResource() && (p.getResource() instanceof ValueSet)) {
707        b.append("valueset=" + getVSSummary((ValueSet) p.getResource()));
708      }
709    }
710    return b.toString();
711  }
712
713  private ValidationResult loadFromCache(String fn) throws FileNotFoundException, IOException {
714    if (fn == null) {
715      return null;
716    }
717    if (!(new File(fn).exists())) {
718      return null;
719    }
720    String cnt = TextFile.fileToString(fn);
721    if (cnt.startsWith("!error: ")) {
722      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(8));
723    } else if (cnt.startsWith("!warning: ")) {
724      return new ValidationResult(IssueSeverity.ERROR, cnt.substring(10));
725    } else {
726      return new ValidationResult(new ConceptDefinitionComponent().setDisplay(cnt));
727    }
728  }
729
730  private void saveToCache(ValidationResult res, String cacheName) throws IOException {
731    if (cacheName == null) {
732      return;
733    }
734    if (res.getDisplay() != null) {
735      TextFile.stringToFile(res.getDisplay(), cacheName);
736    } else if (res.getMessage() != null) {
737      if (res.getSeverity() == IssueSeverity.WARNING) {
738        TextFile.stringToFile("!warning: " + res.getMessage(), cacheName);
739      } else {
740        TextFile.stringToFile("!error: " + res.getMessage(), cacheName);
741      }
742    }
743  }
744
745  private String generateCacheName(Parameters pin) throws IOException {
746    if (cache == null) {
747      return null;
748    }
749    String json = new JsonParser().composeString(pin);
750    return Utilities.path(cache, "vc" + Integer.toString(json.hashCode()) + ".json");
751  }
752
753  @Override
754  public ValueSetExpansionComponent expandVS(ConceptSetComponent inc, boolean heirachical)
755    throws TerminologyServiceException {
756    ValueSet vs = new ValueSet();
757    vs.setCompose(new ValueSetComposeComponent());
758    vs.getCompose().getInclude().add(inc);
759    ValueSetExpansionOutcome vse = expandVS(vs, true, heirachical);
760    ValueSet valueset = vse.getValueset();
761    if (valueset == null) {
762      throw new TerminologyServiceException("Error Expanding ValueSet: " + vse.getError());
763    }
764    return valueset.getExpansion();
765  }
766
767  @Override
768  public ValidationResult validateCode(String system, String code, String display) {
769    try {
770      if (codeSystems.containsKey(system) && codeSystems.get(system) != null) {
771        return verifyCodeInCodeSystem(codeSystems.get(system), system, code, display);
772      } else {
773        return verifyCodeExternal(null,
774          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
775      }
776    } catch (Exception e) {
777      return new ValidationResult(IssueSeverity.FATAL,
778        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
779    }
780  }
781
782
783  @Override
784  public ValidationResult validateCode(Coding code, ValueSet vs) {
785    if (codeSystems.containsKey(code.getSystem()) && codeSystems.get(code.getSystem()) != null) {
786      try {
787        return verifyCodeInCodeSystem(codeSystems.get(code.getSystem()), code.getSystem(),
788          code.getCode(), code.getDisplay());
789      } catch (Exception e) {
790        return new ValidationResult(IssueSeverity.FATAL,
791          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
792            .getMessage());
793      }
794    } else if (vs.hasExpansion()) {
795      try {
796        return verifyCodeInternal(vs, code.getSystem(), code.getCode(), code.getDisplay());
797      } catch (Exception e) {
798        return new ValidationResult(IssueSeverity.FATAL,
799          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
800            .getMessage());
801      }
802    } else {
803      try {
804        return verifyCodeExternal(vs, code, true);
805      } catch (Exception e) {
806        return new ValidationResult(IssueSeverity.WARNING,
807          "Error validating code \"" + code + "\" in system \"" + code.getSystem() + "\": " + e
808            .getMessage());
809      }
810    }
811  }
812
813  @Override
814  public ValidationResult validateCode(CodeableConcept code, ValueSet vs) {
815    try {
816      if (vs.hasExpansion()) {
817        return verifyCodeInternal(vs, code);
818      } else {
819        // we'll try expanding first; if that doesn't work, then we'll just pass it to the server to validate 
820        // ... could be a problem if the server doesn't have the code systems we have locally, so we try not to depend on the server
821        try {
822          ValueSetExpansionOutcome vse = expandVS(vs, true, false);
823          if (vse.getValueset() != null && !hasTooCostlyExpansion(vse.getValueset())) {
824            return verifyCodeInternal(vse.getValueset(), code);
825          }
826        } catch (Exception e) {
827          // failed? we'll just try the server
828        }
829        return verifyCodeExternal(vs, code, true);
830      }
831    } catch (Exception e) {
832      return new ValidationResult(IssueSeverity.FATAL,
833        "Error validating code \"" + code.toString() + "\": " + e.getMessage(),
834        TerminologyServiceErrorClass.SERVER_ERROR);
835    }
836  }
837
838
839  private boolean hasTooCostlyExpansion(ValueSet valueset) {
840    return valueset != null && valueset.hasExpansion() && ToolingExtensions
841      .hasExtension(valueset.getExpansion(),
842        "http://hl7.org/fhir/StructureDefinition/valueset-toocostly");
843  }
844
845  @Override
846  public ValidationResult validateCode(String system, String code, String display, ValueSet vs) {
847    try {
848      if (system == null && display == null) {
849        return verifyCodeInternal(vs, code);
850      }
851      if ((codeSystems.containsKey(system) && codeSystems.get(system) != null) || vs
852        .hasExpansion()) {
853        return verifyCodeInternal(vs, system, code, display);
854      } else {
855        return verifyCodeExternal(vs,
856          new Coding().setSystem(system).setCode(code).setDisplay(display), true);
857      }
858    } catch (Exception e) {
859      return new ValidationResult(IssueSeverity.FATAL,
860        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage(),
861        TerminologyServiceErrorClass.SERVER_ERROR);
862    }
863  }
864
865  @Override
866  public ValidationResult validateCode(String system, String code, String display,
867    ConceptSetComponent vsi) {
868    try {
869      ValueSet vs = new ValueSet();
870      vs.setUrl(Utilities.makeUuidUrn());
871      vs.getCompose().addInclude(vsi);
872      return verifyCodeExternal(vs,
873        new Coding().setSystem(system).setCode(code).setDisplay(display), true);
874    } catch (Exception e) {
875      return new ValidationResult(IssueSeverity.FATAL,
876        "Error validating code \"" + code + "\" in system \"" + system + "\": " + e.getMessage());
877    }
878  }
879
880  public void initTS(String cachePath, String tsServer) throws Exception {
881    cache = cachePath;
882    this.tsServer = tsServer;
883    expansionCache = new ValueSetExpansionCache(this, null);
884    validationCachePath = Utilities.path(cachePath, "validation.cache");
885    try {
886      loadValidationCache();
887    } catch (Exception e) {
888      e.printStackTrace();
889    }
890  }
891
892  protected void loadValidationCache() throws JsonSyntaxException, Exception {
893  }
894
895  @Override
896  public List<ConceptMap> findMapsForSource(String url) {
897    List<ConceptMap> res = new ArrayList<ConceptMap>();
898    for (ConceptMap map : maps.values()) {
899      if (((Reference) map.getSource()).getReference().equals(url)) {
900        res.add(map);
901      }
902    }
903    return res;
904  }
905
906  private ValidationResult verifyCodeInternal(ValueSet vs, CodeableConcept code) throws Exception {
907    for (Coding c : code.getCoding()) {
908      ValidationResult res = verifyCodeInternal(vs, c.getSystem(), c.getCode(), c.getDisplay());
909      if (res.isOk()) {
910        return res;
911      }
912    }
913    if (code.getCoding().isEmpty()) {
914      return new ValidationResult(IssueSeverity.ERROR, "None code provided");
915    } else {
916      return new ValidationResult(IssueSeverity.ERROR,
917        "None of the codes are in the specified value set");
918    }
919  }
920
921  private ValidationResult verifyCodeInternal(ValueSet vs, String system, String code,
922    String display) throws Exception {
923    if (vs.hasExpansion()) {
924      return verifyCodeInExpansion(vs, system, code, display);
925    } else {
926      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
927      if (vse.getValueset() != null) {
928        return verifyCodeExternal(vs,
929          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
930      } else {
931        return verifyCodeInExpansion(vse.getValueset(), system, code, display);
932      }
933    }
934  }
935
936  private ValidationResult verifyCodeInternal(ValueSet vs, String code)
937    throws FileNotFoundException, ETooCostly, IOException, FHIRException {
938    if (vs.hasExpansion()) {
939      return verifyCodeInExpansion(vs, code);
940    } else {
941      ValueSetExpansionOutcome vse = expansionCache.getExpander().expand(vs, null);
942      if (vse.getValueset() == null) {
943        return new ValidationResult(IssueSeverity.ERROR, vse.getError(), vse.getErrorClass());
944      } else {
945        return verifyCodeInExpansion(vse.getValueset(), code);
946      }
947    }
948  }
949
950  private ValidationResult verifyCodeInCodeSystem(CodeSystem cs, String system, String code,
951    String display) throws Exception {
952    ConceptDefinitionComponent cc = findCodeInConcept(cs.getConcept(), code);
953    if (cc == null) {
954      if (cs.getContent().equals(CodeSystem.CodeSystemContentMode.COMPLETE)) {
955        return new ValidationResult(IssueSeverity.ERROR,
956          "Unknown Code " + code + " in " + cs.getUrl());
957      } else if (!cs.getContent().equals(CodeSystem.CodeSystemContentMode.NOTPRESENT)) {
958        return new ValidationResult(IssueSeverity.WARNING,
959          "Unknown Code " + code + " in partial code list of " + cs.getUrl());
960      } else {
961        return verifyCodeExternal(null,
962          new Coding().setSystem(system).setCode(code).setDisplay(display), false);
963      }
964    }
965    //
966    //        return new ValidationResult(IssueSeverity.WARNING, "A definition was found for "+cs.getUrl()+", but it has no codes in the definition");
967    //      return new ValidationResult(IssueSeverity.ERROR, "Unknown Code "+code+" in "+cs.getUrl());
968    if (display == null) {
969      return new ValidationResult(cc);
970    }
971    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
972    if (cc.hasDisplay()) {
973      b.append(cc.getDisplay());
974      if (display.equalsIgnoreCase(cc.getDisplay())) {
975        return new ValidationResult(cc);
976      }
977    }
978    for (ConceptDefinitionDesignationComponent ds : cc.getDesignation()) {
979      b.append(ds.getValue());
980      if (display.equalsIgnoreCase(ds.getValue())) {
981        return new ValidationResult(cc);
982      }
983    }
984    return new ValidationResult(IssueSeverity.WARNING,
985      "Display Name for " + code + " must be one of '" + b.toString() + "'", cc);
986  }
987
988
989  private ValidationResult verifyCodeInExpansion(ValueSet vs, String system, String code,
990    String display) {
991    ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
992    if (cc == null) {
993      return new ValidationResult(IssueSeverity.ERROR,
994        "Unknown Code " + code + " in " + vs.getUrl());
995    }
996    if (display == null) {
997      return new ValidationResult(
998        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
999    }
1000    if (cc.hasDisplay()) {
1001      if (display.equalsIgnoreCase(cc.getDisplay())) {
1002        return new ValidationResult(
1003          new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1004      }
1005      return new ValidationResult(IssueSeverity.WARNING,
1006        "Display Name for " + code + " must be '" + cc.getDisplay() + "'",
1007        new ConceptDefinitionComponent().setCode(code).setDisplay(cc.getDisplay()));
1008    }
1009    return null;
1010  }
1011
1012  private ValidationResult verifyCodeInExpansion(ValueSet vs, String code) throws FHIRException {
1013    if (vs.getExpansion()
1014      .hasExtension("http://hl7.org/fhir/StructureDefinition/valueset-toocostly")) {
1015      throw new FHIRException("Unable to validate core - value set is too costly to expand");
1016    } else {
1017      ValueSetExpansionContainsComponent cc = findCode(vs.getExpansion().getContains(), code);
1018      if (cc == null) {
1019        return new ValidationResult(IssueSeverity.ERROR,
1020          "Unknown Code " + code + " in " + vs.getUrl());
1021      }
1022      return null;
1023    }
1024  }
1025
1026  private ValueSetExpansionContainsComponent findCode(
1027    List<ValueSetExpansionContainsComponent> contains, String code) {
1028    for (ValueSetExpansionContainsComponent cc : contains) {
1029      if (code.equals(cc.getCode())) {
1030        return cc;
1031      }
1032      ValueSetExpansionContainsComponent c = findCode(cc.getContains(), code);
1033      if (c != null) {
1034        return c;
1035      }
1036    }
1037    return null;
1038  }
1039
1040  private ConceptDefinitionComponent findCodeInConcept(List<ConceptDefinitionComponent> concept,
1041    String code) {
1042    for (ConceptDefinitionComponent cc : concept) {
1043      if (code.equals(cc.getCode())) {
1044        return cc;
1045      }
1046      ConceptDefinitionComponent c = findCodeInConcept(cc.getConcept(), code);
1047      if (c != null) {
1048        return c;
1049      }
1050    }
1051    return null;
1052  }
1053
1054  public Set<String> getNonSupportedCodeSystems() {
1055    return nonSupportedCodeSystems;
1056  }
1057
1058  public boolean isCanRunWithoutTerminology() {
1059    return canRunWithoutTerminology;
1060  }
1061
1062  public void setCanRunWithoutTerminology(boolean canRunWithoutTerminology) {
1063    this.canRunWithoutTerminology = canRunWithoutTerminology;
1064  }
1065
1066  public int getExpandCodesLimit() {
1067    return expandCodesLimit;
1068  }
1069
1070  public void setExpandCodesLimit(int expandCodesLimit) {
1071    this.expandCodesLimit = expandCodesLimit;
1072  }
1073
1074  public void setLogger(ILoggingService logger) {
1075    this.logger = logger;
1076  }
1077
1078  public ExpansionProfile getExpansionProfile() {
1079    return expProfile;
1080  }
1081
1082  public void setExpansionProfile(ExpansionProfile expProfile) {
1083    this.expProfile = expProfile;
1084  }
1085
1086  @Override
1087  public boolean isNoTerminologyServer() {
1088    return noTerminologyServer;
1089  }
1090
1091  public String getName() {
1092    return name;
1093  }
1094
1095  public void setName(String name) {
1096    this.name = name;
1097  }
1098
1099  @Override
1100  public Set<String> getResourceNamesAsSet() {
1101    Set<String> res = new HashSet<String>();
1102    res.addAll(getResourceNames());
1103    return res;
1104  }
1105
1106  public void reportStatus(JsonObject json) {
1107    json.addProperty("codeystem-count", codeSystems.size());
1108    json.addProperty("valueset-count", valueSets.size());
1109    json.addProperty("conceptmap-count", maps.size());
1110    json.addProperty("transforms-count", transforms.size());
1111    json.addProperty("structures-count", profiles.size());
1112  }
1113
1114  public void cacheResource(Resource r) throws Exception {
1115    if (r instanceof ValueSet) {
1116      seeValueSet(((ValueSet) r).getUrl(), (ValueSet) r);
1117    } else if (r instanceof CodeSystem) {
1118      seeCodeSystem(((CodeSystem) r).getUrl(), (CodeSystem) r);
1119    } else if (r instanceof StructureDefinition) {
1120      StructureDefinition sd = (StructureDefinition) r;
1121      if ("http://hl7.org/fhir/StructureDefinition/Extension".equals(sd.getBaseDefinition())) {
1122        seeExtensionDefinition(sd.getUrl(), sd);
1123      } else if (sd.getDerivation() == TypeDerivationRule.CONSTRAINT) {
1124        seeProfile(sd.getUrl(), sd);
1125      }
1126    }
1127  }
1128
1129  public void dropResource(String type, String id) throws FHIRException {
1130    if (type.equals("ValueSet")) {
1131      dropValueSet(id);
1132    }
1133    if (type.equals("CodeSystem")) {
1134      dropCodeSystem(id);
1135    }
1136    if (type.equals("StructureDefinition")) {
1137      dropProfile(id);
1138      dropExtensionDefinition(id);
1139    }
1140  }
1141
1142  public boolean isAllowLoadingDuplicates() {
1143    return allowLoadingDuplicates;
1144  }
1145
1146  public void setAllowLoadingDuplicates(boolean allowLoadingDuplicates) {
1147    this.allowLoadingDuplicates = allowLoadingDuplicates;
1148  }
1149
1150  @Override
1151  public StructureDefinition fetchTypeDefinition(String typeName) {
1152    return fetchResource(StructureDefinition.class,
1153      "http://hl7.org/fhir/StructureDefinition/" + typeName);
1154  }
1155}