Source code

001package org.hl7.fhir.r4.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032import java.io.File;
033import java.io.FileNotFoundException;
034import java.io.FileOutputStream;
035import java.io.IOException;
036import java.io.OutputStreamWriter;
037import java.util.ArrayList;
038import java.util.HashMap;
039import java.util.List;
040import java.util.Map;
041
042import org.apache.commons.lang3.StringUtils;
043import org.hl7.fhir.exceptions.FHIRException;
044import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
045import org.hl7.fhir.r4.formats.IParser.OutputStyle;
046import org.hl7.fhir.r4.formats.JsonParser;
047import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
048import org.hl7.fhir.r4.model.CodeableConcept;
049import org.hl7.fhir.r4.model.Coding;
050import org.hl7.fhir.r4.model.UriType;
051import org.hl7.fhir.r4.model.ValueSet;
052import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
053import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
054import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
055import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
056import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
057import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
058import org.hl7.fhir.utilities.TextFile;
059import org.hl7.fhir.utilities.Utilities;
060import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
061import org.hl7.fhir.utilities.validation.ValidationOptions;
062
063import com.google.gson.JsonElement;
064import com.google.gson.JsonNull;
065import com.google.gson.JsonObject;
066import com.google.gson.JsonPrimitive;
067
068/**
069 * This implements a two level cache. - a temporary cache for remmbering
070 * previous local operations - a persistent cache for rembering tx server
071 * operations
072 * 
073 * the cache is a series of pairs: a map, and a list. the map is the loaded
074 * cache, the list is the persiistent cache, carefully maintained in order for
075 * version control consistency
076 * 
077 * @author graha
078 *
079 */
080public class TerminologyCache {
081  public static final boolean TRANSIENT = false;
082  public static final boolean PERMANENT = true;
083  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
084  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
085  private static final String BREAK = "####";
086
087  private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator();
088
089  protected SystemNameKeyGenerator getSystemNameKeyGenerator() {
090    return systemNameKeyGenerator;
091  }
092
093  public class SystemNameKeyGenerator {
094    public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct";
095    public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm";
096    public static final String LOINC_CODESYSTEM_URL = "http://loinc.org";
097    public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org";
098
099    public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/";
100    public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/";
101    public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/";
102
103    public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:";
104    public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47";
105    public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13";
106
107    public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101";
108    public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM";
109
110    public String getNameForSystem(String system) {
111      final int lastPipe = system.lastIndexOf('|');
112      final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe);
113      final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1);
114
115      if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL))
116        return getVersionedSystem("snomed", systemVersion);
117      if (systemBaseName.equals(RXNORM_CODESYSTEM_URL))
118        return getVersionedSystem("rxnorm", systemVersion);
119      if (systemBaseName.equals(LOINC_CODESYSTEM_URL))
120        return getVersionedSystem("loinc", systemVersion);
121      if (systemBaseName.equals(UCUM_CODESYSTEM_URL))
122        return getVersionedSystem("ucum", systemVersion);
123      if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL))
124        return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
125      if (systemBaseName.equals(_11073_CODESYSTEM_URN))
126        return getVersionedSystem("11073", systemVersion);
127      if (systemBaseName.startsWith(ISO_CODESYSTEM_URN))
128        return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""),
129            systemVersion);
130      if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL))
131        return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
132      if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL))
133        return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
134      if (systemBaseName.equals(LANG_CODESYSTEM_URN))
135        return getVersionedSystem("lang", systemVersion);
136      if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN))
137        return getVersionedSystem("mimetypes", systemVersion);
138      if (systemBaseName.equals(DICOM_CODESYSTEM_URL))
139        return getVersionedSystem("dicom", systemVersion);
140      return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"),
141          systemVersion);
142    }
143
144    public String normalizeBaseURL(String baseUrl, String fullUrl) {
145      return fullUrl.substring(baseUrl.length()).replace("/", "");
146    }
147
148    public String getVersionedSystem(String baseSystem, String version) {
149      if (version != null) {
150        return baseSystem + "_" + version;
151      }
152      return baseSystem;
153    }
154  }
155
156  public class CacheToken {
157    private String name;
158    private String key;
159    private String request;
160
161    public void setName(String n) {
162      String systemName = getSystemNameKeyGenerator().getNameForSystem(n);
163      if (name == null)
164        name = systemName;
165      else if (!systemName.equals(name))
166        name = NAME_FOR_NO_SYSTEM;
167    }
168
169    public String getName() {
170      return name;
171    }
172  }
173
174  private class CacheEntry {
175    private String request;
176    private boolean persistent;
177    private ValidationResult v;
178    private ValueSetExpansionOutcome e;
179  }
180
181  private class NamedCache {
182    private String name;
183    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
184    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
185  }
186
187  private Object lock;
188  private String folder;
189  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
190
191  // use lock from the context
192  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
193    super();
194    this.lock = lock;
195    this.folder = folder;
196    if (folder != null)
197      load();
198  }
199
200  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
201    CacheToken ct = new CacheToken();
202    if (code.hasSystem())
203      ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem());
204    else
205      ct.name = NAME_FOR_NO_SYSTEM;
206    JsonParser json = new JsonParser();
207    json.setOutputStyle(OutputStyle.PRETTY);
208    ValueSet vsc = getVSEssense(vs);
209    try {
210      ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :"
211          + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}";
212    } catch (IOException e) {
213      throw new Error(e);
214    }
215    ct.key = String.valueOf(hashNWS(ct.request));
216    return ct;
217  }
218
219  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
220    CacheToken ct = new CacheToken();
221    for (Coding c : code.getCoding()) {
222      if (c.hasSystem())
223        ct.setName(c.getSystem());
224    }
225    JsonParser json = new JsonParser();
226    json.setOutputStyle(OutputStyle.PRETTY);
227    ValueSet vsc = getVSEssense(vs);
228    try {
229      ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :"
230          + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}";
231    } catch (IOException e) {
232      throw new Error(e);
233    }
234    ct.key = String.valueOf(hashNWS(ct.request));
235    return ct;
236  }
237
238  public ValueSet getVSEssense(ValueSet vs) {
239    if (vs == null)
240      return null;
241    ValueSet vsc = new ValueSet();
242    vsc.setCompose(vs.getCompose());
243    if (vs.hasExpansion()) {
244      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
245      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
246    }
247    return vsc;
248  }
249
250  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
251    CacheToken ct = new CacheToken();
252    ValueSet vsc = getVSEssense(vs);
253    for (ConceptSetComponent inc : vs.getCompose().getInclude())
254      if (inc.hasSystem())
255        ct.setName(inc.getSystem());
256    for (ConceptSetComponent inc : vs.getCompose().getExclude())
257      if (inc.hasSystem())
258        ct.setName(inc.getSystem());
259    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
260      if (inc.hasSystem())
261        ct.setName(inc.getSystem());
262    JsonParser json = new JsonParser();
263    json.setOutputStyle(OutputStyle.PRETTY);
264    try {
265      ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :"
266          + json.composeString(vsc) + "}\r\n";
267    } catch (IOException e) {
268      throw new Error(e);
269    }
270    ct.key = String.valueOf(hashNWS(ct.request));
271    return ct;
272  }
273
274  public NamedCache getNamedCache(CacheToken cacheToken) {
275    NamedCache nc = caches.get(cacheToken.name);
276    if (nc == null) {
277      nc = new NamedCache();
278      nc.name = cacheToken.name;
279      caches.put(nc.name, nc);
280    }
281    return nc;
282  }
283
284  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
285    synchronized (lock) {
286      NamedCache nc = getNamedCache(cacheToken);
287      CacheEntry e = nc.map.get(cacheToken.key);
288      if (e == null)
289        return null;
290      else
291        return e.e;
292    }
293  }
294
295  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
296    synchronized (lock) {
297      NamedCache nc = getNamedCache(cacheToken);
298      CacheEntry e = new CacheEntry();
299      e.request = cacheToken.request;
300      e.persistent = persistent;
301      e.e = res;
302      store(cacheToken, persistent, nc, e);
303    }
304  }
305
306  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
307    boolean n = nc.map.containsKey(cacheToken.key);
308    nc.map.put(cacheToken.key, e);
309    if (persistent) {
310      if (n) {
311        for (int i = nc.list.size() - 1; i >= 0; i--) {
312          if (nc.list.get(i).request.equals(e.request)) {
313            nc.list.remove(i);
314          }
315        }
316      }
317      nc.list.add(e);
318      save(nc);
319    }
320  }
321
322  public ValidationResult getValidation(CacheToken cacheToken) {
323    synchronized (lock) {
324      NamedCache nc = getNamedCache(cacheToken);
325      CacheEntry e = nc.map.get(cacheToken.key);
326      if (e == null)
327        return null;
328      else
329        return e.v;
330    }
331  }
332
333  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
334    synchronized (lock) {
335      NamedCache nc = getNamedCache(cacheToken);
336      CacheEntry e = new CacheEntry();
337      e.request = cacheToken.request;
338      e.persistent = persistent;
339      e.v = res;
340      store(cacheToken, persistent, nc, e);
341    }
342  }
343
344  // persistence
345
346  public void save() {
347
348  }
349
350  private void save(NamedCache nc) {
351    if (folder == null)
352      return;
353
354    try {
355      OutputStreamWriter sw = new OutputStreamWriter(new FileOutputStream(Utilities.path(folder, nc.name + ".cache")),
356          "UTF-8");
357      sw.write(ENTRY_MARKER + "\r\n");
358      JsonParser json = new JsonParser();
359      json.setOutputStyle(OutputStyle.PRETTY);
360      for (CacheEntry ce : nc.list) {
361        sw.write(ce.request.trim());
362        sw.write(BREAK + "\r\n");
363        if (ce.e != null) {
364          sw.write("e: {\r\n");
365          if (ce.e.getValueset() != null)
366            sw.write("  \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n");
367          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n");
368        } else {
369          sw.write("v: {\r\n");
370          sw.write("  \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n");
371          sw.write("  \"severity\" : "
372              + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n");
373          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n");
374        }
375        sw.write(ENTRY_MARKER + "\r\n");
376      }
377      sw.close();
378    } catch (Exception e) {
379      System.out.println("error saving " + nc.name + ": " + e.getMessage());
380    }
381  }
382
383  private void load() throws FHIRException {
384    for (String fn : new File(folder).list()) {
385      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
386        try {
387          // System.out.println("Load "+fn);
388          String title = fn.substring(0, fn.lastIndexOf("."));
389          NamedCache nc = new NamedCache();
390          nc.name = title;
391          caches.put(title, nc);
392          System.out.print(" - load " + title + ".cache");
393          String src = TextFile.fileToString(Utilities.path(folder, fn));
394          if (src.startsWith("?"))
395            src = src.substring(1);
396          int i = src.indexOf(ENTRY_MARKER);
397          while (i > -1) {
398            String s = src.substring(0, i);
399            System.out.print(".");
400            src = src.substring(i + ENTRY_MARKER.length() + 1);
401            i = src.indexOf(ENTRY_MARKER);
402            if (!Utilities.noString(s)) {
403              int j = s.indexOf(BREAK);
404              String q = s.substring(0, j);
405              String p = s.substring(j + BREAK.length() + 1).trim();
406              CacheEntry ce = new CacheEntry();
407              ce.persistent = true;
408              ce.request = q;
409              boolean e = p.charAt(0) == 'e';
410              p = p.substring(3);
411              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
412              String error = loadJS(o.get("error"));
413              if (e) {
414                if (o.has("valueSet"))
415                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")),
416                      error, TerminologyServiceErrorClass.UNKNOWN);
417                else
418                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
419              } else {
420                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null
421                    : IssueSeverity.fromCode(o.get("severity").getAsString());
422                String display = loadJS(o.get("display"));
423                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
424              }
425              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
426              nc.list.add(ce);
427            }
428          }
429          System.out.println("done");
430        } catch (Exception e) {
431          throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e);
432        }
433      }
434    }
435  }
436
437  private String loadJS(JsonElement e) {
438    if (e == null)
439      return null;
440    if (!(e instanceof JsonPrimitive))
441      return null;
442    String s = e.getAsString();
443    if ("".equals(s))
444      return null;
445    return s;
446  }
447
448  private String hashNWS(String s) {
449    s = StringUtils.remove(s, ' ');
450    s = StringUtils.remove(s, '\n');
451    s = StringUtils.remove(s, '\r');
452    return String.valueOf(s.hashCode());
453  }
454
455  // management
456
457  public TerminologyCache copy() {
458    // TODO Auto-generated method stub
459    return null;
460  }
461
462  public String summary(ValueSet vs) {
463    if (vs == null)
464      return "null";
465
466    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
467    for (ConceptSetComponent cc : vs.getCompose().getInclude())
468      b.append("Include " + getIncSummary(cc));
469    for (ConceptSetComponent cc : vs.getCompose().getExclude())
470      b.append("Exclude " + getIncSummary(cc));
471    return b.toString();
472  }
473
474  private String getIncSummary(ConceptSetComponent cc) {
475    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
476    for (UriType vs : cc.getValueSet())
477      b.append(vs.asStringValue());
478    String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
479    String system = cc.getSystem();
480    if (cc.hasConcept())
481      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
482    if (cc.hasFilter()) {
483      String s = "";
484      for (ConceptSetFilterComponent f : cc.getFilter()) {
485        if (!Utilities.noString(s))
486          s = s + " & ";
487        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
488      }
489      return "from " + system + " where " + s + vsd;
490    }
491    return "All codes from " + system + vsd;
492  }
493
494  public String summary(Coding code) {
495    return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\"";
496  }
497
498  public String summary(CodeableConcept code) {
499    StringBuilder b = new StringBuilder();
500    b.append("{");
501    boolean first = true;
502    for (Coding c : code.getCoding()) {
503      if (first)
504        first = false;
505      else
506        b.append(",");
507      b.append(summary(c));
508    }
509    b.append("}: \"");
510    b.append(code.getText());
511    b.append("\"");
512    return b.toString();
513  }
514
515}