Source code

001package org.hl7.fhir.r4.context;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032import java.io.File;
033import java.io.FileNotFoundException;
034import java.io.FileOutputStream;
035import java.io.IOException;
036import java.io.OutputStreamWriter;
037import java.util.ArrayList;
038import java.util.HashMap;
039import java.util.List;
040import java.util.Map;
041
042import lombok.extern.slf4j.Slf4j;
043import org.apache.commons.lang3.StringUtils;
044import org.hl7.fhir.exceptions.FHIRException;
045import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
046import org.hl7.fhir.r4.formats.IParser.OutputStyle;
047import org.hl7.fhir.r4.formats.JsonParser;
048import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
049import org.hl7.fhir.r4.model.CodeableConcept;
050import org.hl7.fhir.r4.model.Coding;
051import org.hl7.fhir.r4.model.UriType;
052import org.hl7.fhir.r4.model.ValueSet;
053import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
054import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
055import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
056import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
057import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
058import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
059import org.hl7.fhir.utilities.FileUtilities;
060import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
061import org.hl7.fhir.utilities.Utilities;
062import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
063import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
064import org.hl7.fhir.utilities.validation.ValidationOptions;
065
066import com.google.gson.JsonElement;
067import com.google.gson.JsonNull;
068import com.google.gson.JsonObject;
069import com.google.gson.JsonPrimitive;
070
071/**
072 * This implements a two level cache. - a temporary cache for remmbering
073 * previous local operations - a persistent cache for rembering tx server
074 * operations
075 * 
076 * the cache is a series of pairs: a map, and a list. the map is the loaded
077 * cache, the list is the persiistent cache, carefully maintained in order for
078 * version control consistency
079 * 
080 * @author graha
081 *
082 */
083@MarkedToMoveToAdjunctPackage
084@Slf4j
085public class TerminologyCache {
086  public static final boolean TRANSIENT = false;
087  public static final boolean PERMANENT = true;
088  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
089  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
090  private static final String BREAK = "####";
091
092  private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator();
093
094  protected SystemNameKeyGenerator getSystemNameKeyGenerator() {
095    return systemNameKeyGenerator;
096  }
097
098  public class SystemNameKeyGenerator {
099    public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct";
100    public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm";
101    public static final String LOINC_CODESYSTEM_URL = "http://loinc.org";
102    public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org";
103
104    public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/";
105    public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/";
106    public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/";
107
108    public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:";
109    public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47";
110    public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13";
111
112    public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101";
113    public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM";
114
115    public String getNameForSystem(String system) {
116      final int lastPipe = system.lastIndexOf('|');
117      final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe);
118      final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1);
119
120      if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL))
121        return getVersionedSystem("snomed", systemVersion);
122      if (systemBaseName.equals(RXNORM_CODESYSTEM_URL))
123        return getVersionedSystem("rxnorm", systemVersion);
124      if (systemBaseName.equals(LOINC_CODESYSTEM_URL))
125        return getVersionedSystem("loinc", systemVersion);
126      if (systemBaseName.equals(UCUM_CODESYSTEM_URL))
127        return getVersionedSystem("ucum", systemVersion);
128      if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL))
129        return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
130      if (systemBaseName.equals(_11073_CODESYSTEM_URN))
131        return getVersionedSystem("11073", systemVersion);
132      if (systemBaseName.startsWith(ISO_CODESYSTEM_URN))
133        return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""),
134            systemVersion);
135      if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL))
136        return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
137      if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL))
138        return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion);
139      if (systemBaseName.equals(LANG_CODESYSTEM_URN))
140        return getVersionedSystem("lang", systemVersion);
141      if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN))
142        return getVersionedSystem("mimetypes", systemVersion);
143      if (systemBaseName.equals(DICOM_CODESYSTEM_URL))
144        return getVersionedSystem("dicom", systemVersion);
145      return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"),
146          systemVersion);
147    }
148
149    public String normalizeBaseURL(String baseUrl, String fullUrl) {
150      return fullUrl.substring(baseUrl.length()).replace("/", "");
151    }
152
153    public String getVersionedSystem(String baseSystem, String version) {
154      if (version != null) {
155        return baseSystem + "_" + version;
156      }
157      return baseSystem;
158    }
159  }
160
161  public class CacheToken {
162    private String name;
163    private String key;
164    private String request;
165
166    public void setName(String n) {
167      String systemName = getSystemNameKeyGenerator().getNameForSystem(n);
168      if (name == null)
169        name = systemName;
170      else if (!systemName.equals(name))
171        name = NAME_FOR_NO_SYSTEM;
172    }
173
174    public String getName() {
175      return name;
176    }
177  }
178
179  private class CacheEntry {
180    private String request;
181    private boolean persistent;
182    private ValidationResult v;
183    private ValueSetExpansionOutcome e;
184  }
185
186  private class NamedCache {
187    private String name;
188    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
189    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
190  }
191
192  private Object lock;
193  private String folder;
194  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
195
196  // use lock from the context
197  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
198    super();
199    this.lock = lock;
200    this.folder = folder;
201    if (folder != null)
202      load();
203  }
204
205  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
206    CacheToken ct = new CacheToken();
207    if (code.hasSystem())
208      ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem());
209    else
210      ct.name = NAME_FOR_NO_SYSTEM;
211    JsonParser json = new JsonParser();
212    json.setOutputStyle(OutputStyle.PRETTY);
213    ValueSet vsc = getVSEssense(vs);
214    try {
215      ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :"
216          + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}";
217    } catch (IOException e) {
218      throw new Error(e);
219    }
220    ct.key = String.valueOf(hashNWS(ct.request));
221    return ct;
222  }
223
224  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
225    CacheToken ct = new CacheToken();
226    for (Coding c : code.getCoding()) {
227      if (c.hasSystem())
228        ct.setName(c.getSystem());
229    }
230    JsonParser json = new JsonParser();
231    json.setOutputStyle(OutputStyle.PRETTY);
232    ValueSet vsc = getVSEssense(vs);
233    try {
234      ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :"
235          + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}";
236    } catch (IOException e) {
237      throw new Error(e);
238    }
239    ct.key = String.valueOf(hashNWS(ct.request));
240    return ct;
241  }
242
243  public ValueSet getVSEssense(ValueSet vs) {
244    if (vs == null)
245      return null;
246    ValueSet vsc = new ValueSet();
247    vsc.setCompose(vs.getCompose());
248    if (vs.hasExpansion()) {
249      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
250      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
251    }
252    return vsc;
253  }
254
255  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
256    CacheToken ct = new CacheToken();
257    ValueSet vsc = getVSEssense(vs);
258    for (ConceptSetComponent inc : vs.getCompose().getInclude())
259      if (inc.hasSystem())
260        ct.setName(inc.getSystem());
261    for (ConceptSetComponent inc : vs.getCompose().getExclude())
262      if (inc.hasSystem())
263        ct.setName(inc.getSystem());
264    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
265      if (inc.hasSystem())
266        ct.setName(inc.getSystem());
267    JsonParser json = new JsonParser();
268    json.setOutputStyle(OutputStyle.PRETTY);
269    try {
270      ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :"
271          + json.composeString(vsc) + "}\r\n";
272    } catch (IOException e) {
273      throw new Error(e);
274    }
275    ct.key = String.valueOf(hashNWS(ct.request));
276    return ct;
277  }
278
279  public NamedCache getNamedCache(CacheToken cacheToken) {
280    NamedCache nc = caches.get(cacheToken.name);
281    if (nc == null) {
282      nc = new NamedCache();
283      nc.name = cacheToken.name;
284      caches.put(nc.name, nc);
285    }
286    return nc;
287  }
288
289  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
290    synchronized (lock) {
291      NamedCache nc = getNamedCache(cacheToken);
292      CacheEntry e = nc.map.get(cacheToken.key);
293      if (e == null)
294        return null;
295      else
296        return e.e;
297    }
298  }
299
300  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
301    synchronized (lock) {
302      NamedCache nc = getNamedCache(cacheToken);
303      CacheEntry e = new CacheEntry();
304      e.request = cacheToken.request;
305      e.persistent = persistent;
306      e.e = res;
307      store(cacheToken, persistent, nc, e);
308    }
309  }
310
311  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
312    boolean n = nc.map.containsKey(cacheToken.key);
313    nc.map.put(cacheToken.key, e);
314    if (persistent) {
315      if (n) {
316        for (int i = nc.list.size() - 1; i >= 0; i--) {
317          if (nc.list.get(i).request.equals(e.request)) {
318            nc.list.remove(i);
319          }
320        }
321      }
322      nc.list.add(e);
323      save(nc);
324    }
325  }
326
327  public ValidationResult getValidation(CacheToken cacheToken) {
328    synchronized (lock) {
329      NamedCache nc = getNamedCache(cacheToken);
330      CacheEntry e = nc.map.get(cacheToken.key);
331      if (e == null)
332        return null;
333      else
334        return e.v;
335    }
336  }
337
338  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
339    synchronized (lock) {
340      NamedCache nc = getNamedCache(cacheToken);
341      CacheEntry e = new CacheEntry();
342      e.request = cacheToken.request;
343      e.persistent = persistent;
344      e.v = res;
345      store(cacheToken, persistent, nc, e);
346    }
347  }
348
349  // persistence
350
351  public void save() {
352
353  }
354
355  private void save(NamedCache nc) {
356    if (folder == null)
357      return;
358
359    try {
360      OutputStreamWriter sw = new OutputStreamWriter(ManagedFileAccess.outStream(Utilities.path(folder, nc.name + ".cache")),
361          "UTF-8");
362      sw.write(ENTRY_MARKER + "\r\n");
363      JsonParser json = new JsonParser();
364      json.setOutputStyle(OutputStyle.PRETTY);
365      for (CacheEntry ce : nc.list) {
366        sw.write(ce.request.trim());
367        sw.write(BREAK + "\r\n");
368        if (ce.e != null) {
369          sw.write("e: {\r\n");
370          if (ce.e.getValueset() != null)
371            sw.write("  \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n");
372          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n");
373        } else {
374          sw.write("v: {\r\n");
375          sw.write("  \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n");
376          sw.write("  \"severity\" : "
377              + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n");
378          sw.write("  \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n");
379        }
380        sw.write(ENTRY_MARKER + "\r\n");
381      }
382      sw.close();
383    } catch (Exception e) {
384      log.error("error saving " + nc.name + ": " + e.getMessage());
385    }
386  }
387
388  private void load() throws FHIRException, IOException {
389    for (String fn : ManagedFileAccess.file(folder).list()) {
390      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
391        try {
392          String title = fn.substring(0, fn.lastIndexOf("."));
393          NamedCache nc = new NamedCache();
394          nc.name = title;
395          caches.put(title, nc);
396          log.info(" - load " + title + ".cache");
397          String src = FileUtilities.fileToString(Utilities.path(folder, fn));
398          if (src.startsWith("?"))
399            src = src.substring(1);
400          int i = src.indexOf(ENTRY_MARKER);
401          while (i > -1) {
402            String s = src.substring(0, i);
403            log.info(".");
404            src = src.substring(i + ENTRY_MARKER.length() + 1);
405            i = src.indexOf(ENTRY_MARKER);
406            if (!Utilities.noString(s)) {
407              int j = s.indexOf(BREAK);
408              String q = s.substring(0, j);
409              String p = s.substring(j + BREAK.length() + 1).trim();
410              CacheEntry ce = new CacheEntry();
411              ce.persistent = true;
412              ce.request = q;
413              boolean e = p.charAt(0) == 'e';
414              p = p.substring(3);
415              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
416              String error = loadJS(o.get("error"));
417              if (e) {
418                if (o.has("valueSet"))
419                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")),
420                      error, TerminologyServiceErrorClass.UNKNOWN);
421                else
422                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
423              } else {
424                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null
425                    : IssueSeverity.fromCode(o.get("severity").getAsString());
426                String display = loadJS(o.get("display"));
427                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
428              }
429              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
430              nc.list.add(ce);
431            }
432          }
433          log.info("done");
434        } catch (Exception e) {
435          throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e);
436        }
437      }
438    }
439  }
440
441  private String loadJS(JsonElement e) {
442    if (e == null)
443      return null;
444    if (!(e instanceof JsonPrimitive))
445      return null;
446    String s = e.getAsString();
447    if ("".equals(s))
448      return null;
449    return s;
450  }
451
452  private String hashNWS(String s) {
453    s = StringUtils.remove(s, ' ');
454    s = StringUtils.remove(s, '\n');
455    s = StringUtils.remove(s, '\r');
456    return String.valueOf(s.hashCode());
457  }
458
459  // management
460
461  public TerminologyCache copy() {
462    // TODO Auto-generated method stub
463    return null;
464  }
465
466  public String summary(ValueSet vs) {
467    if (vs == null)
468      return "null";
469
470    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
471    for (ConceptSetComponent cc : vs.getCompose().getInclude())
472      b.append("Include " + getIncSummary(cc));
473    for (ConceptSetComponent cc : vs.getCompose().getExclude())
474      b.append("Exclude " + getIncSummary(cc));
475    return b.toString();
476  }
477
478  private String getIncSummary(ConceptSetComponent cc) {
479    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
480    for (UriType vs : cc.getValueSet())
481      b.append(vs.asStringValue());
482    String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : "";
483    String system = cc.getSystem();
484    if (cc.hasConcept())
485      return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd;
486    if (cc.hasFilter()) {
487      String s = "";
488      for (ConceptSetFilterComponent f : cc.getFilter()) {
489        if (!Utilities.noString(s))
490          s = s + " & ";
491        s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue();
492      }
493      return "from " + system + " where " + s + vsd;
494    }
495    return "All codes from " + system + vsd;
496  }
497
498  public String summary(Coding code) {
499    return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\"";
500  }
501
502  public String summary(CodeableConcept code) {
503    StringBuilder b = new StringBuilder();
504    b.append("{");
505    boolean first = true;
506    for (Coding c : code.getCoding()) {
507      if (first)
508        first = false;
509      else
510        b.append(",");
511      b.append(summary(c));
512    }
513    b.append("}: \"");
514    b.append(code.getText());
515    b.append("\"");
516    return b.toString();
517  }
518
519}