001package org.hl7.fhir.r4.context; 002 003/* 004 Copyright (c) 2011+, HL7, Inc. 005 All rights reserved. 006 007 Redistribution and use in source and binary forms, with or without modification, 008 are permitted provided that the following conditions are met: 009 010 * Redistributions of source code must retain the above copyright notice, this 011 list of conditions and the following disclaimer. 012 * Redistributions in binary form must reproduce the above copyright notice, 013 this list of conditions and the following disclaimer in the documentation 014 and/or other materials provided with the distribution. 015 * Neither the name of HL7 nor the names of its contributors may be used to 016 endorse or promote products derived from this software without specific 017 prior written permission. 018 019 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 022 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 023 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 024 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 025 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 026 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 027 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 028 POSSIBILITY OF SUCH DAMAGE. 029 030 */ 031 032import java.io.File; 033import java.io.FileNotFoundException; 034import java.io.FileOutputStream; 035import java.io.IOException; 036import java.io.OutputStreamWriter; 037import java.util.ArrayList; 038import java.util.HashMap; 039import java.util.List; 040import java.util.Map; 041 042import org.apache.commons.lang3.StringUtils; 043import org.hl7.fhir.exceptions.FHIRException; 044import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult; 045import org.hl7.fhir.r4.formats.IParser.OutputStyle; 046import org.hl7.fhir.r4.formats.JsonParser; 047import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent; 048import org.hl7.fhir.r4.model.CodeableConcept; 049import org.hl7.fhir.r4.model.Coding; 050import org.hl7.fhir.r4.model.UriType; 051import org.hl7.fhir.r4.model.ValueSet; 052import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; 053import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent; 054import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent; 055import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass; 056import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; 057import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; 058import org.hl7.fhir.utilities.TextFile; 059import org.hl7.fhir.utilities.Utilities; 060import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; 061import org.hl7.fhir.utilities.validation.ValidationOptions; 062 063import com.google.gson.JsonElement; 064import com.google.gson.JsonNull; 065import com.google.gson.JsonObject; 066import com.google.gson.JsonPrimitive; 067 068/** 069 * This implements a two level cache. - a temporary cache for remmbering 070 * previous local operations - a persistent cache for rembering tx server 071 * operations 072 * 073 * the cache is a series of pairs: a map, and a list. the map is the loaded 074 * cache, the list is the persiistent cache, carefully maintained in order for 075 * version control consistency 076 * 077 * @author graha 078 * 079 */ 080public class TerminologyCache { 081 public static final boolean TRANSIENT = false; 082 public static final boolean PERMANENT = true; 083 private static final String NAME_FOR_NO_SYSTEM = "all-systems"; 084 private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------"; 085 private static final String BREAK = "####"; 086 087 private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator(); 088 089 protected SystemNameKeyGenerator getSystemNameKeyGenerator() { 090 return systemNameKeyGenerator; 091 } 092 093 public class SystemNameKeyGenerator { 094 public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct"; 095 public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm"; 096 public static final String LOINC_CODESYSTEM_URL = "http://loinc.org"; 097 public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org"; 098 099 public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/"; 100 public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/"; 101 public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/"; 102 103 public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:"; 104 public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47"; 105 public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13"; 106 107 public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101"; 108 public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM"; 109 110 public String getNameForSystem(String system) { 111 final int lastPipe = system.lastIndexOf('|'); 112 final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe); 113 final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1); 114 115 if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL)) 116 return getVersionedSystem("snomed", systemVersion); 117 if (systemBaseName.equals(RXNORM_CODESYSTEM_URL)) 118 return getVersionedSystem("rxnorm", systemVersion); 119 if (systemBaseName.equals(LOINC_CODESYSTEM_URL)) 120 return getVersionedSystem("loinc", systemVersion); 121 if (systemBaseName.equals(UCUM_CODESYSTEM_URL)) 122 return getVersionedSystem("ucum", systemVersion); 123 if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL)) 124 return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 125 if (systemBaseName.equals(_11073_CODESYSTEM_URN)) 126 return getVersionedSystem("11073", systemVersion); 127 if (systemBaseName.startsWith(ISO_CODESYSTEM_URN)) 128 return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""), 129 systemVersion); 130 if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL)) 131 return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 132 if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL)) 133 return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 134 if (systemBaseName.equals(LANG_CODESYSTEM_URN)) 135 return getVersionedSystem("lang", systemVersion); 136 if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN)) 137 return getVersionedSystem("mimetypes", systemVersion); 138 if (systemBaseName.equals(DICOM_CODESYSTEM_URL)) 139 return getVersionedSystem("dicom", systemVersion); 140 return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"), 141 systemVersion); 142 } 143 144 public String normalizeBaseURL(String baseUrl, String fullUrl) { 145 return fullUrl.substring(baseUrl.length()).replace("/", ""); 146 } 147 148 public String getVersionedSystem(String baseSystem, String version) { 149 if (version != null) { 150 return baseSystem + "_" + version; 151 } 152 return baseSystem; 153 } 154 } 155 156 public class CacheToken { 157 private String name; 158 private String key; 159 private String request; 160 161 public void setName(String n) { 162 String systemName = getSystemNameKeyGenerator().getNameForSystem(n); 163 if (name == null) 164 name = systemName; 165 else if (!systemName.equals(name)) 166 name = NAME_FOR_NO_SYSTEM; 167 } 168 169 public String getName() { 170 return name; 171 } 172 } 173 174 private class CacheEntry { 175 private String request; 176 private boolean persistent; 177 private ValidationResult v; 178 private ValueSetExpansionOutcome e; 179 } 180 181 private class NamedCache { 182 private String name; 183 private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries 184 private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>(); 185 } 186 187 private Object lock; 188 private String folder; 189 private Map<String, NamedCache> caches = new HashMap<String, NamedCache>(); 190 191 // use lock from the context 192 public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException { 193 super(); 194 this.lock = lock; 195 this.folder = folder; 196 if (folder != null) 197 load(); 198 } 199 200 public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) { 201 CacheToken ct = new CacheToken(); 202 if (code.hasSystem()) 203 ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem()); 204 else 205 ct.name = NAME_FOR_NO_SYSTEM; 206 JsonParser json = new JsonParser(); 207 json.setOutputStyle(OutputStyle.PRETTY); 208 ValueSet vsc = getVSEssense(vs); 209 try { 210 ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :" 211 + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}"; 212 } catch (IOException e) { 213 throw new Error(e); 214 } 215 ct.key = String.valueOf(hashNWS(ct.request)); 216 return ct; 217 } 218 219 public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) { 220 CacheToken ct = new CacheToken(); 221 for (Coding c : code.getCoding()) { 222 if (c.hasSystem()) 223 ct.setName(c.getSystem()); 224 } 225 JsonParser json = new JsonParser(); 226 json.setOutputStyle(OutputStyle.PRETTY); 227 ValueSet vsc = getVSEssense(vs); 228 try { 229 ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :" 230 + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}"; 231 } catch (IOException e) { 232 throw new Error(e); 233 } 234 ct.key = String.valueOf(hashNWS(ct.request)); 235 return ct; 236 } 237 238 public ValueSet getVSEssense(ValueSet vs) { 239 if (vs == null) 240 return null; 241 ValueSet vsc = new ValueSet(); 242 vsc.setCompose(vs.getCompose()); 243 if (vs.hasExpansion()) { 244 vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter()); 245 vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains()); 246 } 247 return vsc; 248 } 249 250 public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) { 251 CacheToken ct = new CacheToken(); 252 ValueSet vsc = getVSEssense(vs); 253 for (ConceptSetComponent inc : vs.getCompose().getInclude()) 254 if (inc.hasSystem()) 255 ct.setName(inc.getSystem()); 256 for (ConceptSetComponent inc : vs.getCompose().getExclude()) 257 if (inc.hasSystem()) 258 ct.setName(inc.getSystem()); 259 for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains()) 260 if (inc.hasSystem()) 261 ct.setName(inc.getSystem()); 262 JsonParser json = new JsonParser(); 263 json.setOutputStyle(OutputStyle.PRETTY); 264 try { 265 ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :" 266 + json.composeString(vsc) + "}\r\n"; 267 } catch (IOException e) { 268 throw new Error(e); 269 } 270 ct.key = String.valueOf(hashNWS(ct.request)); 271 return ct; 272 } 273 274 public NamedCache getNamedCache(CacheToken cacheToken) { 275 NamedCache nc = caches.get(cacheToken.name); 276 if (nc == null) { 277 nc = new NamedCache(); 278 nc.name = cacheToken.name; 279 caches.put(nc.name, nc); 280 } 281 return nc; 282 } 283 284 public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) { 285 synchronized (lock) { 286 NamedCache nc = getNamedCache(cacheToken); 287 CacheEntry e = nc.map.get(cacheToken.key); 288 if (e == null) 289 return null; 290 else 291 return e.e; 292 } 293 } 294 295 public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) { 296 synchronized (lock) { 297 NamedCache nc = getNamedCache(cacheToken); 298 CacheEntry e = new CacheEntry(); 299 e.request = cacheToken.request; 300 e.persistent = persistent; 301 e.e = res; 302 store(cacheToken, persistent, nc, e); 303 } 304 } 305 306 public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) { 307 boolean n = nc.map.containsKey(cacheToken.key); 308 nc.map.put(cacheToken.key, e); 309 if (persistent) { 310 if (n) { 311 for (int i = nc.list.size() - 1; i >= 0; i--) { 312 if (nc.list.get(i).request.equals(e.request)) { 313 nc.list.remove(i); 314 } 315 } 316 } 317 nc.list.add(e); 318 save(nc); 319 } 320 } 321 322 public ValidationResult getValidation(CacheToken cacheToken) { 323 synchronized (lock) { 324 NamedCache nc = getNamedCache(cacheToken); 325 CacheEntry e = nc.map.get(cacheToken.key); 326 if (e == null) 327 return null; 328 else 329 return e.v; 330 } 331 } 332 333 public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) { 334 synchronized (lock) { 335 NamedCache nc = getNamedCache(cacheToken); 336 CacheEntry e = new CacheEntry(); 337 e.request = cacheToken.request; 338 e.persistent = persistent; 339 e.v = res; 340 store(cacheToken, persistent, nc, e); 341 } 342 } 343 344 // persistence 345 346 public void save() { 347 348 } 349 350 private void save(NamedCache nc) { 351 if (folder == null) 352 return; 353 354 try { 355 OutputStreamWriter sw = new OutputStreamWriter(new FileOutputStream(Utilities.path(folder, nc.name + ".cache")), 356 "UTF-8"); 357 sw.write(ENTRY_MARKER + "\r\n"); 358 JsonParser json = new JsonParser(); 359 json.setOutputStyle(OutputStyle.PRETTY); 360 for (CacheEntry ce : nc.list) { 361 sw.write(ce.request.trim()); 362 sw.write(BREAK + "\r\n"); 363 if (ce.e != null) { 364 sw.write("e: {\r\n"); 365 if (ce.e.getValueset() != null) 366 sw.write(" \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n"); 367 sw.write(" \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n"); 368 } else { 369 sw.write("v: {\r\n"); 370 sw.write(" \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n"); 371 sw.write(" \"severity\" : " 372 + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n"); 373 sw.write(" \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n"); 374 } 375 sw.write(ENTRY_MARKER + "\r\n"); 376 } 377 sw.close(); 378 } catch (Exception e) { 379 System.out.println("error saving " + nc.name + ": " + e.getMessage()); 380 } 381 } 382 383 private void load() throws FHIRException { 384 for (String fn : new File(folder).list()) { 385 if (fn.endsWith(".cache") && !fn.equals("validation.cache")) { 386 try { 387 // System.out.println("Load "+fn); 388 String title = fn.substring(0, fn.lastIndexOf(".")); 389 NamedCache nc = new NamedCache(); 390 nc.name = title; 391 caches.put(title, nc); 392 System.out.print(" - load " + title + ".cache"); 393 String src = TextFile.fileToString(Utilities.path(folder, fn)); 394 if (src.startsWith("?")) 395 src = src.substring(1); 396 int i = src.indexOf(ENTRY_MARKER); 397 while (i > -1) { 398 String s = src.substring(0, i); 399 System.out.print("."); 400 src = src.substring(i + ENTRY_MARKER.length() + 1); 401 i = src.indexOf(ENTRY_MARKER); 402 if (!Utilities.noString(s)) { 403 int j = s.indexOf(BREAK); 404 String q = s.substring(0, j); 405 String p = s.substring(j + BREAK.length() + 1).trim(); 406 CacheEntry ce = new CacheEntry(); 407 ce.persistent = true; 408 ce.request = q; 409 boolean e = p.charAt(0) == 'e'; 410 p = p.substring(3); 411 JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p); 412 String error = loadJS(o.get("error")); 413 if (e) { 414 if (o.has("valueSet")) 415 ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), 416 error, TerminologyServiceErrorClass.UNKNOWN); 417 else 418 ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN); 419 } else { 420 IssueSeverity severity = o.get("severity") instanceof JsonNull ? null 421 : IssueSeverity.fromCode(o.get("severity").getAsString()); 422 String display = loadJS(o.get("display")); 423 ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display)); 424 } 425 nc.map.put(String.valueOf(hashNWS(ce.request)), ce); 426 nc.list.add(ce); 427 } 428 } 429 System.out.println("done"); 430 } catch (Exception e) { 431 throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e); 432 } 433 } 434 } 435 } 436 437 private String loadJS(JsonElement e) { 438 if (e == null) 439 return null; 440 if (!(e instanceof JsonPrimitive)) 441 return null; 442 String s = e.getAsString(); 443 if ("".equals(s)) 444 return null; 445 return s; 446 } 447 448 private String hashNWS(String s) { 449 s = StringUtils.remove(s, ' '); 450 s = StringUtils.remove(s, '\n'); 451 s = StringUtils.remove(s, '\r'); 452 return String.valueOf(s.hashCode()); 453 } 454 455 // management 456 457 public TerminologyCache copy() { 458 // TODO Auto-generated method stub 459 return null; 460 } 461 462 public String summary(ValueSet vs) { 463 if (vs == null) 464 return "null"; 465 466 CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); 467 for (ConceptSetComponent cc : vs.getCompose().getInclude()) 468 b.append("Include " + getIncSummary(cc)); 469 for (ConceptSetComponent cc : vs.getCompose().getExclude()) 470 b.append("Exclude " + getIncSummary(cc)); 471 return b.toString(); 472 } 473 474 private String getIncSummary(ConceptSetComponent cc) { 475 CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); 476 for (UriType vs : cc.getValueSet()) 477 b.append(vs.asStringValue()); 478 String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : ""; 479 String system = cc.getSystem(); 480 if (cc.hasConcept()) 481 return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd; 482 if (cc.hasFilter()) { 483 String s = ""; 484 for (ConceptSetFilterComponent f : cc.getFilter()) { 485 if (!Utilities.noString(s)) 486 s = s + " & "; 487 s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue(); 488 } 489 return "from " + system + " where " + s + vsd; 490 } 491 return "All codes from " + system + vsd; 492 } 493 494 public String summary(Coding code) { 495 return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\""; 496 } 497 498 public String summary(CodeableConcept code) { 499 StringBuilder b = new StringBuilder(); 500 b.append("{"); 501 boolean first = true; 502 for (Coding c : code.getCoding()) { 503 if (first) 504 first = false; 505 else 506 b.append(","); 507 b.append(summary(c)); 508 } 509 b.append("}: \""); 510 b.append(code.getText()); 511 b.append("\""); 512 return b.toString(); 513 } 514 515}