
001package org.hl7.fhir.r4.context; 002 003/* 004 Copyright (c) 2011+, HL7, Inc. 005 All rights reserved. 006 007 Redistribution and use in source and binary forms, with or without modification, 008 are permitted provided that the following conditions are met: 009 010 * Redistributions of source code must retain the above copyright notice, this 011 list of conditions and the following disclaimer. 012 * Redistributions in binary form must reproduce the above copyright notice, 013 this list of conditions and the following disclaimer in the documentation 014 and/or other materials provided with the distribution. 015 * Neither the name of HL7 nor the names of its contributors may be used to 016 endorse or promote products derived from this software without specific 017 prior written permission. 018 019 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 020 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 021 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 022 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 023 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 024 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 025 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 026 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 027 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 028 POSSIBILITY OF SUCH DAMAGE. 029 030 */ 031 032import java.io.File; 033import java.io.FileNotFoundException; 034import java.io.FileOutputStream; 035import java.io.IOException; 036import java.io.OutputStreamWriter; 037import java.util.ArrayList; 038import java.util.HashMap; 039import java.util.List; 040import java.util.Map; 041 042import org.apache.commons.lang3.StringUtils; 043import org.hl7.fhir.exceptions.FHIRException; 044import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult; 045import org.hl7.fhir.r4.formats.IParser.OutputStyle; 046import org.hl7.fhir.r4.formats.JsonParser; 047import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent; 048import org.hl7.fhir.r4.model.CodeableConcept; 049import org.hl7.fhir.r4.model.Coding; 050import org.hl7.fhir.r4.model.UriType; 051import org.hl7.fhir.r4.model.ValueSet; 052import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent; 053import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent; 054import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent; 055import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass; 056import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome; 057import org.hl7.fhir.utilities.CommaSeparatedStringBuilder; 058import org.hl7.fhir.utilities.FileUtilities; 059import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage; 060import org.hl7.fhir.utilities.Utilities; 061import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 062import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity; 063import org.hl7.fhir.utilities.validation.ValidationOptions; 064 065import com.google.gson.JsonElement; 066import com.google.gson.JsonNull; 067import com.google.gson.JsonObject; 068import com.google.gson.JsonPrimitive; 069 070/** 071 * This implements a two level cache. - a temporary cache for remmbering 072 * previous local operations - a persistent cache for rembering tx server 073 * operations 074 * 075 * the cache is a series of pairs: a map, and a list. the map is the loaded 076 * cache, the list is the persiistent cache, carefully maintained in order for 077 * version control consistency 078 * 079 * @author graha 080 * 081 */ 082@MarkedToMoveToAdjunctPackage 083public class TerminologyCache { 084 public static final boolean TRANSIENT = false; 085 public static final boolean PERMANENT = true; 086 private static final String NAME_FOR_NO_SYSTEM = "all-systems"; 087 private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------"; 088 private static final String BREAK = "####"; 089 090 private SystemNameKeyGenerator systemNameKeyGenerator = new SystemNameKeyGenerator(); 091 092 protected SystemNameKeyGenerator getSystemNameKeyGenerator() { 093 return systemNameKeyGenerator; 094 } 095 096 public class SystemNameKeyGenerator { 097 public static final String SNOMED_SCT_CODESYSTEM_URL = "http://snomed.info/sct"; 098 public static final String RXNORM_CODESYSTEM_URL = "http://www.nlm.nih.gov/research/umls/rxnorm"; 099 public static final String LOINC_CODESYSTEM_URL = "http://loinc.org"; 100 public static final String UCUM_CODESYSTEM_URL = "http://unitsofmeasure.org"; 101 102 public static final String HL7_TERMINOLOGY_CODESYSTEM_BASE_URL = "http://terminology.hl7.org/CodeSystem/"; 103 public static final String HL7_SID_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/sid/"; 104 public static final String HL7_FHIR_CODESYSTEM_BASE_URL = "http://hl7.org/fhir/"; 105 106 public static final String ISO_CODESYSTEM_URN = "urn:iso:std:iso:"; 107 public static final String LANG_CODESYSTEM_URN = "urn:ietf:bcp:47"; 108 public static final String MIMETYPES_CODESYSTEM_URN = "urn:ietf:bcp:13"; 109 110 public static final String _11073_CODESYSTEM_URN = "urn:iso:std:iso:11073:10101"; 111 public static final String DICOM_CODESYSTEM_URL = "http://dicom.nema.org/resources/ontology/DCM"; 112 113 public String getNameForSystem(String system) { 114 final int lastPipe = system.lastIndexOf('|'); 115 final String systemBaseName = lastPipe == -1 ? system : system.substring(0, lastPipe); 116 final String systemVersion = lastPipe == -1 ? null : system.substring(lastPipe + 1); 117 118 if (systemBaseName.equals(SNOMED_SCT_CODESYSTEM_URL)) 119 return getVersionedSystem("snomed", systemVersion); 120 if (systemBaseName.equals(RXNORM_CODESYSTEM_URL)) 121 return getVersionedSystem("rxnorm", systemVersion); 122 if (systemBaseName.equals(LOINC_CODESYSTEM_URL)) 123 return getVersionedSystem("loinc", systemVersion); 124 if (systemBaseName.equals(UCUM_CODESYSTEM_URL)) 125 return getVersionedSystem("ucum", systemVersion); 126 if (systemBaseName.startsWith(HL7_SID_CODESYSTEM_BASE_URL)) 127 return getVersionedSystem(normalizeBaseURL(HL7_SID_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 128 if (systemBaseName.equals(_11073_CODESYSTEM_URN)) 129 return getVersionedSystem("11073", systemVersion); 130 if (systemBaseName.startsWith(ISO_CODESYSTEM_URN)) 131 return getVersionedSystem("iso" + systemBaseName.substring(ISO_CODESYSTEM_URN.length()).replace(":", ""), 132 systemVersion); 133 if (systemBaseName.startsWith(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL)) 134 return getVersionedSystem(normalizeBaseURL(HL7_TERMINOLOGY_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 135 if (systemBaseName.startsWith(HL7_FHIR_CODESYSTEM_BASE_URL)) 136 return getVersionedSystem(normalizeBaseURL(HL7_FHIR_CODESYSTEM_BASE_URL, systemBaseName), systemVersion); 137 if (systemBaseName.equals(LANG_CODESYSTEM_URN)) 138 return getVersionedSystem("lang", systemVersion); 139 if (systemBaseName.equals(MIMETYPES_CODESYSTEM_URN)) 140 return getVersionedSystem("mimetypes", systemVersion); 141 if (systemBaseName.equals(DICOM_CODESYSTEM_URL)) 142 return getVersionedSystem("dicom", systemVersion); 143 return getVersionedSystem(systemBaseName.replace("/", "_").replace(":", "_").replace("?", "X").replace("#", "X"), 144 systemVersion); 145 } 146 147 public String normalizeBaseURL(String baseUrl, String fullUrl) { 148 return fullUrl.substring(baseUrl.length()).replace("/", ""); 149 } 150 151 public String getVersionedSystem(String baseSystem, String version) { 152 if (version != null) { 153 return baseSystem + "_" + version; 154 } 155 return baseSystem; 156 } 157 } 158 159 public class CacheToken { 160 private String name; 161 private String key; 162 private String request; 163 164 public void setName(String n) { 165 String systemName = getSystemNameKeyGenerator().getNameForSystem(n); 166 if (name == null) 167 name = systemName; 168 else if (!systemName.equals(name)) 169 name = NAME_FOR_NO_SYSTEM; 170 } 171 172 public String getName() { 173 return name; 174 } 175 } 176 177 private class CacheEntry { 178 private String request; 179 private boolean persistent; 180 private ValidationResult v; 181 private ValueSetExpansionOutcome e; 182 } 183 184 private class NamedCache { 185 private String name; 186 private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries 187 private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>(); 188 } 189 190 private Object lock; 191 private String folder; 192 private Map<String, NamedCache> caches = new HashMap<String, NamedCache>(); 193 194 // use lock from the context 195 public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException { 196 super(); 197 this.lock = lock; 198 this.folder = folder; 199 if (folder != null) 200 load(); 201 } 202 203 public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) { 204 CacheToken ct = new CacheToken(); 205 if (code.hasSystem()) 206 ct.name = getSystemNameKeyGenerator().getNameForSystem(code.getSystem()); 207 else 208 ct.name = NAME_FOR_NO_SYSTEM; 209 JsonParser json = new JsonParser(); 210 json.setOutputStyle(OutputStyle.PRETTY); 211 ValueSet vsc = getVSEssense(vs); 212 try { 213 ct.request = "{\"code\" : " + json.composeString(code, "code") + ", \"valueSet\" :" 214 + (vsc == null ? "null" : json.composeString(vsc)) + (options == null ? "" : ", " + options.toJson()) + "}"; 215 } catch (IOException e) { 216 throw new Error(e); 217 } 218 ct.key = String.valueOf(hashNWS(ct.request)); 219 return ct; 220 } 221 222 public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) { 223 CacheToken ct = new CacheToken(); 224 for (Coding c : code.getCoding()) { 225 if (c.hasSystem()) 226 ct.setName(c.getSystem()); 227 } 228 JsonParser json = new JsonParser(); 229 json.setOutputStyle(OutputStyle.PRETTY); 230 ValueSet vsc = getVSEssense(vs); 231 try { 232 ct.request = "{\"code\" : " + json.composeString(code, "codeableConcept") + ", \"valueSet\" :" 233 + json.composeString(vsc) + (options == null ? "" : ", " + options.toJson()) + "}"; 234 } catch (IOException e) { 235 throw new Error(e); 236 } 237 ct.key = String.valueOf(hashNWS(ct.request)); 238 return ct; 239 } 240 241 public ValueSet getVSEssense(ValueSet vs) { 242 if (vs == null) 243 return null; 244 ValueSet vsc = new ValueSet(); 245 vsc.setCompose(vs.getCompose()); 246 if (vs.hasExpansion()) { 247 vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter()); 248 vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains()); 249 } 250 return vsc; 251 } 252 253 public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) { 254 CacheToken ct = new CacheToken(); 255 ValueSet vsc = getVSEssense(vs); 256 for (ConceptSetComponent inc : vs.getCompose().getInclude()) 257 if (inc.hasSystem()) 258 ct.setName(inc.getSystem()); 259 for (ConceptSetComponent inc : vs.getCompose().getExclude()) 260 if (inc.hasSystem()) 261 ct.setName(inc.getSystem()); 262 for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains()) 263 if (inc.hasSystem()) 264 ct.setName(inc.getSystem()); 265 JsonParser json = new JsonParser(); 266 json.setOutputStyle(OutputStyle.PRETTY); 267 try { 268 ct.request = "{\"hierarchical\" : " + (heirarchical ? "true" : "false") + ", \"valueSet\" :" 269 + json.composeString(vsc) + "}\r\n"; 270 } catch (IOException e) { 271 throw new Error(e); 272 } 273 ct.key = String.valueOf(hashNWS(ct.request)); 274 return ct; 275 } 276 277 public NamedCache getNamedCache(CacheToken cacheToken) { 278 NamedCache nc = caches.get(cacheToken.name); 279 if (nc == null) { 280 nc = new NamedCache(); 281 nc.name = cacheToken.name; 282 caches.put(nc.name, nc); 283 } 284 return nc; 285 } 286 287 public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) { 288 synchronized (lock) { 289 NamedCache nc = getNamedCache(cacheToken); 290 CacheEntry e = nc.map.get(cacheToken.key); 291 if (e == null) 292 return null; 293 else 294 return e.e; 295 } 296 } 297 298 public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) { 299 synchronized (lock) { 300 NamedCache nc = getNamedCache(cacheToken); 301 CacheEntry e = new CacheEntry(); 302 e.request = cacheToken.request; 303 e.persistent = persistent; 304 e.e = res; 305 store(cacheToken, persistent, nc, e); 306 } 307 } 308 309 public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) { 310 boolean n = nc.map.containsKey(cacheToken.key); 311 nc.map.put(cacheToken.key, e); 312 if (persistent) { 313 if (n) { 314 for (int i = nc.list.size() - 1; i >= 0; i--) { 315 if (nc.list.get(i).request.equals(e.request)) { 316 nc.list.remove(i); 317 } 318 } 319 } 320 nc.list.add(e); 321 save(nc); 322 } 323 } 324 325 public ValidationResult getValidation(CacheToken cacheToken) { 326 synchronized (lock) { 327 NamedCache nc = getNamedCache(cacheToken); 328 CacheEntry e = nc.map.get(cacheToken.key); 329 if (e == null) 330 return null; 331 else 332 return e.v; 333 } 334 } 335 336 public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) { 337 synchronized (lock) { 338 NamedCache nc = getNamedCache(cacheToken); 339 CacheEntry e = new CacheEntry(); 340 e.request = cacheToken.request; 341 e.persistent = persistent; 342 e.v = res; 343 store(cacheToken, persistent, nc, e); 344 } 345 } 346 347 // persistence 348 349 public void save() { 350 351 } 352 353 private void save(NamedCache nc) { 354 if (folder == null) 355 return; 356 357 try { 358 OutputStreamWriter sw = new OutputStreamWriter(ManagedFileAccess.outStream(Utilities.path(folder, nc.name + ".cache")), 359 "UTF-8"); 360 sw.write(ENTRY_MARKER + "\r\n"); 361 JsonParser json = new JsonParser(); 362 json.setOutputStyle(OutputStyle.PRETTY); 363 for (CacheEntry ce : nc.list) { 364 sw.write(ce.request.trim()); 365 sw.write(BREAK + "\r\n"); 366 if (ce.e != null) { 367 sw.write("e: {\r\n"); 368 if (ce.e.getValueset() != null) 369 sw.write(" \"valueSet\" : " + json.composeString(ce.e.getValueset()).trim() + ",\r\n"); 370 sw.write(" \"error\" : \"" + Utilities.escapeJson(ce.e.getError()).trim() + "\"\r\n}\r\n"); 371 } else { 372 sw.write("v: {\r\n"); 373 sw.write(" \"display\" : \"" + Utilities.escapeJson(ce.v.getDisplay()).trim() + "\",\r\n"); 374 sw.write(" \"severity\" : " 375 + (ce.v.getSeverity() == null ? "null" : "\"" + ce.v.getSeverity().toCode().trim() + "\"") + ",\r\n"); 376 sw.write(" \"error\" : \"" + Utilities.escapeJson(ce.v.getMessage()).trim() + "\"\r\n}\r\n"); 377 } 378 sw.write(ENTRY_MARKER + "\r\n"); 379 } 380 sw.close(); 381 } catch (Exception e) { 382 System.out.println("error saving " + nc.name + ": " + e.getMessage()); 383 } 384 } 385 386 private void load() throws FHIRException, IOException { 387 for (String fn : ManagedFileAccess.file(folder).list()) { 388 if (fn.endsWith(".cache") && !fn.equals("validation.cache")) { 389 try { 390 // System.out.println("Load "+fn); 391 String title = fn.substring(0, fn.lastIndexOf(".")); 392 NamedCache nc = new NamedCache(); 393 nc.name = title; 394 caches.put(title, nc); 395 System.out.print(" - load " + title + ".cache"); 396 String src = FileUtilities.fileToString(Utilities.path(folder, fn)); 397 if (src.startsWith("?")) 398 src = src.substring(1); 399 int i = src.indexOf(ENTRY_MARKER); 400 while (i > -1) { 401 String s = src.substring(0, i); 402 System.out.print("."); 403 src = src.substring(i + ENTRY_MARKER.length() + 1); 404 i = src.indexOf(ENTRY_MARKER); 405 if (!Utilities.noString(s)) { 406 int j = s.indexOf(BREAK); 407 String q = s.substring(0, j); 408 String p = s.substring(j + BREAK.length() + 1).trim(); 409 CacheEntry ce = new CacheEntry(); 410 ce.persistent = true; 411 ce.request = q; 412 boolean e = p.charAt(0) == 'e'; 413 p = p.substring(3); 414 JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p); 415 String error = loadJS(o.get("error")); 416 if (e) { 417 if (o.has("valueSet")) 418 ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), 419 error, TerminologyServiceErrorClass.UNKNOWN); 420 else 421 ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN); 422 } else { 423 IssueSeverity severity = o.get("severity") instanceof JsonNull ? null 424 : IssueSeverity.fromCode(o.get("severity").getAsString()); 425 String display = loadJS(o.get("display")); 426 ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display)); 427 } 428 nc.map.put(String.valueOf(hashNWS(ce.request)), ce); 429 nc.list.add(ce); 430 } 431 } 432 System.out.println("done"); 433 } catch (Exception e) { 434 throw new FHIRException("Error loading " + fn + ": " + e.getMessage(), e); 435 } 436 } 437 } 438 } 439 440 private String loadJS(JsonElement e) { 441 if (e == null) 442 return null; 443 if (!(e instanceof JsonPrimitive)) 444 return null; 445 String s = e.getAsString(); 446 if ("".equals(s)) 447 return null; 448 return s; 449 } 450 451 private String hashNWS(String s) { 452 s = StringUtils.remove(s, ' '); 453 s = StringUtils.remove(s, '\n'); 454 s = StringUtils.remove(s, '\r'); 455 return String.valueOf(s.hashCode()); 456 } 457 458 // management 459 460 public TerminologyCache copy() { 461 // TODO Auto-generated method stub 462 return null; 463 } 464 465 public String summary(ValueSet vs) { 466 if (vs == null) 467 return "null"; 468 469 CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); 470 for (ConceptSetComponent cc : vs.getCompose().getInclude()) 471 b.append("Include " + getIncSummary(cc)); 472 for (ConceptSetComponent cc : vs.getCompose().getExclude()) 473 b.append("Exclude " + getIncSummary(cc)); 474 return b.toString(); 475 } 476 477 private String getIncSummary(ConceptSetComponent cc) { 478 CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder(); 479 for (UriType vs : cc.getValueSet()) 480 b.append(vs.asStringValue()); 481 String vsd = b.length() > 0 ? " where the codes are in the value sets (" + b.toString() + ")" : ""; 482 String system = cc.getSystem(); 483 if (cc.hasConcept()) 484 return Integer.toString(cc.getConcept().size()) + " codes from " + system + vsd; 485 if (cc.hasFilter()) { 486 String s = ""; 487 for (ConceptSetFilterComponent f : cc.getFilter()) { 488 if (!Utilities.noString(s)) 489 s = s + " & "; 490 s = s + f.getProperty() + " " + f.getOp().toCode() + " " + f.getValue(); 491 } 492 return "from " + system + " where " + s + vsd; 493 } 494 return "All codes from " + system + vsd; 495 } 496 497 public String summary(Coding code) { 498 return code.getSystem() + "#" + code.getCode() + ": \"" + code.getDisplay() + "\""; 499 } 500 501 public String summary(CodeableConcept code) { 502 StringBuilder b = new StringBuilder(); 503 b.append("{"); 504 boolean first = true; 505 for (Coding c : code.getCoding()) { 506 if (first) 507 first = false; 508 else 509 b.append(","); 510 b.append(summary(c)); 511 } 512 b.append("}: \""); 513 b.append(code.getText()); 514 b.append("\""); 515 return b.toString(); 516 } 517 518}