
001package org.hl7.fhir.convertors.misc; 002 003import java.io.File; 004import java.io.FileNotFoundException; 005import java.io.IOException; 006import java.net.URISyntaxException; 007import java.text.ParseException; 008import java.util.ArrayList; 009import java.util.Collections; 010import java.util.HashMap; 011import java.util.List; 012import java.util.Map; 013 014import org.hl7.fhir.exceptions.FHIRException; 015import org.hl7.fhir.r4.formats.IParser.OutputStyle; 016import org.hl7.fhir.r4.formats.JsonParser; 017import org.hl7.fhir.r4.model.CapabilityStatement; 018import org.hl7.fhir.r4.model.CodeSystem; 019import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent; 020import org.hl7.fhir.r4.model.IntegerType; 021import org.hl7.fhir.r4.model.OperationOutcome; 022import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity; 023import org.hl7.fhir.r4.model.OperationOutcome.IssueType; 024import org.hl7.fhir.r4.model.Parameters; 025import org.hl7.fhir.r4.model.Parameters.ParametersParameterComponent; 026import org.hl7.fhir.r4.model.UriType; 027import org.hl7.fhir.r4.model.ValueSet; 028import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent; 029import org.hl7.fhir.r4.terminologies.JurisdictionUtilities; 030import org.hl7.fhir.r4.utils.client.FHIRToolingClient; 031import org.hl7.fhir.utilities.CSVReader; 032import org.hl7.fhir.utilities.IniFile; 033import org.hl7.fhir.utilities.Utilities; 034import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 035import org.hl7.fhir.utilities.http.ManagedWebAccess; 036 037@SuppressWarnings("checkstyle:systemout") 038public class VSACImporter extends OIDBasedValueSetImporter { 039 040 public VSACImporter() throws FHIRException, IOException { 041 super(); 042 init(); 043 } 044 045 public static void main(String[] args) throws FHIRException, IOException, ParseException, URISyntaxException { 046 VSACImporter self = new VSACImporter(); 047 self.process(args[0], args[1], "true".equals(args[2]), "true".equals(args[3])); 048 } 049 050 private void process(String source, String dest, boolean onlyNew, boolean onlyActive) throws FHIRException, IOException, URISyntaxException { 051 CSVReader csv = new CSVReader(ManagedFileAccess.inStream(source)); 052 csv.readHeaders(); 053 Map<String, String> errs = new HashMap<>(); 054 055 ManagedWebAccess.loadFromFHIRSettings(); 056 FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac"); 057 fhirToolingClient.setTimeoutNormal(30000); 058 fhirToolingClient.setTimeoutExpand(30000); 059 060 CapabilityStatement cs = fhirToolingClient.getCapabilitiesStatement(); 061 JsonParser json = new JsonParser(); 062 json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path("[tmp]", "vsac-capability-statement.json")), cs); 063 064 System.out.println("CodeSystems"); 065 CodeSystem css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCREC"); 066 checkHierarchy(fhirToolingClient, css); 067 json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCREC.json")), css); 068 css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCNHSN"); 069 json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCNHSN.json")), css); 070 css = fhirToolingClient.fetchResource(CodeSystem.class, "HSLOC"); 071 json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-HSLOC.json")), css); 072 css = fhirToolingClient.fetchResource(CodeSystem.class, "SOP"); 073 json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-SOP.json")), css); 074 075 System.out.println("Loading"); 076 List<String> oids = new ArrayList<>(); 077 List<String> allOids = new ArrayList<>(); 078 while (csv.line()) { 079 String status = csv.cell("Expansion Status"); 080 if (!onlyActive || "Active".equals(status)) { 081 String oid = csv.cell("OID"); 082 allOids.add(oid); 083 if (!onlyNew || !(ManagedFileAccess.file(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) { 084 oids.add(oid); 085 } 086 } 087 } 088 Collections.sort(oids); 089 System.out.println("Cleaning"); 090 cleanValueSets(allOids, dest); 091 System.out.println("Go: "+oids.size()+" oids"); 092 int i = 0; 093 int j = 0; 094 long t = System.currentTimeMillis(); 095 long tt = System.currentTimeMillis(); 096 for (String oid : oids) { 097 try { 098 long t3 = System.currentTimeMillis(); 099 if (processOid(dest, onlyNew, errs, fhirToolingClient, oid.trim())) { 100 j++; 101 } 102 i++; 103 System.out.print(":"+((System.currentTimeMillis() - t3) / 1000)); 104 if (i % 100 == 0) { 105 long elapsed = System.currentTimeMillis() - t; 106 System.out.println(""); 107 System.out.println(i+": "+j+" ("+((j * 100) / i)+"%) @ "+Utilities.describeDuration(elapsed) 108 +", "+(elapsed/100000)+"sec/vs, estimated "+Utilities.describeDuration(estimate(i, oids.size(), tt))+" remaining"); 109 t = System.currentTimeMillis(); 110 } 111 } catch (Exception e) { 112 e.printStackTrace(); 113 System.out.println("Unable to fetch OID " + oid + ": " + e.getMessage()); 114 errs.put(oid, e.getMessage()); 115 } 116 } 117 118 OperationOutcome oo = new OperationOutcome(); 119 for (String oid : errs.keySet()) { 120 oo.addIssue().setSeverity(IssueSeverity.ERROR).setCode(IssueType.EXCEPTION).setDiagnostics(errs.get(oid)).addLocation(oid); 121 } 122 new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo); 123 System.out.println(); 124 System.out.println("Done. " + i + " ValueSets in "+Utilities.describeDuration(System.currentTimeMillis() - tt)); 125 } 126 127 private void checkHierarchy(FHIRToolingClient client, CodeSystem css) { 128 IniFile ini = new IniFile("/Users/grahamegrieve/temp/vsac-cdc-rec.ini"); 129 System.out.println("hierarchy:"); 130 List<ConceptDefinitionComponent> codes = new ArrayList<>(); 131 int c = 0; 132 List<ConceptDefinitionComponent> list = new ArrayList<>(); 133 Map<String,ConceptDefinitionComponent> map = new HashMap<>(); 134 for (ConceptDefinitionComponent cc : css.getConcept()) { 135 list.add(cc); 136 map.put(cc.getCode(), cc); 137 } 138 css.getConcept().clear(); 139 for (ConceptDefinitionComponent cc : list) { 140 String code = cc.getCode(); 141 ConceptDefinitionComponent parent = map.get(ini.getStringProperty("parents-"+css.getVersion(), code)); 142 if (parent == null) { 143 parent = findParent(client, css, css.getConcept(), code); 144 if (parent == null) { 145 ini.setStringProperty("parents-"+css.getVersion(), code, "null", null); 146 } else { 147 ini.setStringProperty("parents-"+css.getVersion(), code, parent.getCode(), null); 148 } 149 ini.save(); 150 } 151 if (parent == null) { 152 css.getConcept().add(cc); 153 } else { 154 parent.getConcept().add(cc); 155 } 156 157 codes.add(0, cc); 158 c++; 159 if (c % 20 == 0) { 160 System.out.print("."+c); 161 } 162 } 163 System.out.println("Done. "+c+" concepts"); 164 } 165 166 private ConceptDefinitionComponent findParent(FHIRToolingClient client, CodeSystem css, List<ConceptDefinitionComponent> concepts, String code) { 167 if (concepts.size() == 0) { 168 return null; 169 } 170 ConceptDefinitionComponent last = concepts.get(concepts.size() -1); 171 if (last.hasConcept()) { 172 ConceptDefinitionComponent parent = findParent(client, css, last.getConcept(), code); 173 if (parent != null) { 174 return parent; 175 } 176 } 177 178 String reln = getRelationship(client, css, code, last.getCode()); 179 if ("subsumes".equals(reln)) { 180 return last; 181 } else { 182 for (ConceptDefinitionComponent cc : concepts) { 183 reln = getRelationship(client, css, code, cc.getCode()); 184 if ("subsumes".equals(reln)) { 185 return cc; 186 } 187 } 188 return null; 189 } 190 } 191 192 193 private String getRelationship(FHIRToolingClient client, CodeSystem css, String codeA, String codeB) { 194 Map<String, String> params = new HashMap<>(); 195 params.put("system", css.getUrl()); 196 params.put("codeA", codeB); 197 params.put("codeB", codeA); 198 Parameters p = client.subsumes(params); 199 for (ParametersParameterComponent pp : p.getParameter()) { 200 if ("outcome".equals(pp.getName())) { 201 return pp.getValue().primitiveValue(); 202 } 203 } 204 return null; 205 } 206 207 private void cleanValueSets(List<String> allOids, String dest) throws IOException { 208 cleanValueSets(allOids, ManagedFileAccess.file(Utilities.path(dest))); 209 } 210 211 private void cleanValueSets(List<String> allOids, File file) { 212 for (File f : file.listFiles()) { 213 if (f.getName().startsWith("ValueSet-")) { 214 String oid = f.getName().substring(9).replace(".json", ""); 215 if (!allOids.contains(oid)) { 216 f.delete(); 217 } 218 } 219 } 220 221 } 222 223 private long estimate(int i, int size, long tt) { 224 long elapsed = System.currentTimeMillis() - tt; 225 long average = elapsed / i; 226 return (size - i) * average; 227 } 228 229 private boolean processOid(String dest, boolean onlyNew, Map<String, String> errs, FHIRToolingClient fhirToolingClient, String oid) 230 throws IOException, InterruptedException, FileNotFoundException { 231 232 while (true) { 233 boolean ok = true; 234 long t = System.currentTimeMillis(); 235 ValueSet vs = null; 236 try { 237 vs = fhirToolingClient.read(ValueSet.class, oid); 238 } catch (Exception e) { 239 if (e.getMessage().contains("timed out")) { 240 ok = false; 241 } else { 242 errs.put(oid, "Read: " +e.getMessage()); 243 System.out.println("Read "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage()); 244 return false; 245 } 246 } 247 if (ok) { 248 t = System.currentTimeMillis(); 249 try { 250 Parameters p = new Parameters(); 251 p.addParameter("url", new UriType(vs.getUrl())); 252 ValueSet vse = fhirToolingClient.expandValueset(null, p); 253 vs.setExpansion(vse.getExpansion()); 254 } catch (Exception e) { 255 if (e.getMessage().contains("timed out")) { 256 ok = false; 257 } else { 258 errs.put(oid, "Expansion: " +e.getMessage()); 259 System.out.println("Expand "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage()); 260 return false; 261 } 262 } 263 } 264 if (ok) { 265 while (isIncomplete(vs.getExpansion())) { 266 Parameters p = new Parameters(); 267 int offset = vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue(); 268 p.addParameter("offset", offset); 269 p.addParameter("url", new UriType(vs.getUrl())); 270 t = System.currentTimeMillis(); 271 try { 272 ValueSet vse = fhirToolingClient.expandValueset(null, p); 273 vs.getExpansion().getContains().addAll(vse.getExpansion().getContains()); 274 vs.getExpansion().setParameter(vse.getExpansion().getParameter()); 275 } catch (Exception e2) { 276 if (e2.getMessage().contains("timed out")) { 277 ok = false; 278 break; 279 } else { 280 errs.put(oid, "Expansion: " +e2.getMessage()+" @ "+offset); 281 System.out.println("Expand "+oid+" @ "+offset+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e2.getMessage()); 282 return false; 283 } 284 } 285 } 286 } 287 if (ok) { 288 vs.getExpansion().setOffsetElement(null); 289 vs.getExpansion().getParameter().clear(); 290 291 if (vs.hasTitle()) { 292 if (vs.getTitle().equals(vs.getDescription())) { 293 vs.setTitle(vs.getName()); 294 } else { 295 // System.out.println(oid); 296 // System.out.println(" name: "+vs.getName()); 297 // System.out.println(" title: "+vs.getTitle()); 298 // System.out.println(" desc: "+vs.getDescription()); 299 } 300 } else { 301 vs.setTitle(vs.getName()); 302 } 303 if (vs.getUrl().startsWith("https://")) { 304 System.out.println("URL is https: "+vs.getUrl()); 305 } 306 vs.setName(makeValidName(vs.getName())); 307 JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US"); 308 new JsonParser().setOutputStyle(OutputStyle.NORMAL).compose(ManagedFileAccess.outStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs); 309 return true; 310 } 311 } 312 } 313 314 private boolean isIncomplete(ValueSetExpansionComponent expansion) { 315 IntegerType c = expansion.getParameter("count") != null ? expansion.getParameter("count").getValueIntegerType() : new IntegerType(0); 316 IntegerType offset = expansion.getParameter("offset") != null ? expansion.getParameter("offset").getValueIntegerType() : new IntegerType(0); 317 return c.getValue() + offset.getValue() < expansion.getTotal(); 318 } 319 320 private String makeValidName(String name) { 321 StringBuilder b = new StringBuilder(); 322 boolean upper = true; 323 for (char ch : name.toCharArray()) { 324 if (ch == ' ') { 325 upper = true; 326 } else if (Character.isAlphabetic(ch)) { 327 if (upper) { 328 b.append(Character.toUpperCase(ch)); 329 } else { 330 b.append(ch); 331 } 332 upper = false; 333 } else if (Character.isDigit(ch)) { 334 if (b.length() == 0) { 335 b.append('N'); 336 } 337 b.append(ch); 338 } else if (ch == '_' && b.length() != 0) { 339 b.append(ch); 340 } else { 341 upper = true; 342 } 343 } 344 // System.out.println(b.toString()+" from "+name); 345 return b.toString(); 346 } 347}