Source code

001package org.hl7.fhir.convertors.misc;
002
003import java.io.File;
004import java.io.FileInputStream;
005import java.io.FileNotFoundException;
006import java.io.FileOutputStream;
007import java.io.IOException;
008import java.net.URISyntaxException;
009import java.text.ParseException;
010import java.time.Duration;
011import java.util.ArrayList;
012import java.util.Collections;
013import java.util.HashMap;
014import java.util.List;
015import java.util.Map;
016
017import org.hl7.fhir.exceptions.FHIRException;
018import org.hl7.fhir.r4.formats.IParser.OutputStyle;
019import org.hl7.fhir.r4.formats.JsonParser;
020import org.hl7.fhir.r4.model.CapabilityStatement;
021import org.hl7.fhir.r4.model.CodeSystem;
022import org.hl7.fhir.r4.model.IntegerType;
023import org.hl7.fhir.r4.model.OperationOutcome;
024import org.hl7.fhir.r4.model.OperationOutcome.IssueSeverity;
025import org.hl7.fhir.r4.model.OperationOutcome.IssueType;
026import org.hl7.fhir.r4.model.Parameters;
027import org.hl7.fhir.r4.model.UriType;
028import org.hl7.fhir.r4.model.ValueSet;
029import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionComponent;
030import org.hl7.fhir.r4.utils.client.FHIRToolingClient;
031import org.hl7.fhir.r4.terminologies.JurisdictionUtilities;
032import org.hl7.fhir.utilities.CSVReader;
033import org.hl7.fhir.utilities.Utilities;
034import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
035import org.hl7.fhir.utilities.json.model.JsonArray;
036import org.hl7.fhir.utilities.json.model.JsonObject;
037import org.hl7.fhir.utilities.json.model.JsonProperty;
038
039public class VSACImporter extends OIDBasedValueSetImporter {
040  
041  public VSACImporter() throws FHIRException, IOException {
042    super();
043    init();
044  }
045
046  public static void main(String[] args) throws FHIRException, IOException, ParseException, URISyntaxException {
047    VSACImporter self = new VSACImporter();
048    self.process(args[0], args[1], args[2], "true".equals(args[3]), "true".equals(args[4]));
049  }
050
051  private void process(String source, String dest, String apiKey, boolean onlyNew, boolean onlyActive) throws FHIRException, IOException, URISyntaxException {
052    CSVReader csv = new CSVReader(ManagedFileAccess.inStream(source));
053    csv.readHeaders();
054    Map<String, String> errs = new HashMap<>();
055
056    FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac");
057    fhirToolingClient.setUsername("apikey");
058    fhirToolingClient.setPassword(apiKey);
059    fhirToolingClient.setTimeoutNormal(30000);
060    fhirToolingClient.setTimeoutExpand(30000);
061
062    CapabilityStatement cs = fhirToolingClient.getCapabilitiesStatement();
063    JsonParser json = new JsonParser();
064    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path("[tmp]", "vsac-capability-statement.json")), cs);
065
066    System.out.println("CodeSystems");
067    CodeSystem css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCNHSN");
068    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCNHSN.json")), css);
069    css = fhirToolingClient.fetchResource(CodeSystem.class, "CDCREC");
070    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-CDCREC.json")), css);
071    css = fhirToolingClient.fetchResource(CodeSystem.class, "HSLOC");
072    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-HSLOC.json")), css);
073    css = fhirToolingClient.fetchResource(CodeSystem.class, "SOP");
074    json.setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "CodeSystem-SOP.json")), css);
075    
076    System.out.println("Loading");
077    List<String> oids = new ArrayList<>();
078    List<String> allOids = new ArrayList<>();
079    while (csv.line()) {
080      String status = csv.cell("Expansion Status");
081      if (!onlyActive || "Active".equals(status)) {
082        String oid = csv.cell("OID");
083        allOids.add(oid);
084        if (!onlyNew || !(ManagedFileAccess.file(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) {
085          oids.add(oid);
086        }
087      }
088    }
089    Collections.sort(oids);
090    System.out.println("Cleaning");
091    cleanValueSets(allOids, dest);
092    System.out.println("Go: "+oids.size()+" oids");
093    int i = 0;
094    int j = 0;
095    long t = System.currentTimeMillis();
096    long tt = System.currentTimeMillis();
097    for (String oid : oids) {
098      try {
099        long t3 = System.currentTimeMillis();
100        if (processOid(dest, onlyNew, errs, fhirToolingClient, oid.trim())) {
101          j++;
102        }
103        i++;
104        System.out.print(":"+((System.currentTimeMillis() - t3) / 1000));
105        if (i % 100 == 0) {
106          long elapsed = System.currentTimeMillis() - t;
107          System.out.println("");
108          System.out.println(i+": "+j+" ("+((j * 100) / i)+"%) @ "+Utilities.describeDuration(elapsed)
109              +", "+(elapsed/100000)+"sec/vs, estimated "+Utilities.describeDuration(estimate(i, oids.size(), tt))+" remaining");
110          t = System.currentTimeMillis();
111        }
112      } catch (Exception e) {
113        e.printStackTrace();
114        System.out.println("Unable to fetch OID " + oid + ": " + e.getMessage());
115        errs.put(oid, e.getMessage());
116      }
117    }
118    
119    OperationOutcome oo = new OperationOutcome();
120    for (String oid : errs.keySet()) {
121      oo.addIssue().setSeverity(IssueSeverity.ERROR).setCode(IssueType.EXCEPTION).setDiagnostics(errs.get(oid)).addLocation(oid);
122    }
123    new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(ManagedFileAccess.outStream(Utilities.path(dest, "other", "OperationOutcome-vsac-errors.json")), oo);
124    System.out.println("Done. " + i + " ValueSets in "+Utilities.describeDuration(System.currentTimeMillis() - tt));
125  }
126
127  private void cleanValueSets(List<String> allOids, String dest) throws IOException {
128    cleanValueSets(allOids, new File(Utilities.path(dest)));
129  }
130
131  private void cleanValueSets(List<String> allOids, File file) {
132    for (File f : file.listFiles()) {
133      if (f.getName().startsWith("ValueSet-")) {
134        String oid = f.getName().substring(9).replace(".json", "");
135        if (!allOids.contains(oid)) {
136          f.delete();
137        }
138      }
139    }
140    
141  }
142
143  private long estimate(int i, int size, long tt) {
144    long elapsed = System.currentTimeMillis() - tt;
145    long average = elapsed / i;
146    return (size - i) * average;
147  }
148
149  private boolean processOid(String dest, boolean onlyNew, Map<String, String> errs, FHIRToolingClient fhirToolingClient, String oid)
150      throws IOException, InterruptedException, FileNotFoundException {
151    
152    while (true) {
153      boolean ok = true;
154      long t = System.currentTimeMillis();
155      ValueSet vs = null;
156      try {
157        vs = fhirToolingClient.read(ValueSet.class, oid);
158      } catch (Exception e) {
159        if (e.getMessage().contains("timed out")) {
160          ok = false;
161        } else {
162          errs.put(oid, "Read: " +e.getMessage());
163          System.out.println("Read "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
164          return false;
165        }
166      }
167      if (ok) {
168        t = System.currentTimeMillis();
169        try {
170          Parameters p = new Parameters();
171          p.addParameter("url", new UriType(vs.getUrl()));
172          ValueSet vse = fhirToolingClient.expandValueset(null, p);
173          vs.setExpansion(vse.getExpansion());
174        } catch (Exception e) {
175          if (e.getMessage().contains("timed out")) {
176            ok = false;
177          } else {
178            errs.put(oid, "Expansion: " +e.getMessage());
179            System.out.println("Expand "+oid+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e.getMessage());
180            return false;
181          }
182        }
183      }
184      if (ok) {
185        while (isIncomplete(vs.getExpansion())) {
186          Parameters p = new Parameters();
187          int offset = vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue();
188          p.addParameter("offset", offset);
189          p.addParameter("url", new UriType(vs.getUrl()));
190          t = System.currentTimeMillis();
191          try {
192            ValueSet vse = fhirToolingClient.expandValueset(null, p);    
193            vs.getExpansion().getContains().addAll(vse.getExpansion().getContains());
194            vs.getExpansion().setParameter(vse.getExpansion().getParameter());
195          } catch (Exception e2) {
196            if (e2.getMessage().contains("timed out")) {
197              ok = false;
198              break;
199            } else {
200              errs.put(oid, "Expansion: " +e2.getMessage()+" @ "+offset);
201              System.out.println("Expand "+oid+" @ "+offset+" failed @ "+Utilities.describeDuration(System.currentTimeMillis()-t)+"ms: "+e2.getMessage());
202              return false;
203            }
204          } 
205        }
206      }
207      if (ok) {
208        vs.getExpansion().setOffsetElement(null);
209        vs.getExpansion().getParameter().clear();
210
211        if (vs.hasTitle()) {
212          if (vs.getTitle().equals(vs.getDescription())) {
213            vs.setTitle(vs.getName());              
214          } else {
215            //              System.out.println(oid);
216            //              System.out.println("  name: "+vs.getName());
217            //              System.out.println("  title: "+vs.getTitle());
218            //              System.out.println("  desc: "+vs.getDescription());
219          }
220        } else {
221          vs.setTitle(vs.getName());
222        }
223        if (vs.getUrl().startsWith("https://")) {
224          System.out.println("URL is https: "+vs.getUrl());
225        }
226        vs.setName(makeValidName(vs.getName()));
227        JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US");
228        new JsonParser().setOutputStyle(OutputStyle.NORMAL).compose(ManagedFileAccess.outStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
229        return true;
230      }
231    }
232  }
233
234  private boolean isIncomplete(ValueSetExpansionComponent expansion) {
235    IntegerType c = expansion.getParameter("count") != null ? expansion.getParameter("count").getValueIntegerType() : new IntegerType(0);
236    IntegerType offset = expansion.getParameter("offset") != null ? expansion.getParameter("offset").getValueIntegerType() : new IntegerType(0);
237    return c.getValue() + offset.getValue() < expansion.getTotal();
238  }
239
240  private String makeValidName(String name) {
241    StringBuilder b = new StringBuilder();
242    boolean upper = true;
243    for (char ch : name.toCharArray()) {
244      if (ch == ' ') {
245        upper = true;
246      } else if (Character.isAlphabetic(ch)) {
247        if (upper) {
248          b.append(Character.toUpperCase(ch));
249        } else {
250          b.append(ch);
251        }
252        upper = false;
253      } else if (Character.isDigit(ch)) {
254        if (b.length() == 0) {
255          b.append('N');
256        }
257        b.append(ch);
258      } else if (ch == '_' && b.length() != 0) {
259        b.append(ch);
260      } else {
261        upper = true;
262      }
263    }
264//    System.out.println(b.toString()+" from "+name);
265    return b.toString();
266  }
267}