001package org.hl7.fhir.convertors.analytics;
002
003import java.io.IOException;
004import java.util.ArrayList;
005import java.util.HashMap;
006import java.util.HashSet;
007import java.util.List;
008import java.util.Map;
009import java.util.Set;
010
011import javax.xml.parsers.ParserConfigurationException;
012
013import org.hl7.fhir.exceptions.FHIRException;
014import org.hl7.fhir.r5.utils.EOperationOutcome;
015import org.hl7.fhir.utilities.SimpleHTTPClient;
016import org.hl7.fhir.utilities.SimpleHTTPClient.HTTPResult;
017import org.hl7.fhir.utilities.TextFile;
018import org.hl7.fhir.utilities.Utilities;
019import org.hl7.fhir.utilities.json.model.JsonArray;
020import org.hl7.fhir.utilities.json.model.JsonObject;
021import org.hl7.fhir.utilities.json.parser.JsonParser;
022import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
023import org.hl7.fhir.utilities.npm.NpmPackage;
024import org.hl7.fhir.utilities.npm.PackageClient;
025import org.hl7.fhir.utilities.npm.PackageInfo;
026import org.hl7.fhir.utilities.npm.PackageServer;
027import org.hl7.fhir.utilities.xml.XMLUtil;
028import org.w3c.dom.Document;
029import org.w3c.dom.Element;
030import org.xml.sax.SAXException;
031
032public class PackageVisitor {
033  
034  public interface IPackageVisitorProcessor {
035     public void processResource(String pid, NpmPackage npm, String version, String type, String id, byte[] content) throws FHIRException, IOException, EOperationOutcome;
036  }
037
038  private List<String> resourceTypes = new ArrayList<>();
039  private List<String> versions = new ArrayList<>();
040  private boolean corePackages;
041  private boolean oldVersions;
042  private boolean current;
043  private IPackageVisitorProcessor processor;
044  private FilesystemPackageCacheManager pcm;
045  private PackageClient pc;  
046  
047  public List<String> getResourceTypes() {
048    return resourceTypes;
049  }
050
051  public void setResourceTypes(List<String> resourceTypes) {
052    this.resourceTypes = resourceTypes;
053  }
054
055  public List<String> getVersions() {
056    return versions;
057  }
058
059  public void setVersions(List<String> versions) {
060    this.versions = versions;
061  }
062
063
064  public boolean isCurrent() {
065    return current;
066  }
067
068  public void setCurrent(boolean current) {
069    this.current = current;
070  }
071
072  public boolean isCorePackages() {
073    return corePackages;
074  }
075
076
077
078
079  public void setCorePackages(boolean corePackages) {
080    this.corePackages = corePackages;
081  }
082
083
084
085
086  public boolean isOldVersions() {
087    return oldVersions;
088  }
089
090
091
092
093  public void setOldVersions(boolean oldVersions) {
094    this.oldVersions = oldVersions;
095  }
096
097
098
099
100  public IPackageVisitorProcessor getProcessor() {
101    return processor;
102  }
103
104  public void setProcessor(IPackageVisitorProcessor processor) {
105    this.processor = processor;
106  }
107
108  public void visitPackages() throws IOException, ParserConfigurationException, SAXException {
109    System.out.println("Finding packages");
110    pc = new PackageClient(PackageServer.primaryServer());
111    pcm = new FilesystemPackageCacheManager(org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager.FilesystemPackageCacheMode.USER);
112    
113    Map<String, String> cpidMap = getAllCIPackages();
114    Set<String> cpidSet = new HashSet<>();
115    System.out.println("Go: "+cpidMap.size()+" current packages");
116    int i = 0;
117    for (String s : cpidMap.keySet()) {
118      processCurrentPackage(s, cpidMap.get(s), cpidSet, i, cpidMap.size()); 
119      i++;
120    }
121    Set<String> pidList = getAllPackages();
122    System.out.println("Go: "+pidList.size()+" published packages");
123    i = 0;
124    for (String pid : pidList) {    
125      if (!cpidSet.contains(pid)) {
126        cpidSet.add(pid);
127        List<String> vList = listVersions(pid);
128        if (oldVersions) {
129          for (String v : vList) {
130            processPackage(pid, v, i, pidList.size());          
131          }
132        } else if (vList.isEmpty()) {
133          System.out.println("No Packages for "+pid);
134        } else {
135          processPackage(pid, vList.get(vList.size() - 1), i, pidList.size());
136        }
137      }
138      i++;
139    }    
140    JsonObject json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json");
141    i = 0;
142    List<JsonObject> objects = json.getJsonObjects("guides");
143    for (JsonObject o : objects) {
144      String pid = o.asString("npm-name");
145      if (pid != null && !cpidSet.contains(pid)) {
146        cpidSet.add(pid);
147        List<String> vList = listVersions(pid);
148        if (oldVersions) {
149          for (String v : vList) {
150            processPackage(pid, v, i, objects.size());          
151          }
152        } else if (vList.isEmpty()) {
153          System.out.println("No Packages for "+pid);
154        } else {
155          processPackage(pid, vList.get(vList.size() - 1), i, objects.size());
156        }
157      }
158      i++;
159    }
160  }
161
162  private void processCurrentPackage(String url, String pid, Set<String> cpidSet, int i, int t) {
163    try {
164      String[] p = url.split("\\/");
165      String repo = "https://build.fhir.org/ig/"+p[0]+"/"+p[1];
166      NpmPackage npm = NpmPackage.fromUrl(repo+"/package.tgz");
167      String fv = npm.fhirVersion();
168      cpidSet.add(pid);
169      
170      if (corePackages || !corePackage(npm)) {
171        int c = 0;
172        if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
173          for (String type : resourceTypes) {
174            for (String s : npm.listResources(type)) {
175              c++;
176              try {
177                processor.processResource(pid+"#current", npm, fv, type, s, TextFile.streamToBytes(npm.load("package", s)));
178              } catch (Exception e) {
179                System.out.println("####### Error loading "+pid+"#current["+fv+"]/"+type+" ####### "+e.getMessage());
180//                e.printStackTrace();
181              }
182            }
183          }
184        }    
185        System.out.println("Processed: "+pid+"#current: "+c+" resources ("+i+" of "+t+")");  
186      }
187    } catch (Exception e) {      
188      System.out.println("Unable to process: "+pid+"#current: "+e.getMessage());      
189    }
190  }
191
192  private Map<String, String> getAllCIPackages() throws IOException {
193    Map<String, String> res = new HashMap<>();
194    if (current) {
195      JsonArray json = (JsonArray) JsonParser.parseFromUrl("https://build.fhir.org/ig/qas.json");
196      for (JsonObject o  : json.asJsonObjects()) {
197        String url = o.asString("repo");
198        res.put(url, o.asString("package-id"));
199      }
200    }
201    return res;
202  }
203
204  private List<String> listVersions(String pid) throws IOException {
205    List<String> list = new ArrayList<>();
206    if (pid !=null) {
207      for (PackageInfo i : pc.getVersions(pid)) {
208        list.add(i.getVersion());
209      }    
210    }
211    return list;
212  }
213
214  private Set<String> getAllPackages() throws IOException, ParserConfigurationException, SAXException {
215    Set<String> list = new HashSet<>();
216    for (PackageInfo i : pc.search(null, null, null, false)) {
217      list.add(i.getId());
218    }    
219    JsonObject json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json");
220    for (JsonObject ig : json.getJsonObjects("guides")) {
221      list.add(ig.asString("npm-name"));
222    }
223    json = JsonParser.parseObjectFromUrl("https://raw.githubusercontent.com/FHIR/ig-registry/master/package-feeds.json");
224    for (JsonObject feed : json.getJsonObjects("feeds")) {
225      processFeed(list, feed.asString("url"));
226    }
227    
228    return list;
229  }
230
231  private void processFeed(Set<String> list, String str) throws IOException, ParserConfigurationException, SAXException {
232    System.out.println("Feed "+str);
233    try {
234      SimpleHTTPClient fetcher = new SimpleHTTPClient();
235      HTTPResult res = fetcher.get(str+"?nocache=" + System.currentTimeMillis());
236      res.checkThrowException();
237      Document xml = XMLUtil.parseToDom(res.getContent());
238      for (Element channel : XMLUtil.getNamedChildren(xml.getDocumentElement(), "channel")) {
239        for (Element item : XMLUtil.getNamedChildren(channel, "item")) {
240          String pid = XMLUtil.getNamedChildText(item, "title");
241          if (pid.contains("#")) {
242            list.add(pid.substring(0, pid.indexOf("#")));
243          }
244        }
245      }
246    } catch (Exception e) {
247      System.out.println("   "+e.getMessage());
248    }
249  }
250
251
252  private void processPackage(String pid, String v, int i, int t) throws IOException {
253    NpmPackage npm = null;
254    String fv = null;
255    try {
256      npm = pcm.loadPackage(pid, v);
257      fv = npm.fhirVersion();
258    } catch (Throwable e) {
259      System.out.println("Unable to process: "+pid+"#"+v+": "+e.getMessage());      
260    }
261    if (corePackages || !corePackage(npm)) {
262      int c = 0;
263      if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
264        for (String type : resourceTypes) {
265          for (String s : npm.listResources(type)) {
266            c++;
267            try {
268              processor.processResource(pid+"#"+v, npm, fv, type, s, TextFile.streamToBytes(npm.load("package", s)));
269            } catch (Exception e) {
270              System.out.println("####### Error loading "+pid+"#"+v +"["+fv+"]/"+type+" ####### "+e.getMessage());
271              e.printStackTrace();
272            }
273          }
274        }
275      }    
276      System.out.println("Processed: "+pid+"#"+v+": "+c+" resources ("+i+" of "+t+")");  
277    }
278  }
279
280  private boolean corePackage(NpmPackage npm) {
281    return npm != null && !Utilities.noString(npm.name()) && (
282        npm.name().startsWith("hl7.terminology") || 
283        npm.name().startsWith("hl7.fhir.core") || 
284        npm.name().startsWith("hl7.fhir.r2.") || 
285        npm.name().startsWith("hl7.fhir.r2b.") || 
286        npm.name().startsWith("hl7.fhir.r3.") || 
287        npm.name().startsWith("hl7.fhir.r4.") || 
288        npm.name().startsWith("hl7.fhir.r4b.") || 
289        npm.name().startsWith("hl7.fhir.r5."));
290  }
291
292}