Source code

001package org.hl7.fhir.convertors.analytics;
002
003import java.io.File;
004import java.io.FileInputStream;
005import java.io.IOException;
006import java.util.*;
007
008import javax.xml.parsers.ParserConfigurationException;
009
010import org.hl7.fhir.exceptions.FHIRException;
011import org.hl7.fhir.r5.utils.EOperationOutcome;
012import org.hl7.fhir.utilities.FileUtilities;
013import org.hl7.fhir.utilities.Utilities;
014import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
015import org.hl7.fhir.utilities.http.HTTPResult;
016import org.hl7.fhir.utilities.http.ManagedWebAccess;
017import org.hl7.fhir.utilities.json.model.JsonArray;
018import org.hl7.fhir.utilities.json.model.JsonObject;
019import org.hl7.fhir.utilities.json.parser.JsonParser;
020import org.hl7.fhir.utilities.npm.FilesystemPackageCacheManager;
021import org.hl7.fhir.utilities.npm.NpmPackage;
022import org.hl7.fhir.utilities.npm.NpmPackage.PackagedResourceFile;
023import org.hl7.fhir.utilities.npm.PackageClient;
024import org.hl7.fhir.utilities.npm.PackageInfo;
025import org.hl7.fhir.utilities.npm.PackageServer;
026import org.hl7.fhir.utilities.xml.XMLUtil;
027import org.w3c.dom.Document;
028import org.w3c.dom.Element;
029import org.xml.sax.SAXException;
030
031public class PackageVisitor {
032
033  private PackageServer clientPackageServer = null;
034
035  public void setClientPackageServer(PackageServer packageServer) {
036    this.clientPackageServer = packageServer;
037  }
038  private List<PackageServer> cachePackageServers = null;
039  public void setCachePackageServers(List<PackageServer> packageServers) {
040    this.cachePackageServers = packageServers;
041  }
042
043  public static class PackageContext {
044    private String pid;
045    private NpmPackage npm;
046    private String version;
047    protected PackageContext(String pid, NpmPackage npm, String version) {
048      super();
049      this.pid = pid;
050      this.npm = npm;
051      this.version = version;
052    }
053    public String getPid() {
054      return pid;
055    }
056    public NpmPackage getNpm() {
057      return npm;
058    }
059    public String getVersion() {
060      return version;
061    }
062  }
063  
064  public interface IPackageVisitorProcessor {
065    public Object startPackage(PackageContext context) throws FHIRException, IOException, EOperationOutcome;
066    public void processResource(PackageContext context, Object clientContext, String type, String id, byte[] content) throws FHIRException, IOException, EOperationOutcome;
067    public void finishPackage(PackageContext context) throws FHIRException, IOException, EOperationOutcome;
068
069    public void alreadyVisited(String pid) throws FHIRException, IOException, EOperationOutcome;
070  }
071
072  private Set<String> resourceTypes = new HashSet<>();
073  private List<String> versions = new ArrayList<>();
074  private boolean corePackages;
075  private boolean oldVersions;
076  private boolean current;
077  private IPackageVisitorProcessor processor;
078  private FilesystemPackageCacheManager pcm;
079  private PackageClient pc;
080  private String cache;  
081  private int step;
082
083  public Set<String> getResourceTypes() {
084    return resourceTypes;
085  }
086
087  public void setResourceTypes(Set<String> resourceTypes) {
088    this.resourceTypes = resourceTypes;
089  }
090
091  public void setResourceTypes(String... resourceTypes) {
092    this.resourceTypes = new HashSet<String>();
093    for (String s : resourceTypes) {
094      this.resourceTypes.add(s);
095    }
096  }
097
098  public List<String> getVersions() {
099    return versions;
100  }
101
102  public void setVersions(List<String> versions) {
103    this.versions = versions;
104  }
105
106
107  public boolean isCurrent() {
108    return current;
109  }
110
111  public void setCurrent(boolean current) {
112    this.current = current;
113  }
114
115  public boolean isCorePackages() {
116    return corePackages;
117  }
118
119
120
121
122  public String getCache() {
123    return cache;
124  }
125
126  public void setCache(String cache) {
127    this.cache = cache;
128  }
129
130  public void setCorePackages(boolean corePackages) {
131    this.corePackages = corePackages;
132  }
133
134
135
136
137  public boolean isOldVersions() {
138    return oldVersions;
139  }
140
141
142
143
144  public void setOldVersions(boolean oldVersions) {
145    this.oldVersions = oldVersions;
146  }
147
148
149
150
151  public IPackageVisitorProcessor getProcessor() {
152    return processor;
153  }
154
155  public void setProcessor(IPackageVisitorProcessor processor) {
156    this.processor = processor;
157  }
158
159  public void visitPackages() throws IOException, ParserConfigurationException, SAXException, FHIRException, EOperationOutcome {
160    System.out.println("Finding packages");
161    pc = clientPackageServer == null
162      ? new PackageClient(PackageServer.primaryServer())
163      : new PackageClient(clientPackageServer);
164
165    pcm = cachePackageServers == null
166      ? new FilesystemPackageCacheManager.Builder().build()
167      : new FilesystemPackageCacheManager.Builder().withPackageServers(cachePackageServers).build();
168
169    Set<String> pidList = getAllPackages();
170
171    Map<String, String> cpidMap = getAllCIPackages();
172    Set<String> cpidSet = new HashSet<>();
173    System.out.println("Go: "+cpidMap.size()+" current packages");
174    int i = 0;
175    for (String s : cpidMap.keySet()) {
176      processCurrentPackage(cpidMap.get(s), s, cpidSet, i, cpidMap.size()); 
177      i++;
178    }
179
180    System.out.println("Go: "+pidList.size()+" published packages");
181    i = 0;
182    for (String pid : pidList) {  
183      if (pid != null) {
184        if (!cpidSet.contains(pid)) {
185          cpidSet.add(pid);
186          if (step == 0 || step == 3) {
187            List<String> vList = listVersions(pid);
188            if (oldVersions) {
189              for (String v : vList) {
190                processPackage(pid, v, i, pidList.size());          
191              }
192            } else if (vList.isEmpty()) {
193              System.out.println("No Packages for "+pid);
194            } else {
195              processPackage(pid, vList.get(vList.size() - 1), i, pidList.size());
196            }
197          }
198        } else {
199          processor.alreadyVisited(pid);
200        }
201        i++;
202      }    
203    }
204
205    if (step == 0 || step == 3) {
206      JsonObject json = JsonParser.parseObjectFromUrl("https://fhir.github.io/ig-registry/fhir-ig-list.json");
207      i = 0;
208      List<JsonObject> objects = json.getJsonObjects("guides");
209      for (JsonObject o : objects) {
210        String pid = o.asString("npm-name");
211        if (pid != null && !cpidSet.contains(pid)) {
212          cpidSet.add(pid);
213          List<String> vList = listVersions(pid);
214          if (oldVersions) {
215            for (String v : vList) {
216              processPackage(pid, v, i, objects.size());          
217            }
218          } else if (vList.isEmpty()) {
219            System.out.println("No Packages for "+pid);
220          } else {
221            processPackage(pid, vList.get(vList.size() - 1), i, objects.size());
222          }
223        }
224        i++;
225      }
226    }
227  }
228
229  private void processCurrentPackage(String url, String pid, Set<String> cpidSet, int i, int t) {
230    try {
231      cpidSet.add(pid);
232      if (step == 0 || (step == 1 && i < t/2) || (step == 2 && i >= t/2)) {
233        long ms1 = System.currentTimeMillis();
234        String[] p = url.split("\\/");
235        String repo = "https://build.fhir.org/ig/"+p[0]+"/"+p[1];
236        JsonObject manifest = JsonParser.parseObjectFromUrl(repo+"/package.manifest.json");
237        File co = ManagedFileAccess.file(Utilities.path(cache, pid+"."+manifest.asString("date")+".tgz"));
238        if (!co.exists()) {
239
240          HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), repo+"/package.tgz?nocache=" + System.currentTimeMillis());
241          res.checkThrowException();
242          FileUtilities.bytesToFile(res.getContent(), co);
243        }
244        NpmPackage npm = NpmPackage.fromPackage(ManagedFileAccess.inStream(co));          
245        String fv = npm.fhirVersion();
246        long ms2 = System.currentTimeMillis();
247
248        if (corePackages || !corePackage(npm)) {
249          if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
250            PackageContext ctxt = new PackageContext(pid+"#current", npm, fv);
251            boolean ok = false;
252            Object context = null;
253            try {
254              context = processor.startPackage(ctxt);
255              ok = true;
256            } catch (Exception e) {
257              System.out.println("####### Error loading "+pid+"#current["+fv+"]: ####### "+e.getMessage());
258              //                e.printStackTrace();
259            }
260            if (ok) {
261              int c = 0;
262              for (String type : resourceTypes) {
263                for (String s : npm.listResources(type)) {
264                  c++;
265                  try {
266                    processor.processResource(ctxt, context, type, s, FileUtilities.streamToBytes(npm.load("package", s)));
267                  } catch (Exception e) {
268                    System.out.println("####### Error loading "+pid+"#current["+fv+"]/"+type+" ####### "+e.getMessage());
269                    //                e.printStackTrace();
270                  }
271                }
272              }
273              processor.finishPackage(ctxt);
274              System.out.println("Processed: "+pid+"#current: "+c+" resources ("+i+" of "+t+", "+(ms2-ms1)+"/"+(System.currentTimeMillis()-ms2)+"ms)");
275            }
276          } else {
277            System.out.println("Ignored: "+pid+"#current: no version");            
278          }
279        }
280      }
281    } catch (Exception e) {      
282      System.out.println("Unable to process: "+pid+"#current: "+e.getMessage());      
283    }
284  }
285
286  private Map<String, String> getAllCIPackages() throws IOException {
287    System.out.println("Fetch https://build.fhir.org/ig/qas.json");
288    Map<String, String> res = new HashMap<>();
289    if (current) {
290      JsonArray json = (JsonArray) JsonParser.parseFromUrl("https://build.fhir.org/ig/qas.json");
291      for (JsonObject o  : json.asJsonObjects()) {
292        String url = o.asString("repo");
293        String pid = o.asString("package-id");
294        if (url.contains("/branches/master") || url.contains("/branches/main") ) {
295          if (!res.containsKey(pid)) {
296            res.put(pid, url);
297          } else if (!url.equals(res.get(pid))) {
298            System.out.println("Ignore "+url+" already encountered "+pid +" @ "+res.get(pid));
299          }
300        }
301      }
302    }
303    return res;
304  }
305
306  private List<String> listVersions(String pid) throws IOException {
307    List<String> list = new ArrayList<>();
308    if (pid !=null) {
309      for (PackageInfo i : pc.getVersions(pid)) {
310        list.add(i.getVersion());
311      }    
312    }
313    return list;
314  }
315
316  private Set<String> getAllPackages() throws IOException, ParserConfigurationException, SAXException {
317    Set<String> list = new HashSet<>();
318    for (PackageInfo i : pc.search(null, null, null, false)) {
319      list.add(i.getId());
320    }    
321    JsonObject json = JsonParser.parseObjectFromUrl("https://fhir.github.io/ig-registry/fhir-ig-list.json");
322    for (JsonObject ig : json.getJsonObjects("guides")) {
323      list.add(ig.asString("npm-name"));
324    }
325    json = JsonParser.parseObjectFromUrl("https://fhir.github.io/ig-registry/package-feeds.json");
326    for (JsonObject feed : json.getJsonObjects("feeds")) {
327      processFeed(list, feed.asString("url"));
328    }
329
330    return list;
331  }
332
333  private void processFeed(Set<String> list, String str) throws IOException, ParserConfigurationException, SAXException {
334    System.out.println("Feed "+str);
335    try {
336
337      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), str+"?nocache=" + System.currentTimeMillis());
338      res.checkThrowException();
339      Document xml = XMLUtil.parseToDom(res.getContent());
340      for (Element channel : XMLUtil.getNamedChildren(xml.getDocumentElement(), "channel")) {
341        for (Element item : XMLUtil.getNamedChildren(channel, "item")) {
342          String pid = XMLUtil.getNamedChildText(item, "title");
343          if (pid != null && pid.contains("#")) {
344            list.add(pid.substring(0, pid.indexOf("#")));
345          }
346        }
347      }
348    } catch (Exception e) {
349      System.out.println("   "+e.getMessage());
350    }
351  }
352
353
354  private void processPackage(String pid, String v, int i, int t) throws IOException, FHIRException, EOperationOutcome {
355    NpmPackage npm = null;
356    String fv = null;
357    try {
358      npm = pcm.loadPackage(pid, v);
359    } catch (Throwable e) {
360      System.out.println("Unable to load package: "+pid+"#"+v+": "+e.getMessage());
361      return;
362    }
363
364    try {
365      fv = npm.fhirVersion();
366    } catch (Throwable e) {
367      System.out.println("Unable to identify package FHIR version:: "+pid+"#"+v+": "+e.getMessage());
368    }
369    if (corePackages || !corePackage(npm)) {
370      PackageContext ctxt = new PackageContext(pid+"#"+v, npm, fv);
371      boolean ok = false;
372      Object context = null;
373      try {
374        context = processor.startPackage(ctxt);
375        ok = true;
376      } catch (Exception e) {
377        System.out.println("####### Error loading package  "+pid+"#"+v +"["+fv+"]: "+e.getMessage());
378        e.printStackTrace();
379      }
380      if (ok) {
381        int c = 0;
382        if (fv != null && (versions.isEmpty() || versions.contains(fv))) {
383          for (PackagedResourceFile p : npm.listAllResources(resourceTypes)) {
384            c++;
385            try {
386              processor.processResource(ctxt, context, p.getResourceType(), p.getFilename(), FileUtilities.streamToBytes(npm.load(p.getFolder(), p.getFilename())));
387            } catch (Exception e) {
388              System.out.println("####### Error loading "+pid+"#"+v +"["+fv+"]/"+p.getResourceType()+" ####### "+e.getMessage());
389              e.printStackTrace();
390            }
391          }
392        }    
393        processor.finishPackage(ctxt);
394        System.out.println("Processed: "+pid+"#"+v+": "+c+" resources ("+i+" of "+t+")");  
395      }
396    }
397  }
398
399  private boolean corePackage(NpmPackage npm) {
400    return npm != null && !Utilities.noString(npm.name()) && (
401        npm.name().startsWith("hl7.terminology") || 
402        npm.name().startsWith("hl7.fhir.core") || 
403        npm.name().startsWith("hl7.fhir.r2.") || 
404        npm.name().startsWith("hl7.fhir.r2b.") || 
405        npm.name().startsWith("hl7.fhir.r3.") || 
406        npm.name().startsWith("hl7.fhir.r4.") || 
407        npm.name().startsWith("hl7.fhir.r4b.") || 
408        npm.name().startsWith("hl7.fhir.r5."));
409  }
410
411  public int getStep() {
412    return step;
413  }
414
415  public void setStep(int step) {
416    this.step = step;
417  }
418
419}