Source code

001package org.hl7.fhir.r5.testfactory;
002
003import java.io.ByteArrayInputStream;
004import java.io.ByteArrayOutputStream;
005import java.io.File;
006import java.io.FileOutputStream;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.PrintStream;
010import java.sql.SQLException;
011import java.util.ArrayList;
012import java.util.HashMap;
013import java.util.List;
014import java.util.Locale;
015import java.util.Map;
016import java.util.UUID;
017
018import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
019import org.hl7.fhir.exceptions.FHIRException;
020import org.hl7.fhir.r5.context.IWorkerContext;
021import org.hl7.fhir.r5.elementmodel.Element;
022import org.hl7.fhir.r5.elementmodel.Manager;
023import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
024import org.hl7.fhir.r5.fhirpath.ExpressionNode.CollectionStatus;
025import org.hl7.fhir.r5.fhirpath.FHIRPathEngine;
026import org.hl7.fhir.r5.fhirpath.FHIRPathEngine.IEvaluationContext.FunctionDefinition;
027import org.hl7.fhir.r5.fhirpath.FHIRPathUtilityClasses.FunctionDetails;
028import org.hl7.fhir.r5.fhirpath.TypeDetails;
029import org.hl7.fhir.r5.formats.IParser.OutputStyle;
030import org.hl7.fhir.r5.liquid.BaseTableWrapper;
031import org.hl7.fhir.r5.liquid.LiquidEngine;
032import org.hl7.fhir.r5.liquid.LiquidEngine.LiquidDocument;
033import org.hl7.fhir.r5.model.Base;
034import org.hl7.fhir.r5.model.StringType;
035import org.hl7.fhir.r5.model.StructureDefinition;
036import org.hl7.fhir.r5.model.ValueSet;
037import org.hl7.fhir.r5.testfactory.dataprovider.TableDataProvider;
038import org.hl7.fhir.r5.testfactory.dataprovider.ValueSetDataProvider;
039import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
040import org.hl7.fhir.utilities.FhirPublication;
041import org.hl7.fhir.utilities.FileUtilities;
042import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
043import org.hl7.fhir.utilities.Utilities;
044import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
045import org.hl7.fhir.utilities.http.HTTPResult;
046import org.hl7.fhir.utilities.http.ManagedWebAccess;
047import org.hl7.fhir.utilities.json.JsonException;
048import org.hl7.fhir.utilities.json.model.JsonObject;
049import org.hl7.fhir.utilities.json.parser.JsonParser;
050
051import ca.uhn.fhir.context.support.IValidationSupport.ValueSetExpansionOutcome;
052
053@MarkedToMoveToAdjunctPackage
054public class TestDataFactory {
055
056  public static class DataTable extends Base {
057    List<String> columns = new ArrayList<String>();
058    List<List<String>> rows = new ArrayList<List<String>>();
059    
060    @Override
061    public String fhirType() {
062      return "DataTable";
063    }
064    @Override
065    public String getIdBase() {
066      return null;
067    }
068    @Override
069    public void setIdBase(String value) {
070      throw new Error("Readonly");
071    }
072    @Override
073    public Base copy() {
074      return this;
075    }
076    
077    
078    public List<String> getColumns() {
079      return columns;
080    }
081    public List<List<String>> getRows() {
082      return rows;
083    }
084    @Override
085    public FhirPublication getFHIRPublicationVersion() {
086      return FhirPublication.R5;
087    }
088
089    public Base[] getProperty(int hash, String name, boolean checkValid) throws FHIRException {
090      if (rows != null && "rows".equals(name)) {
091        Base[] l = new Base[rows.size()];
092        for (int i = 0; i < rows.size(); i++) {
093          l[i] = BaseTableWrapper.forRow(columns, rows.get(i));
094        }
095        return l;      
096      }
097      return super.getProperty(hash, name, checkValid);
098    }
099    
100    public String cell(int row, String col) {
101      if (row >= 0 && row < rows.size()) {
102        List<String> r = rows.get(row);
103        int c = -1;
104        if (Utilities.isInteger(col)) {
105          c = Utilities.parseInt(col, -1);
106        } else {
107          c = columns.indexOf(col);
108        }
109        if (c > -1 && c  < r.size()) {
110          return r.get(c);
111        }
112      }
113      return null;
114    }
115    public String lookup(String lcol, String val, String rcol) {
116      for (int i = 0; i < rows.size(); i++) {
117        if (val.equals(cell(i, lcol))) {
118          return cell(i, rcol);
119        }
120      }
121      return null;
122    }
123  }
124    
125  public static class CellLookupFunction extends FunctionDefinition {
126
127    @Override
128    public String name() {
129      return "cell";
130    }
131
132    @Override
133    public FunctionDetails details() {
134      return new FunctionDetails("Lookup a data element", 2, 2);
135    }
136
137    @Override
138    public TypeDetails check(FHIRPathEngine engine, Object appContext, TypeDetails focus, List<TypeDetails> parameters) {
139      return new TypeDetails(CollectionStatus.SINGLETON, "string");
140    }
141
142    @Override
143    public List<Base> execute(FHIRPathEngine engine, Object appContext, List<Base> focus, List<List<Base>> parameters) {
144      int row = Utilities.parseInt(parameters.get(0).get(0).primitiveValue(), 0);
145      String col = parameters.get(1).get(0).primitiveValue();
146      DataTable dt = (DataTable) focus.get(0);
147      
148      List<Base> res = new ArrayList<Base>();
149      String s = dt.cell(row, col);
150      if (!Utilities.noString(s)) {
151        res.add(new StringType(s));
152      }
153      return res;   
154    }
155  }
156
157  public static class TableLookupFunction extends FunctionDefinition {
158
159    @Override
160    public String name() {
161      return "lookup";
162    }
163
164    @Override
165    public FunctionDetails details() {
166      return new FunctionDetails("Lookup a value in a table", 4, 4);
167    }
168
169    @Override
170    public TypeDetails check(FHIRPathEngine engine, Object appContext, TypeDetails focus, List<TypeDetails> parameters) {
171      return new TypeDetails(CollectionStatus.SINGLETON, "string");
172    }
173
174    @Override
175    public List<Base> execute(FHIRPathEngine engine, Object appContext, List<Base> focus, List<List<Base>> parameters) {
176
177      List<Base> res = new ArrayList<Base>();
178      if (focus.get(0) instanceof BaseTableWrapper && parameters.size() == 4 && parameters.get(0).size() == 1 && parameters.get(1).size() == 1 && parameters.get(2).size() == 1 && parameters.get(3).size() == 1) {
179        BaseTableWrapper dt = (BaseTableWrapper) focus.get(0);
180        String table = parameters.get(0).get(0).primitiveValue(); 
181        String lcol = parameters.get(1).get(0).primitiveValue();
182        String val = parameters.get(2).get(0).primitiveValue();
183        String rcol = parameters.get(3).get(0).primitiveValue();
184        if (table != null && lcol != null && val != null && rcol != null) {
185          DataTable tbl = dt.getTables().get(table);
186          if (tbl != null) {
187            String s = tbl.lookup(lcol, val, rcol);
188            if (!Utilities.noString(s)) {
189              res.add(new StringType(s));
190            }
191          }
192        }
193      }
194      return res;
195    }
196    
197  }
198  
199  private String rootFolder;
200  private LiquidEngine liquid;
201  private PrintStream log;
202  private IWorkerContext context;
203  private String canonical;
204  private FhirFormat format;
205  private File localData;
206  private FHIRPathEngine fpe;
207  private JsonObject details;
208  private String name;
209  private boolean testing;
210  private Map<String, String> profileMap;
211  private Locale locale;
212  
213  public TestDataFactory(IWorkerContext context, JsonObject details, LiquidEngine liquid, FHIRPathEngine fpe, String canonical, String rootFolder, String logFolder, Map<String, String> profileMap, Locale locale) throws IOException {
214    super();
215    this.context = context;
216    this.rootFolder = rootFolder;
217    this.canonical = canonical;
218    this.details = details;
219    this.liquid = liquid;
220    this.fpe = fpe;
221    this.profileMap = profileMap;
222    this.locale = locale;
223
224    this.name = details.asString("name");
225    if (Utilities.noString(name)) {
226      throw new FHIRException("Factory has no name");
227    }
228    log = new PrintStream(new FileOutputStream(Utilities.path(logFolder, name+".log"))); 
229    format = "json".equals(details.asString("format")) ? FhirFormat.JSON : FhirFormat.XML;
230  }
231  
232  public String getName() {
233    return name;
234  }
235  
236  public void execute() throws FHIRException, IOException {
237    String mode = details.asString( "mode");
238    if ("liquid".equals(mode)) {
239      executeLiquid();
240    } else if ("profile".equals(mode)) {
241      executeProfile();
242    } else {
243      error("Factory "+getName()+" mode '"+mode+"' unknown");
244    }
245    log("finished successfully");
246    log.close();
247  }
248  
249
250  private void logDataScheme(DataTable tbl, Map<String, DataTable> tables) throws IOException {
251    log("data: "+CommaSeparatedStringBuilder.join(",", tbl.getColumns()));
252    for (String tn : Utilities.sorted(tables.keySet())) {
253      log("tn: "+CommaSeparatedStringBuilder.join(",", tables.get(tn).getColumns()));
254    }
255  }
256  private void logDataScheme(TableDataProvider tbl, Map<String, DataTable> tables) throws IOException {
257    log("data: "+CommaSeparatedStringBuilder.join(",", tbl.columns()));
258    for (String tn : Utilities.sorted(tables.keySet())) {
259      log("tn: "+CommaSeparatedStringBuilder.join(",", tables.get(tn).getColumns()));
260    }
261  }
262
263  
264  private void executeProfile() throws IOException {
265    try {
266      checkDownloadBaseData();
267      
268      TableDataProvider tbl = loadTable(Utilities.path(rootFolder, details.asString( "data")));
269      Map<String, DataTable> tables = new HashMap<>();
270      if (details.has("tables")) {
271        JsonObject tablesJ = details.getJsonObject("tables");
272        for (String n : tablesJ.getNames()) {
273          tables.put(n, loadData(Utilities.path(rootFolder, tablesJ.asString(n))));
274        } 
275      }
276      logDataScheme(tbl, tables);
277      ProfileBasedFactory factory = new ProfileBasedFactory(fpe, localData.getAbsolutePath(), tbl, tables, details.forceArray("mappings"));
278      factory.setLog(log);
279      factory.setTesting(testing);
280      factory.setMarkProfile(details.asBoolean("mark-profile"));
281      String purl = details.asString( "profile");
282      StructureDefinition profile = context.fetchResource(StructureDefinition.class, purl);
283      if (profile == null) {
284        error("Unable to find profile "+purl);
285      } else if (!profile.hasSnapshot()) {
286        error("Profile "+purl+" doesn't have a snapshot");
287      }
288      
289      if ("true".equals(details.asString("bundle"))) {
290        byte[] data = runBundle(profile, factory, tbl);
291        String fn = Utilities.path(rootFolder, details.asString( "filename"));
292        FileUtilities.bytesToFile(data, fn);
293        profileMap.put(FileUtilities.changeFileExt(fn, ""), profile.getVersionedUrl());
294      } else {
295        while (tbl.nextRow()) {
296          if (rowPasses(factory)) {
297            byte[] data = factory.generateFormat(profile, format);
298            String fn = Utilities.path(rootFolder, getFileName(details.asString( "filename"), tbl.columns(), tbl.cells()));
299            FileUtilities.bytesToFile(data, fn);
300            profileMap.put(FileUtilities.changeFileExt(fn, ""), profile.getVersionedUrl());
301          }
302        }
303      }
304    } catch (Exception e) {
305      System.out.println("Error running test factory '"+getName()+"': "+e.getMessage());
306      log("Error running test case '"+getName()+"': "+e.getMessage());
307      e.printStackTrace(log);
308      throw new FHIRException(e);
309    }
310  }
311
312  private void checkDownloadBaseData() throws IOException {
313    localData = ManagedFileAccess.file(Utilities.path("[tmp]", "fhir-test-data.db"));  
314    File localInfo = ManagedFileAccess.file(Utilities.path("[tmp]", "fhir-test-data.json"));  
315    try {
316      JsonObject local = localInfo.exists() ? JsonParser.parseObject(localInfo) : null; 
317      JsonObject json = JsonParser.parseObjectFromUrl("http://fhir.org/downloads/test-data-versions.json");
318      JsonObject current = json.forceArray("versions").get(0).asJsonObject();
319      if (current == null) {
320        throw new FHIRException("No current information about FHIR downloads");
321      }
322      String date = current.asString("date");
323      if (date == null) {
324        throw new FHIRException("No date on current information about FHIR downloads");
325      }
326      String filename = current.asString("filename");
327      if (filename == null) {
328        throw new FHIRException("No filename on current information about FHIR downloads");
329      }
330      if (local == null || !date.equals(local.asString("date"))) {
331        HTTPResult data = ManagedWebAccess.get(Utilities.strings("general"), "http://fhir.org/downloads/"+filename);
332        FileUtilities.bytesToFile(data.getContent(), localData);
333        local = new JsonObject();
334        local.set("date", date);
335        JsonParser.compose(current, localInfo, true);
336      }
337    } catch (Exception e) {
338      if (!localData.exists()) {
339        log("Unable to download copy of FHIR testing data: "+ e.getMessage());
340        throw new FHIRException("Unable to download copy of FHIR testing data", e);
341      }
342    }
343  }
344
345  private byte[] runBundle(StructureDefinition profile, ProfileBasedFactory factory, TableDataProvider tbl) throws IOException, FHIRException, SQLException {
346    Element bundle = Manager.parse(context, bundleShell(), FhirFormat.JSON).get(0).getElement();
347    bundle.makeElement("id").setValue(UUID.randomUUID().toString().toLowerCase());
348    
349    while (tbl.nextRow()) {
350      if (rowPasses(factory)) {
351        Element resource = factory.generate(profile);
352        Element be = bundle.makeElement("entry");
353        be.makeElement("fullUrl").setValue(Utilities.pathURL(canonical, "test", resource.fhirType(), resource.getIdBase()));
354        be.makeElement("resource").getChildren().addAll(resource.getChildren());
355      }
356    }
357    log("Saving Bundle");
358    ByteArrayOutputStream bs = new ByteArrayOutputStream();
359    Manager.compose(context, bundle, bs, format, OutputStyle.PRETTY, null);
360    return bs.toByteArray();
361  }
362
363  private boolean rowPasses(ProfileBasedFactory factory) throws IOException {
364    if (details.has("filter")) {
365      List<String> ls = new ArrayList<String>();
366      String res = factory.evaluateExpression(ls, details.get("filter"), "filter");
367      for (String l : ls) {
368        log(l);
369      }
370      return  Utilities.existsInList(res, "1", "true");
371    } else {
372      return true;
373    }
374  }
375
376  private TableDataProvider loadTable(String path) throws IOException, InvalidFormatException {
377    log("Load Data From "+path);
378    return loadTableProvider(path, locale);
379  }
380
381  private void error(String msg) throws IOException {
382    log(msg);
383    log.close();
384    throw new FHIRException(msg);
385  }
386
387  private void log(String msg) throws IOException {
388    log.append(msg+"\r\n");    
389  }
390
391  public void executeLiquid() throws IOException {
392    try {
393      LiquidDocument template = liquid.parse(FileUtilities.fileToString(Utilities.path(rootFolder, details.asString( "liquid"))), "liquid");
394      log("liquid compiled");
395      DataTable dt = loadData(Utilities.path(rootFolder, details.asString( "data")));
396      Map<String, DataTable> tables = new HashMap<>();
397      liquid.getVars().clear();
398      if (details.has("tables")) {
399        JsonObject tablesJ = details.getJsonObject("tables");
400        for (String n : tablesJ.getNames()) {
401          DataTable v = loadData(Utilities.path(rootFolder, tablesJ.asString(n)));
402          liquid.getVars().put(n, v);
403          tables.put(n, v);
404        } 
405      }
406
407      logDataScheme(dt, tables);
408      
409      logStrings("columns", dt.columns);
410      if ("true".equals(details.asString( "bundle"))) {
411        byte[] data = runBundle(template, dt);
412        FileUtilities.bytesToFile(data, Utilities.path(rootFolder, details.asString( "filename")));
413      } else {
414        for (List<String> row : dt.rows) { 
415          byte[] data = runInstance(template, dt.columns, row);
416          FileUtilities.bytesToFile(data, Utilities.path(rootFolder, getFileName(details.asString( "filename"), dt.columns, row)));
417        }
418      }
419    } catch (Exception e) {
420      System.out.println("Error running test factory '"+getName()+"': "+e.getMessage());
421      log("Error running test case '"+getName()+"': "+e.getMessage());
422      e.printStackTrace(log);
423      throw new FHIRException(e);
424    }
425  }
426
427  private void logStrings(String name, List<String> columns) throws IOException {
428    log(name+": "+CommaSeparatedStringBuilder.join(", ", columns));    
429  }
430
431  private String getFileName(String name, List<String> columns, List<String> values) {
432    for (int i = 0; i < columns.size(); i++) {
433      name = name.replace("$"+columns.get(i)+"$", values.get(i));
434    }
435    return name;
436  }
437
438  private byte[] runInstance(LiquidDocument template, List<String> columns, List<String> row) throws JsonException, IOException {
439    logStrings("row", row);
440    BaseTableWrapper base = BaseTableWrapper.forRow(columns, row);
441    String cnt = liquid.evaluate(template, base, this).trim();
442    if (format == FhirFormat.JSON) {
443      JsonObject j = JsonParser.parseObject(cnt, true);
444      return JsonParser.composeBytes(j, true);
445    } else {
446      return FileUtilities.stringToBytes(cnt);
447    }
448  }
449
450  private byte[] runBundle(LiquidDocument template, DataTable dt) throws JsonException, IOException {
451    Element bundle = Manager.parse(context, bundleShell(), FhirFormat.JSON).get(0).getElement();
452    bundle.makeElement("id").setValue(UUID.randomUUID().toString().toLowerCase());
453    
454    for (List<String> row : dt.rows) { 
455      byte[] data = runInstance(template, dt.columns, row);
456      Element resource = Manager.parse(context, new ByteArrayInputStream(data), format).get(0).getElement();
457      Element be = bundle.makeElement("entry");
458      be.makeElement("fullUrl").setValue(Utilities.pathURL(canonical, "test", resource.fhirType(), resource.getIdBase()));
459      be.makeElement("resource").getChildren().addAll(resource.getChildren());
460    }
461    log("Saving Bundle");
462    ByteArrayOutputStream bs = new ByteArrayOutputStream();
463    Manager.compose(context, bundle, bs, format, OutputStyle.PRETTY, null);
464    return bs.toByteArray();
465  }
466
467  private InputStream bundleShell() throws IOException {
468    String bundle = "{\"resourceType\" : \"Bundle\", \"type\" : \"collection\"}";
469    return new ByteArrayInputStream(FileUtilities.stringToBytes(bundle));
470  }
471
472  private DataTable loadData(String path) throws FHIRException, IOException, InvalidFormatException {
473    log("Load Data From "+path);
474    TableDataProvider tbl = loadTableProvider(path, locale);
475
476    DataTable dt = new DataTable();
477    for (String n : tbl.columns()) {
478      dt.columns.add(n);
479    }
480    int t = dt.columns.size();
481    while (tbl.nextRow()) {
482      List<String> values = new ArrayList<String>();
483      for (String b : tbl.cells()) {
484        values.add(b);
485      }
486      while (values.size() < t) {
487        values.add("");
488      }
489      while (values.size() > t) {
490        values.remove(values.size()-1);
491      }
492      dt.rows.add(values);
493    }
494    return dt;
495  }
496
497  public TableDataProvider loadTableProvider(String path, Locale locale) {
498    TableDataProvider tbl;
499    if (Utilities.isAbsoluteUrl(path)) {
500      ValueSet vs = context.findTxResource(ValueSet.class, path);
501      if (vs == null) {
502        throw new FHIRException("ValueSet "+path+" not found");
503      } else {
504        org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome exp = context.expandVS(vs, true, false);
505        if (exp.isOk()) {
506          tbl = new ValueSetDataProvider(exp.getValueset().getExpansion());
507        } else {
508          throw new FHIRException("ValueSet "+path+" coult not be expanded: "+exp.getError());
509        }
510      }
511    } else {
512      tbl = TableDataProvider.forFile(path, locale);
513    }
514    return tbl;
515  }
516
517  public String statedLog() {
518    return name+".log";
519  }
520
521  public boolean isTesting() {
522    return testing;
523  }
524
525  public void setTesting(boolean testing) {
526    this.testing = testing;
527  }
528  
529  
530}