Source code

001package org.hl7.fhir.r5.testfactory;
002
003import java.io.ByteArrayInputStream;
004import java.io.ByteArrayOutputStream;
005import java.io.File;
006import java.io.FileOutputStream;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.PrintStream;
010import java.sql.SQLException;
011import java.util.ArrayList;
012import java.util.HashMap;
013import java.util.List;
014import java.util.Locale;
015import java.util.Map;
016import java.util.UUID;
017
018import lombok.extern.slf4j.Slf4j;
019import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
020import org.hl7.fhir.exceptions.FHIRException;
021import org.hl7.fhir.r5.context.IWorkerContext;
022import org.hl7.fhir.r5.elementmodel.Element;
023import org.hl7.fhir.r5.elementmodel.Manager;
024import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
025import org.hl7.fhir.r5.fhirpath.ExpressionNode.CollectionStatus;
026import org.hl7.fhir.r5.fhirpath.FHIRPathEngine;
027import org.hl7.fhir.r5.fhirpath.FHIRPathEngine.IEvaluationContext.FunctionDefinition;
028import org.hl7.fhir.r5.fhirpath.FHIRPathUtilityClasses.FunctionDetails;
029import org.hl7.fhir.r5.fhirpath.TypeDetails;
030import org.hl7.fhir.r5.formats.IParser.OutputStyle;
031import org.hl7.fhir.r5.liquid.BaseTableWrapper;
032import org.hl7.fhir.r5.liquid.LiquidEngine;
033import org.hl7.fhir.r5.liquid.LiquidEngine.LiquidDocument;
034import org.hl7.fhir.r5.model.Base;
035import org.hl7.fhir.r5.model.StringType;
036import org.hl7.fhir.r5.model.StructureDefinition;
037import org.hl7.fhir.r5.model.ValueSet;
038import org.hl7.fhir.r5.testfactory.dataprovider.TableDataProvider;
039import org.hl7.fhir.r5.testfactory.dataprovider.ValueSetDataProvider;
040import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
041import org.hl7.fhir.utilities.FhirPublication;
042import org.hl7.fhir.utilities.FileUtilities;
043import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
044import org.hl7.fhir.utilities.Utilities;
045import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
046import org.hl7.fhir.utilities.http.HTTPResult;
047import org.hl7.fhir.utilities.http.ManagedWebAccess;
048import org.hl7.fhir.utilities.json.JsonException;
049import org.hl7.fhir.utilities.json.model.JsonObject;
050import org.hl7.fhir.utilities.json.parser.JsonParser;
051
052@MarkedToMoveToAdjunctPackage
053@Slf4j
054public class TestDataFactory {
055
056  public static class DataTable extends Base {
057    List<String> columns = new ArrayList<String>();
058    List<List<String>> rows = new ArrayList<List<String>>();
059    
060    @Override
061    public String fhirType() {
062      return "DataTable";
063    }
064    @Override
065    public String getIdBase() {
066      return null;
067    }
068    @Override
069    public void setIdBase(String value) {
070      throw new Error("Readonly");
071    }
072    @Override
073    public Base copy() {
074      return this;
075    }
076    
077    
078    public List<String> getColumns() {
079      return columns;
080    }
081    public List<List<String>> getRows() {
082      return rows;
083    }
084    @Override
085    public FhirPublication getFHIRPublicationVersion() {
086      return FhirPublication.R5;
087    }
088
089    public Base[] getProperty(int hash, String name, boolean checkValid) throws FHIRException {
090      if (rows != null && "rows".equals(name)) {
091        Base[] l = new Base[rows.size()];
092        for (int i = 0; i < rows.size(); i++) {
093          l[i] = BaseTableWrapper.forRow(columns, rows.get(i));
094        }
095        return l;      
096      }
097      return super.getProperty(hash, name, checkValid);
098    }
099    
100    public String cell(int row, String col) {
101      if (row >= 0 && row < rows.size()) {
102        List<String> r = rows.get(row);
103        int c = -1;
104        if (Utilities.isInteger(col)) {
105          c = Utilities.parseInt(col, -1);
106        } else {
107          c = columns.indexOf(col);
108        }
109        if (c > -1 && c  < r.size()) {
110          return r.get(c);
111        }
112      }
113      return null;
114    }
115    public String lookup(String lcol, String val, String rcol) {
116      for (int i = 0; i < rows.size(); i++) {
117        if (val.equals(cell(i, lcol))) {
118          return cell(i, rcol);
119        }
120      }
121      return null;
122    }
123  }
124    
125  public static class CellLookupFunction extends FunctionDefinition {
126
127    @Override
128    public String name() {
129      return "cell";
130    }
131
132    @Override
133    public FunctionDetails details() {
134      return new FunctionDetails("Lookup a data element", 2, 2);
135    }
136
137    @Override
138    public TypeDetails check(FHIRPathEngine engine, Object appContext, TypeDetails focus, List<TypeDetails> parameters) {
139      return new TypeDetails(CollectionStatus.SINGLETON, "string");
140    }
141
142    @Override
143    public List<Base> execute(FHIRPathEngine engine, Object appContext, List<Base> focus, List<List<Base>> parameters) {
144      int row = Utilities.parseInt(parameters.get(0).get(0).primitiveValue(), 0);
145      String col = parameters.get(1).get(0).primitiveValue();
146      DataTable dt = (DataTable) focus.get(0);
147      
148      List<Base> res = new ArrayList<Base>();
149      String s = dt.cell(row, col);
150      if (!Utilities.noString(s)) {
151        res.add(new StringType(s));
152      }
153      return res;   
154    }
155  }
156
157  public static class TableLookupFunction extends FunctionDefinition {
158
159    @Override
160    public String name() {
161      return "lookup";
162    }
163
164    @Override
165    public FunctionDetails details() {
166      return new FunctionDetails("Lookup a value in a table", 4, 4);
167    }
168
169    @Override
170    public TypeDetails check(FHIRPathEngine engine, Object appContext, TypeDetails focus, List<TypeDetails> parameters) {
171      return new TypeDetails(CollectionStatus.SINGLETON, "string");
172    }
173
174    @Override
175    public List<Base> execute(FHIRPathEngine engine, Object appContext, List<Base> focus, List<List<Base>> parameters) {
176
177      List<Base> res = new ArrayList<Base>();
178      if (focus.get(0) instanceof BaseTableWrapper && parameters.size() == 4 && parameters.get(0).size() == 1 && parameters.get(1).size() == 1 && parameters.get(2).size() == 1 && parameters.get(3).size() == 1) {
179        BaseTableWrapper dt = (BaseTableWrapper) focus.get(0);
180        String table = parameters.get(0).get(0).primitiveValue(); 
181        String lcol = parameters.get(1).get(0).primitiveValue();
182        String val = parameters.get(2).get(0).primitiveValue();
183        String rcol = parameters.get(3).get(0).primitiveValue();
184        if (table != null && lcol != null && val != null && rcol != null) {
185          DataTable tbl = dt.getTables().get(table);
186          if (tbl != null) {
187            String s = tbl.lookup(lcol, val, rcol);
188            if (!Utilities.noString(s)) {
189              res.add(new StringType(s));
190            }
191          }
192        }
193      }
194      return res;
195    }
196    
197  }
198  
199  private String rootFolder;
200  private LiquidEngine liquid;
201  private PrintStream testLog;
202  private IWorkerContext context;
203  private String canonical;
204  private FhirFormat format;
205  private File localData;
206  private FHIRPathEngine fpe;
207  private JsonObject details;
208  private String name;
209  private boolean testing;
210  private Map<String, String> profileMap;
211  private Locale locale;
212  
213  public TestDataFactory(IWorkerContext context, JsonObject details, LiquidEngine liquid, FHIRPathEngine fpe, String canonical, String rootFolder, String logFolder, Map<String, String> profileMap, Locale locale) throws IOException {
214    super();
215    this.context = context;
216    this.rootFolder = rootFolder;
217    this.canonical = canonical;
218    this.details = details;
219    this.liquid = liquid;
220    this.fpe = fpe;
221    this.profileMap = profileMap;
222    this.locale = locale;
223
224    this.name = details.asString("name");
225    if (Utilities.noString(name)) {
226      throw new FHIRException("Factory has no name");
227    }
228    testLog = new PrintStream(new FileOutputStream(Utilities.path(logFolder, name+".log")));
229    format = "json".equals(details.asString("format")) ? FhirFormat.JSON : FhirFormat.XML;
230  }
231  
232  public String getName() {
233    return name;
234  }
235  
236  public void execute() throws FHIRException, IOException {
237    String mode = details.asString( "mode");
238    if ("liquid".equals(mode)) {
239      executeLiquid();
240    } else if ("profile".equals(mode)) {
241      executeProfile();
242    } else {
243      error("Factory "+getName()+" mode '"+mode+"' unknown");
244    }
245    log("finished successfully");
246    testLog.close();
247  }
248  
249
250  private void logDataScheme(DataTable tbl, Map<String, DataTable> tables) throws IOException {
251    log("data: "+CommaSeparatedStringBuilder.join(",", tbl.getColumns()));
252    for (String tn : Utilities.sorted(tables.keySet())) {
253      log("tn: "+CommaSeparatedStringBuilder.join(",", tables.get(tn).getColumns()));
254    }
255  }
256  private void logDataScheme(TableDataProvider tbl, Map<String, DataTable> tables) throws IOException {
257    log("data: "+CommaSeparatedStringBuilder.join(",", tbl.columns()));
258    for (String tn : Utilities.sorted(tables.keySet())) {
259      log("tn: "+CommaSeparatedStringBuilder.join(",", tables.get(tn).getColumns()));
260    }
261  }
262
263  
264  private void executeProfile() throws IOException {
265    try {
266      checkDownloadBaseData();
267      
268      TableDataProvider tbl = loadTable(Utilities.path(rootFolder, details.asString( "data")));
269      Map<String, DataTable> tables = new HashMap<>();
270      if (details.has("tables")) {
271        JsonObject tablesJ = details.getJsonObject("tables");
272        for (String n : tablesJ.getNames()) {
273          tables.put(n, loadData(Utilities.path(rootFolder, tablesJ.asString(n))));
274        } 
275      }
276      logDataScheme(tbl, tables);
277      ProfileBasedFactory factory = new ProfileBasedFactory(fpe, localData.getAbsolutePath(), tbl, tables, details.forceArray("mappings"));
278      factory.setLog(testLog);
279      factory.setTesting(testing);
280      factory.setMarkProfile(details.asBoolean("mark-profile"));
281      String purl = details.asString( "profile");
282      StructureDefinition profile = context.fetchResource(StructureDefinition.class, purl);
283      if (profile == null) {
284        error("Unable to find profile "+purl);
285      } else if (!profile.hasSnapshot()) {
286        error("Profile "+purl+" doesn't have a snapshot");
287      }
288      
289      if ("true".equals(details.asString("bundle"))) {
290        byte[] data = runBundle(profile, factory, tbl);
291        String fn = Utilities.path(rootFolder, details.asString( "filename"));
292        FileUtilities.bytesToFile(data, fn);
293        profileMap.put(FileUtilities.changeFileExt(fn, ""), profile.getVersionedUrl());
294      } else {
295        while (tbl.nextRow()) {
296          if (rowPasses(factory)) {
297            byte[] data = factory.generateFormat(profile, format);
298            String fn = Utilities.path(rootFolder, getFileName(details.asString( "filename"), tbl.columns(), tbl.cells()));
299            FileUtilities.createDirectory(FileUtilities.getDirectoryForFile(fn));
300            FileUtilities.bytesToFile(data, fn);
301            profileMap.put(FileUtilities.changeFileExt(fn, ""), profile.getVersionedUrl());
302          }
303        }
304      }
305    } catch (Exception e) {
306      log.error("Error running test factory '"+getName()+"': "+e.getMessage());
307      log("Error running test case '"+getName()+"': "+e.getMessage());
308      e.printStackTrace(testLog);
309      throw new FHIRException(e);
310    }
311  }
312
313  private void checkDownloadBaseData() throws IOException {
314    localData = ManagedFileAccess.file(Utilities.path("[tmp]", "fhir-test-data.db"));  
315    File localInfo = ManagedFileAccess.file(Utilities.path("[tmp]", "fhir-test-data.json"));  
316    try {
317      JsonObject local = localInfo.exists() ? JsonParser.parseObject(localInfo) : null; 
318      JsonObject json = JsonParser.parseObjectFromUrl("http://fhir.org/downloads/test-data-versions.json");
319      JsonObject current = json.forceArray("versions").get(0).asJsonObject();
320      if (current == null) {
321        throw new FHIRException("No current information about FHIR downloads");
322      }
323      String date = current.asString("date");
324      if (date == null) {
325        throw new FHIRException("No date on current information about FHIR downloads");
326      }
327      String filename = current.asString("filename");
328      if (filename == null) {
329        throw new FHIRException("No filename on current information about FHIR downloads");
330      }
331      if (local == null || !date.equals(local.asString("date"))) {
332        HTTPResult data = ManagedWebAccess.get(Utilities.strings("general"), "http://fhir.org/downloads/"+filename);
333        FileUtilities.bytesToFile(data.getContent(), localData);
334        local = new JsonObject();
335        local.set("date", date);
336        JsonParser.compose(current, localInfo, true);
337      }
338    } catch (Exception e) {
339      if (!localData.exists()) {
340        log("Unable to download copy of FHIR testing data: "+ e.getMessage());
341        throw new FHIRException("Unable to download copy of FHIR testing data", e);
342      }
343    }
344  }
345
346  private byte[] runBundle(StructureDefinition profile, ProfileBasedFactory factory, TableDataProvider tbl) throws IOException, FHIRException, SQLException {
347    Element bundle = Manager.parse(context, bundleShell(), FhirFormat.JSON).get(0).getElement();
348    bundle.makeElement("id").setValue(UUID.randomUUID().toString().toLowerCase());
349    
350    while (tbl.nextRow()) {
351      if (rowPasses(factory)) {
352        Element resource = factory.generate(profile);
353        Element be = bundle.makeElement("entry");
354        be.makeElement("fullUrl").setValue(Utilities.pathURL(canonical, "test", resource.fhirType(), resource.getIdBase()));
355        be.makeElement("resource").getChildren().addAll(resource.getChildren());
356      }
357    }
358    log("Saving Bundle");
359    ByteArrayOutputStream bs = new ByteArrayOutputStream();
360    Manager.compose(context, bundle, bs, format, OutputStyle.PRETTY, null);
361    return bs.toByteArray();
362  }
363
364  private boolean rowPasses(ProfileBasedFactory factory) throws IOException {
365    if (details.has("filter")) {
366      List<String> ls = new ArrayList<String>();
367      String res = factory.evaluateExpression(ls, details.get("filter"), "filter");
368      for (String l : ls) {
369        log(l);
370      }
371      return  Utilities.existsInList(res, "1", "true");
372    } else {
373      return true;
374    }
375  }
376
377  private TableDataProvider loadTable(String path) throws IOException, InvalidFormatException {
378    log("Load Data From "+path);
379    return loadTableProvider(path, locale);
380  }
381
382  private void error(String msg) throws IOException {
383    log(msg);
384    testLog.close();
385    throw new FHIRException(msg);
386  }
387
388  private void log(String msg) throws IOException {
389    testLog.append(msg+"\r\n");
390  }
391
392  public void executeLiquid() throws IOException {
393    try {
394      LiquidDocument template = liquid.parse(FileUtilities.fileToString(Utilities.path(rootFolder, details.asString( "liquid"))), "liquid");
395      log("liquid compiled");
396      DataTable dt = loadData(Utilities.path(rootFolder, details.asString( "data")));
397      Map<String, DataTable> tables = new HashMap<>();
398      liquid.getVars().clear();
399      if (details.has("tables")) {
400        JsonObject tablesJ = details.getJsonObject("tables");
401        for (String n : tablesJ.getNames()) {
402          DataTable v = loadData(Utilities.path(rootFolder, tablesJ.asString(n)));
403          liquid.getVars().put(n, v);
404          tables.put(n, v);
405        } 
406      }
407
408      logDataScheme(dt, tables);
409      
410      logStrings("columns", dt.columns);
411      if ("true".equals(details.asString( "bundle"))) {
412        byte[] data = runBundle(template, dt);
413        FileUtilities.bytesToFile(data, Utilities.path(rootFolder, details.asString( "filename")));
414      } else {
415        for (List<String> row : dt.rows) { 
416          byte[] data = runInstance(template, dt.columns, row);
417          FileUtilities.bytesToFile(data, Utilities.path(rootFolder, getFileName(details.asString( "filename"), dt.columns, row)));
418        }
419      }
420    } catch (Exception e) {
421      log.error("Error running test factory '"+getName()+"': "+e.getMessage());
422      log("Error running test case '"+getName()+"': "+e.getMessage());
423      e.printStackTrace(testLog);
424      throw new FHIRException(e);
425    }
426  }
427
428  private void logStrings(String name, List<String> columns) throws IOException {
429    log(name+": "+CommaSeparatedStringBuilder.join(", ", columns));    
430  }
431
432  private String getFileName(String name, List<String> columns, List<String> values) {
433    for (int i = 0; i < columns.size(); i++) {
434      name = name.replace("$"+columns.get(i)+"$", values.get(i));
435    }
436    return name;
437  }
438
439  private byte[] runInstance(LiquidDocument template, List<String> columns, List<String> row) throws JsonException, IOException {
440    logStrings("row", row);
441    BaseTableWrapper base = BaseTableWrapper.forRow(columns, row);
442    String cnt = liquid.evaluate(template, base, this).trim();
443    if (format == FhirFormat.JSON) {
444      JsonObject j = JsonParser.parseObject(cnt, true);
445      return JsonParser.composeBytes(j, true);
446    } else {
447      return FileUtilities.stringToBytes(cnt);
448    }
449  }
450
451  private byte[] runBundle(LiquidDocument template, DataTable dt) throws JsonException, IOException {
452    Element bundle = Manager.parse(context, bundleShell(), FhirFormat.JSON).get(0).getElement();
453    bundle.makeElement("id").setValue(UUID.randomUUID().toString().toLowerCase());
454    
455    for (List<String> row : dt.rows) { 
456      byte[] data = runInstance(template, dt.columns, row);
457      Element resource = Manager.parse(context, new ByteArrayInputStream(data), format).get(0).getElement();
458      Element be = bundle.makeElement("entry");
459      be.makeElement("fullUrl").setValue(Utilities.pathURL(canonical, "test", resource.fhirType(), resource.getIdBase()));
460      be.makeElement("resource").getChildren().addAll(resource.getChildren());
461    }
462    log("Saving Bundle");
463    ByteArrayOutputStream bs = new ByteArrayOutputStream();
464    Manager.compose(context, bundle, bs, format, OutputStyle.PRETTY, null);
465    return bs.toByteArray();
466  }
467
468  private InputStream bundleShell() throws IOException {
469    String bundle = "{\"resourceType\" : \"Bundle\", \"type\" : \"collection\"}";
470    return new ByteArrayInputStream(FileUtilities.stringToBytes(bundle));
471  }
472
473  private DataTable loadData(String path) throws FHIRException, IOException, InvalidFormatException {
474    log("Load Data From "+path);
475    TableDataProvider tbl = loadTableProvider(path, locale);
476
477    DataTable dt = new DataTable();
478    for (String n : tbl.columns()) {
479      dt.columns.add(n);
480    }
481    int t = dt.columns.size();
482    while (tbl.nextRow()) {
483      List<String> values = new ArrayList<String>();
484      for (String b : tbl.cells()) {
485        values.add(b);
486      }
487      while (values.size() < t) {
488        values.add("");
489      }
490      while (values.size() > t) {
491        values.remove(values.size()-1);
492      }
493      dt.rows.add(values);
494    }
495    return dt;
496  }
497
498  public TableDataProvider loadTableProvider(String path, Locale locale) {
499    TableDataProvider tbl;
500    if (Utilities.isAbsoluteUrl(path)) {
501      ValueSet vs = context.findTxResource(ValueSet.class, path);
502      if (vs == null) {
503        throw new FHIRException("ValueSet "+path+" not found");
504      } else {
505        org.hl7.fhir.r5.terminologies.expansion.ValueSetExpansionOutcome exp = context.expandVS(vs, true, false);
506        if (exp.isOk()) {
507          tbl = new ValueSetDataProvider(exp.getValueset().getExpansion());
508        } else {
509          throw new FHIRException("ValueSet "+path+" coult not be expanded: "+exp.getError());
510        }
511      }
512    } else {
513      tbl = TableDataProvider.forFile(path, locale);
514    }
515    return tbl;
516  }
517
518  public String statedLog() {
519    return name+".log";
520  }
521
522  public boolean isTesting() {
523    return testing;
524  }
525
526  public void setTesting(boolean testing) {
527    this.testing = testing;
528  }
529  
530  
531}