001package org.hl7.fhir.r4.utils.sql;
002
003import java.math.BigDecimal;
004import java.util.ArrayList;
005import java.util.List;
006
007import org.apache.commons.codec.binary.Base64;
008import org.hl7.fhir.exceptions.FHIRException;
009import org.hl7.fhir.exceptions.PathEngineException;
010import org.hl7.fhir.r4.context.IWorkerContext;
011import org.hl7.fhir.r4.fhirpath.ExpressionNode;
012import org.hl7.fhir.r4.fhirpath.FHIRPathEngine;
013import org.hl7.fhir.r4.fhirpath.TypeDetails;
014import org.hl7.fhir.r4.fhirpath.ExpressionNode.CollectionStatus;
015import org.hl7.fhir.r4.fhirpath.FHIRPathEngine.IEvaluationContext;
016import org.hl7.fhir.r4.fhirpath.FHIRPathUtilityClasses.FunctionDetails;
017import org.hl7.fhir.r4.model.Base;
018import org.hl7.fhir.r4.model.Base64BinaryType;
019import org.hl7.fhir.r4.model.BaseDateTimeType;
020import org.hl7.fhir.r4.model.BooleanType;
021import org.hl7.fhir.r4.model.DecimalType;
022import org.hl7.fhir.r4.model.IntegerType;
023import org.hl7.fhir.r4.model.Property;
024import org.hl7.fhir.r4.model.StringType;
025import org.hl7.fhir.r4.model.ValueSet;
026import org.hl7.fhir.r4.utils.sql.Cell;
027import org.hl7.fhir.r4.utils.sql.Column;
028import org.hl7.fhir.r4.utils.sql.Provider;
029import org.hl7.fhir.r4.utils.sql.Storage;
030import org.hl7.fhir.r4.utils.sql.Store;
031import org.hl7.fhir.r4.utils.sql.Validator;
032import org.hl7.fhir.r4.utils.sql.Value;
033import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
034import org.hl7.fhir.utilities.json.model.JsonObject;
035import org.hl7.fhir.utilities.validation.ValidationMessage;
036
037
038/**
039 * How to use the Runner:
040 * 
041 * create a resource, and fill out:
042 *   the context (supports the FHIRPathEngine)
043 *   a store that handles the output 
044 *   a tracker - if you want
045 *   
046 * Once it's created, you either run it as a batch, or in trickle mode
047 * 
048 *   (1) Batch Mode
049 *   
050 *    * provide a provider 
051 *    * call execute() with a ViewDefinition
052 *    * wait... (watch with an observer if you want to track progress)
053 *   
054 *   (2) Trickle Mode
055 *    * call 'prepare', and keep the WorkContext that's returned
056 *    * each time there's a resource to process, call processResource and pass in the workContext and the resource
057 *    * when done, call finish(WorkContext)
058 */
059
060@MarkedToMoveToAdjunctPackage
061public class Runner implements IEvaluationContext {
062  
063  public interface IRunnerObserver {
064    public void handleRow(Base resource, int total, int cursor);
065  }
066  
067  public class WorkContext {
068    private JsonObject vd;
069    private Store store;
070    protected WorkContext(JsonObject vd) {
071      super();
072      this.vd = vd;
073    }
074    
075  }
076  private IWorkerContext context;
077  private Provider provider;
078  private Storage storage;
079  private IRunnerObserver observer;
080  private List<String> prohibitedNames = new ArrayList<String>();
081  private FHIRPathEngine fpe;
082
083  private String resourceName;
084  private List<ValidationMessage> issues;
085  private int resCount;
086
087
088  public IWorkerContext getContext() {
089    return context;
090  }
091  public void setContext(IWorkerContext context) {
092    this.context = context;
093  }
094
095  public Provider getProvider() {
096    return provider;
097  }
098  public void setProvider(Provider provider) {
099    this.provider = provider;
100  }
101
102  public Storage getStorage() {
103    return storage;
104  }
105  public void setStorage(Storage storage) {
106    this.storage = storage;
107  }
108
109  public List<String> getProhibitedNames() {
110    return prohibitedNames;
111  }
112
113  public void execute(JsonObject viewDefinition) {
114    execute("$", viewDefinition);
115  }
116
117  public void execute(String path, JsonObject viewDefinition) {
118    WorkContext wc = prepare(path, viewDefinition);
119    try {
120      evaluate(wc);
121    } finally {
122      finish(wc);
123    }
124  }
125
126  private void evaluate(WorkContext wc) {
127    List<Base> data = provider.fetch(resourceName);
128
129    int i = 0;
130    for (Base b : data) {
131      if (observer != null) {
132        observer.handleRow(b, data.size(), i);
133      }
134      processResource(wc.vd, wc.store, b);
135      i++;
136    }
137  }
138
139  public WorkContext prepare(String path, JsonObject viewDefinition) {
140    WorkContext wc = new WorkContext(viewDefinition);
141    if (context == null) {
142      throw new FHIRException("No context provided");
143    }
144    fpe = new FHIRPathEngine(context);
145    fpe.setHostServices(this);
146    fpe.setEmitSQLonFHIRWarning(true);
147    if (viewDefinition == null) {
148      throw new FHIRException("No viewDefinition provided");
149    }
150    if (provider == null) {
151      throw new FHIRException("No provider provided");
152    }
153    if (storage == null) {
154      throw new FHIRException("No storage provided");
155    }
156    Validator validator = new Validator(context, fpe, prohibitedNames, storage.supportsArrays(), storage.supportsComplexTypes(), storage.needsName());
157    validator.checkViewDefinition(path, viewDefinition);
158    issues = validator.getIssues();
159    validator.dump();
160    validator.check();
161    resourceName = validator.getResourceName();
162    wc.store = storage.createStore(wc.vd.asString("name"), (List<Column>) wc.vd.getUserData("columns"));
163    return wc;
164  }
165  
166  public void processResource(WorkContext wc, Base b) {
167    if (observer != null) {
168      observer.handleRow(b, -1, resCount);
169    }
170    processResource(wc.vd, wc.store, b);
171    resCount++;
172    wc.store.flush();
173  }
174  
175  private void processResource(JsonObject vd, Store store, Base b) {
176    boolean ok = true;
177    for (JsonObject w : vd.getJsonObjects("where")) {
178      String expr = w.asString("path");
179      ExpressionNode node = fpe.parse(expr);
180      boolean pass = fpe.evaluateToBoolean(vd, b, b, b, node);
181      if (!pass) {
182        ok = false;
183        break;
184      }  
185    }
186    if (ok) {
187      List<List<Cell>> rows = new ArrayList<>();
188      rows.add(new ArrayList<Cell>());
189
190      for (JsonObject select : vd.getJsonObjects("select")) {
191        executeSelect(vd, select, b, rows);
192      }
193      for (List<Cell> row : rows) {
194        storage.addRow(store, row);
195      }
196    }
197  }
198  
199  public void finish(WorkContext wc) {
200    storage.finish(wc.store);
201  }
202
203  private void executeSelect(JsonObject vd, JsonObject select, Base b, List<List<Cell>> rows) {
204    List<Base> focus = new ArrayList<>();
205
206    if (select.has("forEach")) {
207      focus.addAll(executeForEach(vd, select, b));
208    } else if (select.has("forEachOrNull")) {
209
210      focus.addAll(executeForEachOrNull(vd, select, b));  
211      if (focus.isEmpty()) {
212        List<Column> columns = (List<Column>) select.getUserData("columns");
213        for (List<Cell> row : rows) {
214          for (Column c : columns) {
215            Cell cell = cell(row, c.getName());
216            if (cell == null) {
217              row.add(new Cell(c, null));
218            }
219          }
220        }
221        return;
222      }
223    } else {
224      focus.add(b);
225    }
226
227    //  } else if (select.has("unionAll")) {
228    //    focus.addAll(executeUnion(select, b));
229
230    List<List<Cell>> tempRows = new ArrayList<>();
231    tempRows.addAll(rows);
232    rows.clear();
233
234    for (Base f : focus) {
235      List<List<Cell>> rowsToAdd = cloneRows(tempRows);  
236
237      for (JsonObject column : select.getJsonObjects("column")) {
238        executeColumn(vd, column, f, rowsToAdd);
239      }
240
241      for (JsonObject sub : select.getJsonObjects("select")) {
242        executeSelect(vd, sub, f, rowsToAdd);
243      }
244
245      executeUnionAll(vd, select.getJsonObjects("unionAll"), f, rowsToAdd);
246
247      rows.addAll(rowsToAdd);
248    }
249  }
250
251  private void executeUnionAll(JsonObject vd, List<JsonObject> unionList,  Base b, List<List<Cell>> rows) {
252    if (unionList.isEmpty()) {
253      return;
254    }
255    List<List<Cell>> sourceRows = new ArrayList<>();
256    sourceRows.addAll(rows);
257    rows.clear();
258
259    for (JsonObject union : unionList) {
260      List<List<Cell>> tempRows = new ArrayList<>();
261      tempRows.addAll(sourceRows);      
262      executeSelect(vd, union, b, tempRows);
263      rows.addAll(tempRows);
264    }
265  }
266
267  private List<List<Cell>> cloneRows(List<List<Cell>> rows) {
268    List<List<Cell>> list = new ArrayList<>();
269    for (List<Cell> row : rows) {
270      list.add(cloneRow(row));
271    }
272    return list;
273  }
274
275  private List<Cell> cloneRow(List<Cell> cells) {
276    List<Cell> list = new ArrayList<>();
277    for (Cell cell : cells) {
278      list.add(cell.copy());
279    }
280    return list;
281  }
282
283  private List<Base> executeForEach(JsonObject vd, JsonObject focus, Base b) {
284    ExpressionNode n = (ExpressionNode) focus.getUserData("forEach");
285    List<Base> result = new ArrayList<>();
286    result.addAll(fpe.evaluate(vd, b, n));
287    return result;  
288  }
289
290  private List<Base> executeForEachOrNull(JsonObject vd, JsonObject focus, Base b) {
291    ExpressionNode n = (ExpressionNode) focus.getUserData("forEachOrNull");
292    List<Base> result = new ArrayList<>();
293    result.addAll(fpe.evaluate(vd, b, n));
294    return result;  
295  }
296
297  private void executeColumn(JsonObject vd, JsonObject column, Base b, List<List<Cell>> rows) {
298    ExpressionNode n = (ExpressionNode) column.getUserData("path");
299    List<Base> bl2 = new ArrayList<>();
300    if (b != null) {
301      bl2.addAll(fpe.evaluate(vd, b, n));
302    }
303    Column col = (Column) column.getUserData("column");
304    if (col == null) {
305      System.out.println("Error");
306    } else {
307      for (List<Cell> row : rows) {
308        Cell c = cell(row, col.getName());
309        if (c == null) {
310          c = new Cell(col);
311          row.add(c);
312        }      
313        if (!bl2.isEmpty()) {
314          if (bl2.size() + c.getValues().size() > 1) {
315            // this is a problem if collection != true or if the storage can't deal with it 
316            // though this should've been picked up before now - but there are circumstances where it wouldn't be
317            if (!c.getColumn().isColl()) {
318              throw new FHIRException("The column "+c.getColumn().getName()+" is not allowed multiple values, but at least one row has multiple values");
319            }
320          }
321          for (Base b2 : bl2) {
322            c.getValues().add(genValue(c.getColumn(), b2));
323          }
324        }
325      }
326    }
327  }
328
329
330  private Value genValue(Column column, Base b) {
331    if (column.getKind() == null) {
332      throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type (null) for column "+column.getName()); // can't happen
333    }
334    switch (column.getKind()) {
335    case Binary:
336      if (b instanceof Base64BinaryType) {
337        Base64BinaryType bb = (Base64BinaryType) b;
338        return Value.makeBinary(bb.primitiveValue(), bb.getValue());
339      } else if (b.isBooleanPrimitive()) { // ElementModel
340        return Value.makeBinary(b.primitiveValue(), Base64.decodeBase64(b.primitiveValue()));
341      } else {
342        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a binary column for column "+column.getName());
343      }
344    case Boolean:
345      if (b instanceof BooleanType) {
346        BooleanType bb = (BooleanType) b;
347        return Value.makeBoolean(bb.primitiveValue(), bb.booleanValue());
348      } else if (b.isBooleanPrimitive()) { // ElementModel
349        return Value.makeBoolean(b.primitiveValue(), "true".equals(b.primitiveValue()));
350      } else {
351        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a boolean column for column "+column.getName());
352      }
353    case Complex:
354      if (b.isPrimitive()) {
355        throw new FHIRException("Attempt to add a primitive type "+b.fhirType()+" to a complex column for column "+column.getName());
356      } else {
357        return Value.makeComplex(b);
358      }
359    case DateTime:
360      if (b instanceof BaseDateTimeType) {
361        BaseDateTimeType d = (BaseDateTimeType) b;
362        return Value.makeDate(d.primitiveValue(), d.getValue());
363      } else if (b.isPrimitive() && b.isDateTime()) { // ElementModel
364        return Value.makeDate(b.primitiveValue(), b.dateTimeValue().getValue());
365      } else {
366        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
367      }
368    case Decimal:
369      if (b instanceof DecimalType) {
370        DecimalType d = (DecimalType) b;
371        return Value.makeDecimal(d.primitiveValue(), d.getValue());
372      } else if (b.isPrimitive()) { // ElementModel
373        return Value.makeDecimal(b.primitiveValue(), new BigDecimal(b.primitiveValue()));
374      } else {
375        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
376      }
377    case Integer:
378      if (b instanceof IntegerType) {
379        IntegerType i = (IntegerType) b;
380        return Value.makeInteger(i.primitiveValue(), i.getValue());
381      } else if (b.isPrimitive()) { // ElementModel
382        return Value.makeInteger(b.primitiveValue(), Integer.valueOf(b.primitiveValue()));
383      } else {
384        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
385      }
386    case String: 
387      if (b.isPrimitive()) {
388        return Value.makeString(b.primitiveValue());
389      } else {
390        throw new FHIRException("Attempt to add a complex type "+b.fhirType()+" to a string column for column "+column.getName());
391      }
392    case Time:
393      if (b.fhirType().equals("time")) {
394        return Value.makeString(b.primitiveValue());
395      } else {
396        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a time column for column "+column.getName());
397      }
398    default:
399      throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type for column "+column.getName());
400    }
401  }
402
403  private Column column(String columnName, List<Column> columns) {
404    for (Column t : columns) {
405      if (t.getName().equalsIgnoreCase(columnName)) {
406        return t;
407      }
408    }
409    return null;
410  }
411
412  private Cell cell(List<Cell> cells, String columnName) {
413    for (Cell t : cells) {
414      if (t.getColumn().getName().equalsIgnoreCase(columnName)) {
415        return t;
416      }
417    }
418    return null;
419  }
420
421  @Override
422  public List<Base> resolveConstant(FHIRPathEngine engine, Object appContext, String name, boolean beforeContext, boolean explicitConstant) throws PathEngineException {
423    List<Base> list = new ArrayList<Base>();
424    if (explicitConstant) {
425      JsonObject vd = (JsonObject) appContext;
426      JsonObject constant = findConstant(vd, name);
427      if (constant != null) {
428        Base b = (Base) constant.getUserData("value");
429        if (b != null) {
430          list.add(b);
431        }
432      }
433    }
434    return list;    
435  }
436
437  @Override
438  public TypeDetails resolveConstantType(FHIRPathEngine engine, Object appContext, String name, boolean explicitConstant) throws PathEngineException {
439    if (explicitConstant) {
440      JsonObject vd = (JsonObject) appContext;
441      JsonObject constant = findConstant(vd, name.substring(1));
442      if (constant != null) {
443        Base b = (Base) constant.getUserData("value");
444        if (b != null) {
445          return new TypeDetails(CollectionStatus.SINGLETON, b.fhirType());
446        }
447      }
448    }
449    return null;
450  }
451
452  private JsonObject findConstant(JsonObject vd, String name) {
453    for (JsonObject o : vd.getJsonObjects("constant")) {
454      if (name.equals(o.asString("name"))) {
455        return o;
456      }
457    }
458    return null;
459  }
460  @Override
461  public boolean log(String argument, List<Base> focus) {
462    throw new Error("Not implemented yet: log");
463  }
464
465  @Override
466  public FunctionDetails resolveFunction(FHIRPathEngine engine, String functionName) {
467    switch (functionName) {
468    case "getResourceKey" : return new FunctionDetails("Unique Key for resource", 0, 0);
469    case "getReferenceKey" : return new FunctionDetails("Unique Key for resource that is the target of the reference", 0, 1);
470    default:  return null;
471    }
472  }
473  @Override
474  public TypeDetails checkFunction(FHIRPathEngine engine, Object appContext, String functionName, TypeDetails focus, List<TypeDetails> parameters) throws PathEngineException {
475    switch (functionName) {
476    case "getResourceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string");
477    case "getReferenceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string");
478    default: throw new Error("Not known: "+functionName);
479    }
480  }
481
482  @Override
483  public List<Base> executeFunction(FHIRPathEngine engine, Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
484    switch (functionName) {
485    case "getResourceKey" : return executeResourceKey(focus);
486    case "getReferenceKey" : return executeReferenceKey(null, focus, parameters);
487    default: throw new Error("Not known: "+functionName);
488    }
489  }
490
491  private List<Base> executeResourceKey(List<Base> focus) {
492    List<Base> base = new ArrayList<Base>();
493    if (focus.size() == 1) {
494      Base res = focus.get(0);
495      if (!res.hasUserData("Storage.key")) {
496        String key = storage.getKeyForSourceResource(res);
497        if (key == null) {
498          throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase());
499        } else {
500          res.setUserData("Storage.key", key);
501        }
502      }
503      base.add(new StringType(res.getUserString("Storage.key")));
504    }
505    return base;
506  }
507
508  private List<Base> executeReferenceKey(Base rootResource, List<Base> focus, List<List<Base>> parameters) {
509    String rt = null;
510    if (parameters.size() > 0) {
511      rt = parameters.get(0).get(0).primitiveValue();
512      if (rt.startsWith("FHIR.")) {
513        rt = rt.substring(5);
514      }
515    }
516    List<Base> base = new ArrayList<Base>();
517    if (focus.size() == 1) {
518      Base res = focus.get(0);
519      String ref = null;
520      if (res.fhirType().equals("Reference")) {
521        ref = getRef(res);
522      } else if (res.isPrimitive()) {
523        ref = res.primitiveValue();
524      } else {
525        throw new FHIRException("Unable to generate a reference key based on a "+res.fhirType());
526      }
527      if (ref !=  null) {
528        Base target = provider.resolveReference(rootResource, ref, rt);
529        if (target != null) {
530          if (!res.hasUserData("Storage.key")) {
531            String key = storage.getKeyForTargetResource(target);
532            if (key == null) {
533              throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase());
534            } else {
535              res.setUserData("Storage.key", key);
536            }
537          }
538          base.add(new StringType(res.getUserString("Storage.key")));
539        }
540      }
541    }
542    return base;
543  }
544
545  private String getRef(Base res) {
546    Property prop = res.getChildByName("reference");
547    if (prop != null && prop.getValues().size() == 1) {
548      return prop.getValues().get(0).primitiveValue();
549    }
550    return null;
551  }
552
553  @Override
554  public Base resolveReference(FHIRPathEngine engine, Object appContext, String url, Base refContext) throws FHIRException {
555    throw new Error("Not implemented yet: resolveReference");
556  }
557
558  @Override
559  public boolean conformsToProfile(FHIRPathEngine engine, Object appContext, Base item, String url) throws FHIRException {
560    throw new Error("Not implemented yet: conformsToProfile");
561  }
562
563  @Override
564  public ValueSet resolveValueSet(FHIRPathEngine engine, Object appContext, String url) {
565    throw new Error("Not implemented yet: resolveValueSet");
566  }
567  @Override
568  public boolean paramIsType(String name, int index) {
569    return "getReferenceKey".equals(name);
570  }
571  public List<ValidationMessage> getIssues() {
572    return issues;
573  }
574
575
576}