001package org.hl7.fhir.r4.utils.sql;
002
003import java.math.BigDecimal;
004import java.util.ArrayList;
005import java.util.List;
006
007import lombok.extern.slf4j.Slf4j;
008import org.apache.commons.codec.binary.Base64;
009import org.hl7.fhir.exceptions.FHIRException;
010import org.hl7.fhir.exceptions.PathEngineException;
011import org.hl7.fhir.r4.context.IWorkerContext;
012import org.hl7.fhir.r4.fhirpath.ExpressionNode;
013import org.hl7.fhir.r4.fhirpath.FHIRPathEngine;
014import org.hl7.fhir.r4.fhirpath.TypeDetails;
015import org.hl7.fhir.r4.fhirpath.ExpressionNode.CollectionStatus;
016import org.hl7.fhir.r4.fhirpath.IHostApplicationServices;
017import org.hl7.fhir.r4.fhirpath.FHIRPathUtilityClasses.FunctionDetails;
018import org.hl7.fhir.r4.model.Base;
019import org.hl7.fhir.r4.model.Base64BinaryType;
020import org.hl7.fhir.r4.model.BaseDateTimeType;
021import org.hl7.fhir.r4.model.BooleanType;
022import org.hl7.fhir.r4.model.DecimalType;
023import org.hl7.fhir.r4.model.IntegerType;
024import org.hl7.fhir.r4.model.Property;
025import org.hl7.fhir.r4.model.StringType;
026import org.hl7.fhir.r4.model.ValueSet;
027import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
028import org.hl7.fhir.utilities.fhirpath.FHIRPathConstantEvaluationMode;
029import org.hl7.fhir.utilities.json.model.JsonObject;
030import org.hl7.fhir.utilities.validation.ValidationMessage;
031
032
033/**
034 * How to use the Runner:
035 * 
036 * create a resource, and fill out:
037 *   the context (supports the FHIRPathEngine)
038 *   a store that handles the output 
039 *   a tracker - if you want
040 *   
041 * Once it's created, you either run it as a batch, or in trickle mode
042 * 
043 *   (1) Batch Mode
044 *   
045 *    * provide a provider 
046 *    * call execute() with a ViewDefinition
047 *    * wait... (watch with an observer if you want to track progress)
048 *   
049 *   (2) Trickle Mode
050 *    * call 'prepare', and keep the WorkContext that's returned
051 *    * each time there's a resource to process, call processResource and pass in the workContext and the resource
052 *    * when done, call finish(WorkContext)
053 */
054
055@MarkedToMoveToAdjunctPackage
056@Slf4j
057public class Runner implements IHostApplicationServices {
058  
059  public interface IRunnerObserver {
060    public void handleRow(Base resource, int total, int cursor);
061  }
062  
063  public class WorkContext {
064    private JsonObject vd;
065    private Store store;
066    protected WorkContext(JsonObject vd) {
067      super();
068      this.vd = vd;
069    }
070    
071  }
072  private IWorkerContext context;
073  private Provider provider;
074  private Storage storage;
075  private IRunnerObserver observer;
076  private List<String> prohibitedNames = new ArrayList<String>();
077  private FHIRPathEngine fpe;
078
079  private String resourceName;
080  private List<ValidationMessage> issues;
081  private int resCount;
082
083
084  public IWorkerContext getContext() {
085    return context;
086  }
087  public void setContext(IWorkerContext context) {
088    this.context = context;
089  }
090
091  public Provider getProvider() {
092    return provider;
093  }
094  public void setProvider(Provider provider) {
095    this.provider = provider;
096  }
097
098  public Storage getStorage() {
099    return storage;
100  }
101  public void setStorage(Storage storage) {
102    this.storage = storage;
103  }
104
105  public List<String> getProhibitedNames() {
106    return prohibitedNames;
107  }
108
109  public void execute(JsonObject viewDefinition) {
110    execute("$", viewDefinition);
111  }
112
113  public void execute(String path, JsonObject viewDefinition) {
114    WorkContext wc = prepare(path, viewDefinition);
115    try {
116      evaluate(wc);
117    } finally {
118      finish(wc);
119    }
120  }
121
122  private void evaluate(WorkContext wc) {
123    List<Base> data = provider.fetch(resourceName);
124
125    int i = 0;
126    for (Base b : data) {
127      if (observer != null) {
128        observer.handleRow(b, data.size(), i);
129      }
130      processResource(wc.vd, wc.store, b);
131      i++;
132    }
133  }
134
135  public WorkContext prepare(String path, JsonObject viewDefinition) {
136    WorkContext wc = new WorkContext(viewDefinition);
137    if (context == null) {
138      throw new FHIRException("No context provided");
139    }
140    fpe = new FHIRPathEngine(context);
141    fpe.setHostServices(this);
142    fpe.setEmitSQLonFHIRWarning(true);
143    if (viewDefinition == null) {
144      throw new FHIRException("No viewDefinition provided");
145    }
146    if (provider == null) {
147      throw new FHIRException("No provider provided");
148    }
149    if (storage == null) {
150      throw new FHIRException("No storage provided");
151    }
152    Validator validator = new Validator(context, fpe, prohibitedNames, storage.supportsArrays(), storage.supportsComplexTypes(), storage.needsName());
153    validator.checkViewDefinition(path, viewDefinition);
154    issues = validator.getIssues();
155    validator.dump();
156    validator.check();
157    resourceName = validator.getResourceName();
158    wc.store = storage.createStore(wc.vd.asString("name"), (List<Column>) wc.vd.getUserData("columns"));
159    return wc;
160  }
161  
162  public void processResource(WorkContext wc, Base b) {
163    if (observer != null) {
164      observer.handleRow(b, -1, resCount);
165    }
166    processResource(wc.vd, wc.store, b);
167    resCount++;
168    wc.store.flush();
169  }
170  
171  private void processResource(JsonObject vd, Store store, Base b) {
172    boolean ok = true;
173    for (JsonObject w : vd.getJsonObjects("where")) {
174      String expr = w.asString("path");
175      ExpressionNode node = fpe.parse(expr);
176      boolean pass = fpe.evaluateToBoolean(vd, b, b, b, node);
177      if (!pass) {
178        ok = false;
179        break;
180      }  
181    }
182    if (ok) {
183      List<List<Cell>> rows = new ArrayList<>();
184      rows.add(new ArrayList<Cell>());
185
186      for (JsonObject select : vd.getJsonObjects("select")) {
187        executeSelect(vd, select, b, rows);
188      }
189      for (List<Cell> row : rows) {
190        storage.addRow(store, row);
191      }
192    }
193  }
194  
195  public void finish(WorkContext wc) {
196    storage.finish(wc.store);
197  }
198
199  private void executeSelect(JsonObject vd, JsonObject select, Base b, List<List<Cell>> rows) {
200    List<Base> focus = new ArrayList<>();
201
202    if (select.has("forEach")) {
203      focus.addAll(executeForEach(vd, select, b));
204    } else if (select.has("forEachOrNull")) {
205
206      focus.addAll(executeForEachOrNull(vd, select, b));  
207      if (focus.isEmpty()) {
208        List<Column> columns = (List<Column>) select.getUserData("columns");
209        for (List<Cell> row : rows) {
210          for (Column c : columns) {
211            Cell cell = cell(row, c.getName());
212            if (cell == null) {
213              row.add(new Cell(c, null));
214            }
215          }
216        }
217        return;
218      }
219    } else {
220      focus.add(b);
221    }
222
223    //  } else if (select.has("unionAll")) {
224    //    focus.addAll(executeUnion(select, b));
225
226    List<List<Cell>> tempRows = new ArrayList<>();
227    tempRows.addAll(rows);
228    rows.clear();
229
230    for (Base f : focus) {
231      List<List<Cell>> rowsToAdd = cloneRows(tempRows);  
232
233      for (JsonObject column : select.getJsonObjects("column")) {
234        executeColumn(vd, column, f, rowsToAdd);
235      }
236
237      for (JsonObject sub : select.getJsonObjects("select")) {
238        executeSelect(vd, sub, f, rowsToAdd);
239      }
240
241      executeUnionAll(vd, select.getJsonObjects("unionAll"), f, rowsToAdd);
242
243      rows.addAll(rowsToAdd);
244    }
245  }
246
247  private void executeUnionAll(JsonObject vd, List<JsonObject> unionList,  Base b, List<List<Cell>> rows) {
248    if (unionList.isEmpty()) {
249      return;
250    }
251    List<List<Cell>> sourceRows = new ArrayList<>();
252    sourceRows.addAll(rows);
253    rows.clear();
254
255    for (JsonObject union : unionList) {
256      List<List<Cell>> tempRows = new ArrayList<>();
257      tempRows.addAll(sourceRows);      
258      executeSelect(vd, union, b, tempRows);
259      rows.addAll(tempRows);
260    }
261  }
262
263  private List<List<Cell>> cloneRows(List<List<Cell>> rows) {
264    List<List<Cell>> list = new ArrayList<>();
265    for (List<Cell> row : rows) {
266      list.add(cloneRow(row));
267    }
268    return list;
269  }
270
271  private List<Cell> cloneRow(List<Cell> cells) {
272    List<Cell> list = new ArrayList<>();
273    for (Cell cell : cells) {
274      list.add(cell.copy());
275    }
276    return list;
277  }
278
279  private List<Base> executeForEach(JsonObject vd, JsonObject focus, Base b) {
280    ExpressionNode n = (ExpressionNode) focus.getUserData("forEach");
281    List<Base> result = new ArrayList<>();
282    result.addAll(fpe.evaluate(vd, b, n));
283    return result;  
284  }
285
286  private List<Base> executeForEachOrNull(JsonObject vd, JsonObject focus, Base b) {
287    ExpressionNode n = (ExpressionNode) focus.getUserData("forEachOrNull");
288    List<Base> result = new ArrayList<>();
289    result.addAll(fpe.evaluate(vd, b, n));
290    return result;  
291  }
292
293  private void executeColumn(JsonObject vd, JsonObject column, Base b, List<List<Cell>> rows) {
294    ExpressionNode n = (ExpressionNode) column.getUserData("path");
295    List<Base> bl2 = new ArrayList<>();
296    if (b != null) {
297      bl2.addAll(fpe.evaluate(vd, b, n));
298    }
299    Column col = (Column) column.getUserData("column");
300    if (col == null) {
301      log.error("Error");
302    } else {
303      for (List<Cell> row : rows) {
304        Cell c = cell(row, col.getName());
305        if (c == null) {
306          c = new Cell(col);
307          row.add(c);
308        }      
309        if (!bl2.isEmpty()) {
310          if (bl2.size() + c.getValues().size() > 1) {
311            // this is a problem if collection != true or if the storage can't deal with it 
312            // though this should've been picked up before now - but there are circumstances where it wouldn't be
313            if (!c.getColumn().isColl()) {
314              throw new FHIRException("The column "+c.getColumn().getName()+" is not allowed multiple values, but at least one row has multiple values");
315            }
316          }
317          for (Base b2 : bl2) {
318            c.getValues().add(genValue(c.getColumn(), b2));
319          }
320        }
321      }
322    }
323  }
324
325
326  private Value genValue(Column column, Base b) {
327    if (column.getKind() == null) {
328      throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type (null) for column "+column.getName()); // can't happen
329    }
330    switch (column.getKind()) {
331    case Binary:
332      if (b instanceof Base64BinaryType) {
333        Base64BinaryType bb = (Base64BinaryType) b;
334        return Value.makeBinary(bb.primitiveValue(), bb.getValue());
335      } else if (b.isBooleanPrimitive()) { // ElementModel
336        return Value.makeBinary(b.primitiveValue(), Base64.decodeBase64(b.primitiveValue()));
337      } else {
338        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a binary column for column "+column.getName());
339      }
340    case Boolean:
341      if (b instanceof BooleanType) {
342        BooleanType bb = (BooleanType) b;
343        return Value.makeBoolean(bb.primitiveValue(), bb.booleanValue());
344      } else if (b.isBooleanPrimitive()) { // ElementModel
345        return Value.makeBoolean(b.primitiveValue(), "true".equals(b.primitiveValue()));
346      } else {
347        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a boolean column for column "+column.getName());
348      }
349    case Complex:
350      if (b.isPrimitive()) {
351        throw new FHIRException("Attempt to add a primitive type "+b.fhirType()+" to a complex column for column "+column.getName());
352      } else {
353        return Value.makeComplex(b);
354      }
355    case DateTime:
356      if (b instanceof BaseDateTimeType) {
357        BaseDateTimeType d = (BaseDateTimeType) b;
358        return Value.makeDate(d.primitiveValue(), d.getValue());
359      } else if (b.isPrimitive() && b.isDateTime()) { // ElementModel
360        return Value.makeDate(b.primitiveValue(), b.dateTimeValue().getValue());
361      } else {
362        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
363      }
364    case Decimal:
365      if (b instanceof DecimalType) {
366        DecimalType d = (DecimalType) b;
367        return Value.makeDecimal(d.primitiveValue(), d.getValue());
368      } else if (b.isPrimitive()) { // ElementModel
369        return Value.makeDecimal(b.primitiveValue(), new BigDecimal(b.primitiveValue()));
370      } else {
371        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
372      }
373    case Integer:
374      if (b instanceof IntegerType) {
375        IntegerType i = (IntegerType) b;
376        return Value.makeInteger(i.primitiveValue(), i.getValue());
377      } else if (b.isPrimitive()) { // ElementModel
378        return Value.makeInteger(b.primitiveValue(), Integer.valueOf(b.primitiveValue()));
379      } else {
380        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
381      }
382    case String: 
383      if (b.isPrimitive()) {
384        return Value.makeString(b.primitiveValue());
385      } else {
386        throw new FHIRException("Attempt to add a complex type "+b.fhirType()+" to a string column for column "+column.getName());
387      }
388    case Time:
389      if (b.fhirType().equals("time")) {
390        return Value.makeString(b.primitiveValue());
391      } else {
392        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a time column for column "+column.getName());
393      }
394    default:
395      throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type for column "+column.getName());
396    }
397  }
398
399  private Column column(String columnName, List<Column> columns) {
400    for (Column t : columns) {
401      if (t.getName().equalsIgnoreCase(columnName)) {
402        return t;
403      }
404    }
405    return null;
406  }
407
408  private Cell cell(List<Cell> cells, String columnName) {
409    for (Cell t : cells) {
410      if (t.getColumn().getName().equalsIgnoreCase(columnName)) {
411        return t;
412      }
413    }
414    return null;
415  }
416
417  @Override
418  public List<Base> resolveConstant(FHIRPathEngine engine, Object appContext, String name, FHIRPathConstantEvaluationMode mode) throws PathEngineException {
419    List<Base> list = new ArrayList<Base>();
420    if (mode == FHIRPathConstantEvaluationMode.EXPLICIT) {
421      JsonObject vd = (JsonObject) appContext;
422      JsonObject constant = findConstant(vd, name);
423      if (constant != null) {
424        Base b = (Base) constant.getUserData("value");
425        if (b != null) {
426          list.add(b);
427        }
428      }
429    }
430    return list;    
431  }
432
433  @Override
434  public TypeDetails resolveConstantType(FHIRPathEngine engine, Object appContext, String name, FHIRPathConstantEvaluationMode mode) throws PathEngineException {
435    if (mode == FHIRPathConstantEvaluationMode.EXPLICIT) {
436      JsonObject vd = (JsonObject) appContext;
437      JsonObject constant = findConstant(vd, name.substring(1));
438      if (constant != null) {
439        Base b = (Base) constant.getUserData("value");
440        if (b != null) {
441          return new TypeDetails(CollectionStatus.SINGLETON, b.fhirType());
442        }
443      }
444    }
445    return null;
446  }
447
448  private JsonObject findConstant(JsonObject vd, String name) {
449    for (JsonObject o : vd.getJsonObjects("constant")) {
450      if (name.equals(o.asString("name"))) {
451        return o;
452      }
453    }
454    return null;
455  }
456  @Override
457  public boolean log(String argument, List<Base> focus) {
458    throw new Error("Not implemented yet: log");
459  }
460
461  @Override
462  public FunctionDetails resolveFunction(FHIRPathEngine engine, String functionName) {
463    switch (functionName) {
464    case "getResourceKey" : return new FunctionDetails("Unique Key for resource", 0, 0);
465    case "getReferenceKey" : return new FunctionDetails("Unique Key for resource that is the target of the reference", 0, 1);
466    default:  return null;
467    }
468  }
469  @Override
470  public TypeDetails checkFunction(FHIRPathEngine engine, Object appContext, String functionName, TypeDetails focus, List<TypeDetails> parameters) throws PathEngineException {
471    switch (functionName) {
472    case "getResourceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string");
473    case "getReferenceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string");
474    default: throw new Error("Not known: "+functionName);
475    }
476  }
477
478  @Override
479  public List<Base> executeFunction(FHIRPathEngine engine, Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
480    switch (functionName) {
481    case "getResourceKey" : return executeResourceKey(focus);
482    case "getReferenceKey" : return executeReferenceKey(null, focus, parameters);
483    default: throw new Error("Not known: "+functionName);
484    }
485  }
486
487  private List<Base> executeResourceKey(List<Base> focus) {
488    List<Base> base = new ArrayList<Base>();
489    if (focus.size() == 1) {
490      Base res = focus.get(0);
491      if (!res.hasUserData("Storage.key")) {
492        String key = storage.getKeyForSourceResource(res);
493        if (key == null) {
494          throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase());
495        } else {
496          res.setUserData("Storage.key", key);
497        }
498      }
499      base.add(new StringType(res.getUserString("Storage.key")));
500    }
501    return base;
502  }
503
504  private List<Base> executeReferenceKey(Base rootResource, List<Base> focus, List<List<Base>> parameters) {
505    String rt = null;
506    if (parameters.size() > 0) {
507      rt = parameters.get(0).get(0).primitiveValue();
508      if (rt.startsWith("FHIR.")) {
509        rt = rt.substring(5);
510      }
511    }
512    List<Base> base = new ArrayList<Base>();
513    if (focus.size() == 1) {
514      Base res = focus.get(0);
515      String ref = null;
516      if (res.fhirType().equals("Reference")) {
517        ref = getRef(res);
518      } else if (res.isPrimitive()) {
519        ref = res.primitiveValue();
520      } else {
521        throw new FHIRException("Unable to generate a reference key based on a "+res.fhirType());
522      }
523      if (ref !=  null) {
524        Base target = provider.resolveReference(rootResource, ref, rt);
525        if (target != null) {
526          if (!res.hasUserData("Storage.key")) {
527            String key = storage.getKeyForTargetResource(target);
528            if (key == null) {
529              throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase());
530            } else {
531              res.setUserData("Storage.key", key);
532            }
533          }
534          base.add(new StringType(res.getUserString("Storage.key")));
535        }
536      }
537    }
538    return base;
539  }
540
541  private String getRef(Base res) {
542    Property prop = res.getChildByName("reference");
543    if (prop != null && prop.getValues().size() == 1) {
544      return prop.getValues().get(0).primitiveValue();
545    }
546    return null;
547  }
548
549  @Override
550  public Base resolveReference(FHIRPathEngine engine, Object appContext, String url, Base refContext) throws FHIRException {
551    throw new Error("Not implemented yet: resolveReference");
552  }
553
554  @Override
555  public boolean conformsToProfile(FHIRPathEngine engine, Object appContext, Base item, String url) throws FHIRException {
556    throw new Error("Not implemented yet: conformsToProfile");
557  }
558
559  @Override
560  public ValueSet resolveValueSet(FHIRPathEngine engine, Object appContext, String url) {
561    throw new Error("Not implemented yet: resolveValueSet");
562  }
563  @Override
564  public boolean paramIsType(String name, int index) {
565    return "getReferenceKey".equals(name);
566  }
567  public List<ValidationMessage> getIssues() {
568    return issues;
569  }
570
571
572}