001package org.hl7.fhir.r4.utils.sql;
002
003import java.math.BigDecimal;
004import java.util.ArrayList;
005import java.util.List;
006
007import lombok.extern.slf4j.Slf4j;
008import org.apache.commons.codec.binary.Base64;
009import org.hl7.fhir.exceptions.FHIRException;
010import org.hl7.fhir.exceptions.PathEngineException;
011import org.hl7.fhir.r4.context.IWorkerContext;
012import org.hl7.fhir.r4.fhirpath.ExpressionNode;
013import org.hl7.fhir.r4.fhirpath.FHIRPathEngine;
014import org.hl7.fhir.r4.fhirpath.TypeDetails;
015import org.hl7.fhir.r4.fhirpath.ExpressionNode.CollectionStatus;
016import org.hl7.fhir.r4.fhirpath.FHIRPathEngine.IEvaluationContext;
017import org.hl7.fhir.r4.fhirpath.FHIRPathUtilityClasses.FunctionDetails;
018import org.hl7.fhir.r4.model.Base;
019import org.hl7.fhir.r4.model.Base64BinaryType;
020import org.hl7.fhir.r4.model.BaseDateTimeType;
021import org.hl7.fhir.r4.model.BooleanType;
022import org.hl7.fhir.r4.model.DecimalType;
023import org.hl7.fhir.r4.model.IntegerType;
024import org.hl7.fhir.r4.model.Property;
025import org.hl7.fhir.r4.model.StringType;
026import org.hl7.fhir.r4.model.ValueSet;
027import org.hl7.fhir.r4.utils.sql.Cell;
028import org.hl7.fhir.r4.utils.sql.Column;
029import org.hl7.fhir.r4.utils.sql.Provider;
030import org.hl7.fhir.r4.utils.sql.Storage;
031import org.hl7.fhir.r4.utils.sql.Store;
032import org.hl7.fhir.r4.utils.sql.Validator;
033import org.hl7.fhir.r4.utils.sql.Value;
034import org.hl7.fhir.utilities.MarkedToMoveToAdjunctPackage;
035import org.hl7.fhir.utilities.json.model.JsonObject;
036import org.hl7.fhir.utilities.validation.ValidationMessage;
037
038
039/**
040 * How to use the Runner:
041 * 
042 * create a resource, and fill out:
043 *   the context (supports the FHIRPathEngine)
044 *   a store that handles the output 
045 *   a tracker - if you want
046 *   
047 * Once it's created, you either run it as a batch, or in trickle mode
048 * 
049 *   (1) Batch Mode
050 *   
051 *    * provide a provider 
052 *    * call execute() with a ViewDefinition
053 *    * wait... (watch with an observer if you want to track progress)
054 *   
055 *   (2) Trickle Mode
056 *    * call 'prepare', and keep the WorkContext that's returned
057 *    * each time there's a resource to process, call processResource and pass in the workContext and the resource
058 *    * when done, call finish(WorkContext)
059 */
060
061@MarkedToMoveToAdjunctPackage
062@Slf4j
063public class Runner implements IEvaluationContext {
064  
065  public interface IRunnerObserver {
066    public void handleRow(Base resource, int total, int cursor);
067  }
068  
069  public class WorkContext {
070    private JsonObject vd;
071    private Store store;
072    protected WorkContext(JsonObject vd) {
073      super();
074      this.vd = vd;
075    }
076    
077  }
078  private IWorkerContext context;
079  private Provider provider;
080  private Storage storage;
081  private IRunnerObserver observer;
082  private List<String> prohibitedNames = new ArrayList<String>();
083  private FHIRPathEngine fpe;
084
085  private String resourceName;
086  private List<ValidationMessage> issues;
087  private int resCount;
088
089
090  public IWorkerContext getContext() {
091    return context;
092  }
093  public void setContext(IWorkerContext context) {
094    this.context = context;
095  }
096
097  public Provider getProvider() {
098    return provider;
099  }
100  public void setProvider(Provider provider) {
101    this.provider = provider;
102  }
103
104  public Storage getStorage() {
105    return storage;
106  }
107  public void setStorage(Storage storage) {
108    this.storage = storage;
109  }
110
111  public List<String> getProhibitedNames() {
112    return prohibitedNames;
113  }
114
115  public void execute(JsonObject viewDefinition) {
116    execute("$", viewDefinition);
117  }
118
119  public void execute(String path, JsonObject viewDefinition) {
120    WorkContext wc = prepare(path, viewDefinition);
121    try {
122      evaluate(wc);
123    } finally {
124      finish(wc);
125    }
126  }
127
128  private void evaluate(WorkContext wc) {
129    List<Base> data = provider.fetch(resourceName);
130
131    int i = 0;
132    for (Base b : data) {
133      if (observer != null) {
134        observer.handleRow(b, data.size(), i);
135      }
136      processResource(wc.vd, wc.store, b);
137      i++;
138    }
139  }
140
141  public WorkContext prepare(String path, JsonObject viewDefinition) {
142    WorkContext wc = new WorkContext(viewDefinition);
143    if (context == null) {
144      throw new FHIRException("No context provided");
145    }
146    fpe = new FHIRPathEngine(context);
147    fpe.setHostServices(this);
148    fpe.setEmitSQLonFHIRWarning(true);
149    if (viewDefinition == null) {
150      throw new FHIRException("No viewDefinition provided");
151    }
152    if (provider == null) {
153      throw new FHIRException("No provider provided");
154    }
155    if (storage == null) {
156      throw new FHIRException("No storage provided");
157    }
158    Validator validator = new Validator(context, fpe, prohibitedNames, storage.supportsArrays(), storage.supportsComplexTypes(), storage.needsName());
159    validator.checkViewDefinition(path, viewDefinition);
160    issues = validator.getIssues();
161    validator.dump();
162    validator.check();
163    resourceName = validator.getResourceName();
164    wc.store = storage.createStore(wc.vd.asString("name"), (List<Column>) wc.vd.getUserData("columns"));
165    return wc;
166  }
167  
168  public void processResource(WorkContext wc, Base b) {
169    if (observer != null) {
170      observer.handleRow(b, -1, resCount);
171    }
172    processResource(wc.vd, wc.store, b);
173    resCount++;
174    wc.store.flush();
175  }
176  
177  private void processResource(JsonObject vd, Store store, Base b) {
178    boolean ok = true;
179    for (JsonObject w : vd.getJsonObjects("where")) {
180      String expr = w.asString("path");
181      ExpressionNode node = fpe.parse(expr);
182      boolean pass = fpe.evaluateToBoolean(vd, b, b, b, node);
183      if (!pass) {
184        ok = false;
185        break;
186      }  
187    }
188    if (ok) {
189      List<List<Cell>> rows = new ArrayList<>();
190      rows.add(new ArrayList<Cell>());
191
192      for (JsonObject select : vd.getJsonObjects("select")) {
193        executeSelect(vd, select, b, rows);
194      }
195      for (List<Cell> row : rows) {
196        storage.addRow(store, row);
197      }
198    }
199  }
200  
201  public void finish(WorkContext wc) {
202    storage.finish(wc.store);
203  }
204
205  private void executeSelect(JsonObject vd, JsonObject select, Base b, List<List<Cell>> rows) {
206    List<Base> focus = new ArrayList<>();
207
208    if (select.has("forEach")) {
209      focus.addAll(executeForEach(vd, select, b));
210    } else if (select.has("forEachOrNull")) {
211
212      focus.addAll(executeForEachOrNull(vd, select, b));  
213      if (focus.isEmpty()) {
214        List<Column> columns = (List<Column>) select.getUserData("columns");
215        for (List<Cell> row : rows) {
216          for (Column c : columns) {
217            Cell cell = cell(row, c.getName());
218            if (cell == null) {
219              row.add(new Cell(c, null));
220            }
221          }
222        }
223        return;
224      }
225    } else {
226      focus.add(b);
227    }
228
229    //  } else if (select.has("unionAll")) {
230    //    focus.addAll(executeUnion(select, b));
231
232    List<List<Cell>> tempRows = new ArrayList<>();
233    tempRows.addAll(rows);
234    rows.clear();
235
236    for (Base f : focus) {
237      List<List<Cell>> rowsToAdd = cloneRows(tempRows);  
238
239      for (JsonObject column : select.getJsonObjects("column")) {
240        executeColumn(vd, column, f, rowsToAdd);
241      }
242
243      for (JsonObject sub : select.getJsonObjects("select")) {
244        executeSelect(vd, sub, f, rowsToAdd);
245      }
246
247      executeUnionAll(vd, select.getJsonObjects("unionAll"), f, rowsToAdd);
248
249      rows.addAll(rowsToAdd);
250    }
251  }
252
253  private void executeUnionAll(JsonObject vd, List<JsonObject> unionList,  Base b, List<List<Cell>> rows) {
254    if (unionList.isEmpty()) {
255      return;
256    }
257    List<List<Cell>> sourceRows = new ArrayList<>();
258    sourceRows.addAll(rows);
259    rows.clear();
260
261    for (JsonObject union : unionList) {
262      List<List<Cell>> tempRows = new ArrayList<>();
263      tempRows.addAll(sourceRows);      
264      executeSelect(vd, union, b, tempRows);
265      rows.addAll(tempRows);
266    }
267  }
268
269  private List<List<Cell>> cloneRows(List<List<Cell>> rows) {
270    List<List<Cell>> list = new ArrayList<>();
271    for (List<Cell> row : rows) {
272      list.add(cloneRow(row));
273    }
274    return list;
275  }
276
277  private List<Cell> cloneRow(List<Cell> cells) {
278    List<Cell> list = new ArrayList<>();
279    for (Cell cell : cells) {
280      list.add(cell.copy());
281    }
282    return list;
283  }
284
285  private List<Base> executeForEach(JsonObject vd, JsonObject focus, Base b) {
286    ExpressionNode n = (ExpressionNode) focus.getUserData("forEach");
287    List<Base> result = new ArrayList<>();
288    result.addAll(fpe.evaluate(vd, b, n));
289    return result;  
290  }
291
292  private List<Base> executeForEachOrNull(JsonObject vd, JsonObject focus, Base b) {
293    ExpressionNode n = (ExpressionNode) focus.getUserData("forEachOrNull");
294    List<Base> result = new ArrayList<>();
295    result.addAll(fpe.evaluate(vd, b, n));
296    return result;  
297  }
298
299  private void executeColumn(JsonObject vd, JsonObject column, Base b, List<List<Cell>> rows) {
300    ExpressionNode n = (ExpressionNode) column.getUserData("path");
301    List<Base> bl2 = new ArrayList<>();
302    if (b != null) {
303      bl2.addAll(fpe.evaluate(vd, b, n));
304    }
305    Column col = (Column) column.getUserData("column");
306    if (col == null) {
307      log.error("Error");
308    } else {
309      for (List<Cell> row : rows) {
310        Cell c = cell(row, col.getName());
311        if (c == null) {
312          c = new Cell(col);
313          row.add(c);
314        }      
315        if (!bl2.isEmpty()) {
316          if (bl2.size() + c.getValues().size() > 1) {
317            // this is a problem if collection != true or if the storage can't deal with it 
318            // though this should've been picked up before now - but there are circumstances where it wouldn't be
319            if (!c.getColumn().isColl()) {
320              throw new FHIRException("The column "+c.getColumn().getName()+" is not allowed multiple values, but at least one row has multiple values");
321            }
322          }
323          for (Base b2 : bl2) {
324            c.getValues().add(genValue(c.getColumn(), b2));
325          }
326        }
327      }
328    }
329  }
330
331
332  private Value genValue(Column column, Base b) {
333    if (column.getKind() == null) {
334      throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type (null) for column "+column.getName()); // can't happen
335    }
336    switch (column.getKind()) {
337    case Binary:
338      if (b instanceof Base64BinaryType) {
339        Base64BinaryType bb = (Base64BinaryType) b;
340        return Value.makeBinary(bb.primitiveValue(), bb.getValue());
341      } else if (b.isBooleanPrimitive()) { // ElementModel
342        return Value.makeBinary(b.primitiveValue(), Base64.decodeBase64(b.primitiveValue()));
343      } else {
344        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a binary column for column "+column.getName());
345      }
346    case Boolean:
347      if (b instanceof BooleanType) {
348        BooleanType bb = (BooleanType) b;
349        return Value.makeBoolean(bb.primitiveValue(), bb.booleanValue());
350      } else if (b.isBooleanPrimitive()) { // ElementModel
351        return Value.makeBoolean(b.primitiveValue(), "true".equals(b.primitiveValue()));
352      } else {
353        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a boolean column for column "+column.getName());
354      }
355    case Complex:
356      if (b.isPrimitive()) {
357        throw new FHIRException("Attempt to add a primitive type "+b.fhirType()+" to a complex column for column "+column.getName());
358      } else {
359        return Value.makeComplex(b);
360      }
361    case DateTime:
362      if (b instanceof BaseDateTimeType) {
363        BaseDateTimeType d = (BaseDateTimeType) b;
364        return Value.makeDate(d.primitiveValue(), d.getValue());
365      } else if (b.isPrimitive() && b.isDateTime()) { // ElementModel
366        return Value.makeDate(b.primitiveValue(), b.dateTimeValue().getValue());
367      } else {
368        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
369      }
370    case Decimal:
371      if (b instanceof DecimalType) {
372        DecimalType d = (DecimalType) b;
373        return Value.makeDecimal(d.primitiveValue(), d.getValue());
374      } else if (b.isPrimitive()) { // ElementModel
375        return Value.makeDecimal(b.primitiveValue(), new BigDecimal(b.primitiveValue()));
376      } else {
377        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
378      }
379    case Integer:
380      if (b instanceof IntegerType) {
381        IntegerType i = (IntegerType) b;
382        return Value.makeInteger(i.primitiveValue(), i.getValue());
383      } else if (b.isPrimitive()) { // ElementModel
384        return Value.makeInteger(b.primitiveValue(), Integer.valueOf(b.primitiveValue()));
385      } else {
386        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an integer column for column "+column.getName());
387      }
388    case String: 
389      if (b.isPrimitive()) {
390        return Value.makeString(b.primitiveValue());
391      } else {
392        throw new FHIRException("Attempt to add a complex type "+b.fhirType()+" to a string column for column "+column.getName());
393      }
394    case Time:
395      if (b.fhirType().equals("time")) {
396        return Value.makeString(b.primitiveValue());
397      } else {
398        throw new FHIRException("Attempt to add a type "+b.fhirType()+" to a time column for column "+column.getName());
399      }
400    default:
401      throw new FHIRException("Attempt to add a type "+b.fhirType()+" to an unknown column type for column "+column.getName());
402    }
403  }
404
405  private Column column(String columnName, List<Column> columns) {
406    for (Column t : columns) {
407      if (t.getName().equalsIgnoreCase(columnName)) {
408        return t;
409      }
410    }
411    return null;
412  }
413
414  private Cell cell(List<Cell> cells, String columnName) {
415    for (Cell t : cells) {
416      if (t.getColumn().getName().equalsIgnoreCase(columnName)) {
417        return t;
418      }
419    }
420    return null;
421  }
422
423  @Override
424  public List<Base> resolveConstant(FHIRPathEngine engine, Object appContext, String name, boolean beforeContext, boolean explicitConstant) throws PathEngineException {
425    List<Base> list = new ArrayList<Base>();
426    if (explicitConstant) {
427      JsonObject vd = (JsonObject) appContext;
428      JsonObject constant = findConstant(vd, name);
429      if (constant != null) {
430        Base b = (Base) constant.getUserData("value");
431        if (b != null) {
432          list.add(b);
433        }
434      }
435    }
436    return list;    
437  }
438
439  @Override
440  public TypeDetails resolveConstantType(FHIRPathEngine engine, Object appContext, String name, boolean explicitConstant) throws PathEngineException {
441    if (explicitConstant) {
442      JsonObject vd = (JsonObject) appContext;
443      JsonObject constant = findConstant(vd, name.substring(1));
444      if (constant != null) {
445        Base b = (Base) constant.getUserData("value");
446        if (b != null) {
447          return new TypeDetails(CollectionStatus.SINGLETON, b.fhirType());
448        }
449      }
450    }
451    return null;
452  }
453
454  private JsonObject findConstant(JsonObject vd, String name) {
455    for (JsonObject o : vd.getJsonObjects("constant")) {
456      if (name.equals(o.asString("name"))) {
457        return o;
458      }
459    }
460    return null;
461  }
462  @Override
463  public boolean log(String argument, List<Base> focus) {
464    throw new Error("Not implemented yet: log");
465  }
466
467  @Override
468  public FunctionDetails resolveFunction(FHIRPathEngine engine, String functionName) {
469    switch (functionName) {
470    case "getResourceKey" : return new FunctionDetails("Unique Key for resource", 0, 0);
471    case "getReferenceKey" : return new FunctionDetails("Unique Key for resource that is the target of the reference", 0, 1);
472    default:  return null;
473    }
474  }
475  @Override
476  public TypeDetails checkFunction(FHIRPathEngine engine, Object appContext, String functionName, TypeDetails focus, List<TypeDetails> parameters) throws PathEngineException {
477    switch (functionName) {
478    case "getResourceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string");
479    case "getReferenceKey" : return new TypeDetails(CollectionStatus.SINGLETON, "string");
480    default: throw new Error("Not known: "+functionName);
481    }
482  }
483
484  @Override
485  public List<Base> executeFunction(FHIRPathEngine engine, Object appContext, List<Base> focus, String functionName, List<List<Base>> parameters) {
486    switch (functionName) {
487    case "getResourceKey" : return executeResourceKey(focus);
488    case "getReferenceKey" : return executeReferenceKey(null, focus, parameters);
489    default: throw new Error("Not known: "+functionName);
490    }
491  }
492
493  private List<Base> executeResourceKey(List<Base> focus) {
494    List<Base> base = new ArrayList<Base>();
495    if (focus.size() == 1) {
496      Base res = focus.get(0);
497      if (!res.hasUserData("Storage.key")) {
498        String key = storage.getKeyForSourceResource(res);
499        if (key == null) {
500          throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase());
501        } else {
502          res.setUserData("Storage.key", key);
503        }
504      }
505      base.add(new StringType(res.getUserString("Storage.key")));
506    }
507    return base;
508  }
509
510  private List<Base> executeReferenceKey(Base rootResource, List<Base> focus, List<List<Base>> parameters) {
511    String rt = null;
512    if (parameters.size() > 0) {
513      rt = parameters.get(0).get(0).primitiveValue();
514      if (rt.startsWith("FHIR.")) {
515        rt = rt.substring(5);
516      }
517    }
518    List<Base> base = new ArrayList<Base>();
519    if (focus.size() == 1) {
520      Base res = focus.get(0);
521      String ref = null;
522      if (res.fhirType().equals("Reference")) {
523        ref = getRef(res);
524      } else if (res.isPrimitive()) {
525        ref = res.primitiveValue();
526      } else {
527        throw new FHIRException("Unable to generate a reference key based on a "+res.fhirType());
528      }
529      if (ref !=  null) {
530        Base target = provider.resolveReference(rootResource, ref, rt);
531        if (target != null) {
532          if (!res.hasUserData("Storage.key")) {
533            String key = storage.getKeyForTargetResource(target);
534            if (key == null) {
535              throw new FHIRException("Unidentified resource: "+res.fhirType()+"/"+res.getIdBase());
536            } else {
537              res.setUserData("Storage.key", key);
538            }
539          }
540          base.add(new StringType(res.getUserString("Storage.key")));
541        }
542      }
543    }
544    return base;
545  }
546
547  private String getRef(Base res) {
548    Property prop = res.getChildByName("reference");
549    if (prop != null && prop.getValues().size() == 1) {
550      return prop.getValues().get(0).primitiveValue();
551    }
552    return null;
553  }
554
555  @Override
556  public Base resolveReference(FHIRPathEngine engine, Object appContext, String url, Base refContext) throws FHIRException {
557    throw new Error("Not implemented yet: resolveReference");
558  }
559
560  @Override
561  public boolean conformsToProfile(FHIRPathEngine engine, Object appContext, Base item, String url) throws FHIRException {
562    throw new Error("Not implemented yet: conformsToProfile");
563  }
564
565  @Override
566  public ValueSet resolveValueSet(FHIRPathEngine engine, Object appContext, String url) {
567    throw new Error("Not implemented yet: resolveValueSet");
568  }
569  @Override
570  public boolean paramIsType(String name, int index) {
571    return "getReferenceKey".equals(name);
572  }
573  public List<ValidationMessage> getIssues() {
574    return issues;
575  }
576
577
578}