001package ca.uhn.fhir.jpa.search.lastn;
002
003/*-
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.i18n.Msg;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.jpa.dao.TolerantJsonParser;
026import ca.uhn.fhir.jpa.model.config.PartitionSettings;
027import ca.uhn.fhir.jpa.model.entity.IBaseResourceEntity;
028import ca.uhn.fhir.jpa.model.util.CodeSystemHash;
029import ca.uhn.fhir.jpa.search.lastn.json.CodeJson;
030import ca.uhn.fhir.jpa.search.lastn.json.ObservationJson;
031import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
032import ca.uhn.fhir.jpa.searchparam.util.LastNParameterHelper;
033import ca.uhn.fhir.model.api.IQueryParameterType;
034import ca.uhn.fhir.parser.IParser;
035import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
036import ca.uhn.fhir.rest.param.DateParam;
037import ca.uhn.fhir.rest.param.ParamPrefixEnum;
038import ca.uhn.fhir.rest.param.ReferenceParam;
039import ca.uhn.fhir.rest.param.TokenParam;
040import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
041import com.fasterxml.jackson.core.JsonProcessingException;
042import com.fasterxml.jackson.databind.ObjectMapper;
043import com.google.common.annotations.VisibleForTesting;
044import org.apache.commons.lang3.Validate;
045import org.elasticsearch.action.DocWriteResponse;
046import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
047import org.elasticsearch.action.index.IndexRequest;
048import org.elasticsearch.action.index.IndexResponse;
049import org.elasticsearch.action.search.SearchRequest;
050import org.elasticsearch.action.search.SearchResponse;
051import org.elasticsearch.client.RequestOptions;
052import org.elasticsearch.client.RestHighLevelClient;
053import org.elasticsearch.client.indices.CreateIndexRequest;
054import org.elasticsearch.client.indices.CreateIndexResponse;
055import org.elasticsearch.client.indices.GetIndexRequest;
056import org.elasticsearch.index.query.BoolQueryBuilder;
057import org.elasticsearch.index.query.MatchQueryBuilder;
058import org.elasticsearch.index.query.QueryBuilders;
059import org.elasticsearch.index.query.RangeQueryBuilder;
060import org.elasticsearch.index.reindex.DeleteByQueryRequest;
061import org.elasticsearch.search.SearchHit;
062import org.elasticsearch.search.SearchHits;
063import org.elasticsearch.search.aggregations.AggregationBuilder;
064import org.elasticsearch.search.aggregations.AggregationBuilders;
065import org.elasticsearch.search.aggregations.Aggregations;
066import org.elasticsearch.search.aggregations.BucketOrder;
067import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder;
068import org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder;
069import org.elasticsearch.search.aggregations.bucket.composite.ParsedComposite;
070import org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder;
071import org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms;
072import org.elasticsearch.search.aggregations.bucket.terms.Terms;
073import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
074import org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
075import org.elasticsearch.search.builder.SearchSourceBuilder;
076import org.elasticsearch.search.sort.SortOrder;
077import org.elasticsearch.xcontent.XContentType;
078import org.hl7.fhir.instance.model.api.IBaseResource;
079import org.slf4j.Logger;
080import org.slf4j.LoggerFactory;
081import org.springframework.beans.factory.annotation.Autowired;
082
083import javax.annotation.Nullable;
084import java.io.BufferedReader;
085import java.io.IOException;
086import java.io.InputStreamReader;
087import java.util.ArrayList;
088import java.util.Arrays;
089import java.util.Collection;
090import java.util.List;
091import java.util.function.Function;
092import java.util.stream.Collectors;
093
094import static org.apache.commons.lang3.StringUtils.isBlank;
095
096public class ElasticsearchSvcImpl implements IElasticsearchSvc {
097
098        private static final Logger ourLog = LoggerFactory.getLogger(ElasticsearchSvcImpl.class);
099
100        // Index Constants
101        public static final String OBSERVATION_INDEX = "observation_index";
102        public static final String OBSERVATION_CODE_INDEX = "code_index";
103        public static final String OBSERVATION_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedSearchParamLastNEntity";
104        public static final String CODE_DOCUMENT_TYPE = "ca.uhn.fhir.jpa.model.entity.ObservationIndexedCodeCodeableConceptEntity";
105        public static final String OBSERVATION_INDEX_SCHEMA_FILE = "ObservationIndexSchema.json";
106        public static final String OBSERVATION_CODE_INDEX_SCHEMA_FILE = "ObservationCodeIndexSchema.json";
107
108        // Aggregation Constants
109        private static final String GROUP_BY_SUBJECT = "group_by_subject";
110        private static final String GROUP_BY_SYSTEM = "group_by_system";
111        private static final String GROUP_BY_CODE = "group_by_code";
112        private static final String MOST_RECENT_EFFECTIVE = "most_recent_effective";
113
114        // Observation index document element names
115        private static final String OBSERVATION_IDENTIFIER_FIELD_NAME = "identifier";
116        private static final String OBSERVATION_SUBJECT_FIELD_NAME = "subject";
117        private static final String OBSERVATION_CODEVALUE_FIELD_NAME = "codeconceptcodingcode";
118        private static final String OBSERVATION_CODESYSTEM_FIELD_NAME = "codeconceptcodingsystem";
119        private static final String OBSERVATION_CODEHASH_FIELD_NAME = "codeconceptcodingcode_system_hash";
120        private static final String OBSERVATION_CODEDISPLAY_FIELD_NAME = "codeconceptcodingdisplay";
121        private static final String OBSERVATION_CODE_TEXT_FIELD_NAME = "codeconcepttext";
122        private static final String OBSERVATION_EFFECTIVEDTM_FIELD_NAME = "effectivedtm";
123        private static final String OBSERVATION_CATEGORYHASH_FIELD_NAME = "categoryconceptcodingcode_system_hash";
124        private static final String OBSERVATION_CATEGORYVALUE_FIELD_NAME = "categoryconceptcodingcode";
125        private static final String OBSERVATION_CATEGORYSYSTEM_FIELD_NAME = "categoryconceptcodingsystem";
126        private static final String OBSERVATION_CATEGORYDISPLAY_FIELD_NAME = "categoryconceptcodingdisplay";
127        private static final String OBSERVATION_CATEGORYTEXT_FIELD_NAME = "categoryconcepttext";
128
129        // Code index document element names
130        private static final String CODE_HASH = "codingcode_system_hash";
131        private static final String CODE_TEXT = "text";
132
133        private static final String OBSERVATION_RESOURCE_NAME = "Observation";
134
135        private final RestHighLevelClient myRestHighLevelClient;
136
137        private final ObjectMapper objectMapper = new ObjectMapper();
138
139        @Autowired
140        private PartitionSettings myPartitionSettings;
141
142        @Autowired
143        private FhirContext myContext;
144
145        //This constructor used to inject a dummy partitionsettings in test.
146        public ElasticsearchSvcImpl(PartitionSettings thePartitionSetings, String theProtocol, String theHostname, @Nullable String theUsername, @Nullable String thePassword) {
147                this(theProtocol, theHostname, theUsername, thePassword);
148                this.myPartitionSettings = thePartitionSetings;
149        }
150
151        public ElasticsearchSvcImpl(String theProtocol, String theHostname, @Nullable String theUsername, @Nullable String thePassword) {
152                myRestHighLevelClient = ElasticsearchRestClientFactory.createElasticsearchHighLevelRestClient(theProtocol, theHostname, theUsername, thePassword);
153
154                try {
155                        createObservationIndexIfMissing();
156                        createObservationCodeIndexIfMissing();
157                } catch (IOException theE) {
158                        throw new RuntimeException(Msg.code(1175) + "Failed to create document index", theE);
159                }
160        }
161
162        private String getIndexSchema(String theSchemaFileName) throws IOException {
163                InputStreamReader input = new InputStreamReader(ElasticsearchSvcImpl.class.getResourceAsStream(theSchemaFileName));
164                BufferedReader reader = new BufferedReader(input);
165                StringBuilder sb = new StringBuilder();
166                String str;
167                while ((str = reader.readLine()) != null) {
168                        sb.append(str);
169                }
170
171                return sb.toString();
172        }
173
174        private void createObservationIndexIfMissing() throws IOException {
175                if (indexExists(OBSERVATION_INDEX)) {
176                        return;
177                }
178                String observationMapping = getIndexSchema(OBSERVATION_INDEX_SCHEMA_FILE);
179                if (!createIndex(OBSERVATION_INDEX, observationMapping)) {
180                        throw new RuntimeException(Msg.code(1176) + "Failed to create observation index");
181                }
182        }
183
184        private void createObservationCodeIndexIfMissing() throws IOException {
185                if (indexExists(OBSERVATION_CODE_INDEX)) {
186                        return;
187                }
188                String observationCodeMapping = getIndexSchema(OBSERVATION_CODE_INDEX_SCHEMA_FILE);
189                if (!createIndex(OBSERVATION_CODE_INDEX, observationCodeMapping)) {
190                        throw new RuntimeException(Msg.code(1177) + "Failed to create observation code index");
191                }
192
193        }
194
195        private boolean createIndex(String theIndexName, String theMapping) throws IOException {
196                CreateIndexRequest request = new CreateIndexRequest(theIndexName);
197                request.source(theMapping, XContentType.JSON);
198                CreateIndexResponse createIndexResponse = myRestHighLevelClient.indices().create(request, RequestOptions.DEFAULT);
199                return createIndexResponse.isAcknowledged();
200
201        }
202
203        private boolean indexExists(String theIndexName) throws IOException {
204                GetIndexRequest request = new GetIndexRequest(theIndexName);
205                return myRestHighLevelClient.indices().exists(request, RequestOptions.DEFAULT);
206        }
207
208        @Override
209        public List<String> executeLastN(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, Integer theMaxResultsToFetch) {
210                Validate.isTrue(!myPartitionSettings.isPartitioningEnabled(), "$lastn is not currently supported on partitioned servers");
211
212                String[] topHitsInclude = {OBSERVATION_IDENTIFIER_FIELD_NAME};
213                return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, topHitsInclude,
214                        ObservationJson::getIdentifier, theMaxResultsToFetch);
215        }
216
217        private <T> List<T> buildAndExecuteSearch(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext,
218                                                                                                                        String[] topHitsInclude, Function<ObservationJson, T> setValue, Integer theMaxResultsToFetch) {
219                String patientParamName = LastNParameterHelper.getPatientParamName(theFhirContext);
220                String subjectParamName = LastNParameterHelper.getSubjectParamName(theFhirContext);
221                List<T> searchResults = new ArrayList<>();
222                if (theSearchParameterMap.containsKey(patientParamName)
223                        || theSearchParameterMap.containsKey(subjectParamName)) {
224                        for (String subject : getSubjectReferenceCriteria(patientParamName, subjectParamName, theSearchParameterMap)) {
225                                if (theMaxResultsToFetch != null && searchResults.size() >= theMaxResultsToFetch) {
226                                        break;
227                                }
228                                SearchRequest myLastNRequest = buildObservationsSearchRequest(subject, theSearchParameterMap, theFhirContext,
229                                        createObservationSubjectAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude));
230                                ourLog.debug("ElasticSearch query: {}", myLastNRequest.source().toString());
231                                try {
232                                        SearchResponse lastnResponse = executeSearchRequest(myLastNRequest);
233                                        searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext,
234                                                theMaxResultsToFetch));
235                                } catch (IOException theE) {
236                                        throw new InvalidRequestException(Msg.code(1178) + "Unable to execute LastN request", theE);
237                                }
238                        }
239                } else {
240                        SearchRequest myLastNRequest = buildObservationsSearchRequest(theSearchParameterMap, theFhirContext,
241                                createObservationCodeAggregationBuilder(getMaxParameter(theSearchParameterMap), topHitsInclude));
242                        ourLog.debug("ElasticSearch query: {}", myLastNRequest.source().toString());
243                        try {
244                                SearchResponse lastnResponse = executeSearchRequest(myLastNRequest);
245                                searchResults.addAll(buildObservationList(lastnResponse, setValue, theSearchParameterMap, theFhirContext,
246                                        theMaxResultsToFetch));
247                        } catch (IOException theE) {
248                                throw new InvalidRequestException(Msg.code(1179) + "Unable to execute LastN request", theE);
249                        }
250                }
251                return searchResults;
252        }
253
254        private int getMaxParameter(SearchParameterMap theSearchParameterMap) {
255                if (theSearchParameterMap.getLastNMax() == null) {
256                        return 1;
257                } else {
258                        return theSearchParameterMap.getLastNMax();
259                }
260        }
261
262        private List<String> getSubjectReferenceCriteria(String thePatientParamName, String theSubjectParamName, SearchParameterMap theSearchParameterMap) {
263                List<String> subjectReferenceCriteria = new ArrayList<>();
264
265                List<List<IQueryParameterType>> patientParams = new ArrayList<>();
266                if (theSearchParameterMap.get(thePatientParamName) != null) {
267                        patientParams.addAll(theSearchParameterMap.get(thePatientParamName));
268                }
269                if (theSearchParameterMap.get(theSubjectParamName) != null) {
270                        patientParams.addAll(theSearchParameterMap.get(theSubjectParamName));
271                }
272                for (List<? extends IQueryParameterType> nextSubjectList : patientParams) {
273                        subjectReferenceCriteria.addAll(getReferenceValues(nextSubjectList));
274                }
275                return subjectReferenceCriteria;
276        }
277
278        private List<String> getReferenceValues(List<? extends IQueryParameterType> referenceParams) {
279                List<String> referenceList = new ArrayList<>();
280
281                for (IQueryParameterType nextOr : referenceParams) {
282
283                        if (nextOr instanceof ReferenceParam) {
284                                ReferenceParam ref = (ReferenceParam) nextOr;
285                                if (isBlank(ref.getChain())) {
286                                        referenceList.add(ref.getValue());
287                                }
288                        } else {
289                                throw new IllegalArgumentException(Msg.code(1180) + "Invalid token type (expecting ReferenceParam): " + nextOr.getClass());
290                        }
291                }
292                return referenceList;
293        }
294
295        private CompositeAggregationBuilder createObservationSubjectAggregationBuilder(Integer theMaxNumberObservationsPerCode, String[] theTopHitsInclude) {
296                CompositeValuesSourceBuilder<?> subjectValuesBuilder = new TermsValuesSourceBuilder(OBSERVATION_SUBJECT_FIELD_NAME).field(OBSERVATION_SUBJECT_FIELD_NAME);
297                List<CompositeValuesSourceBuilder<?>> compositeAggSubjectSources = new ArrayList<>();
298                compositeAggSubjectSources.add(subjectValuesBuilder);
299                CompositeAggregationBuilder compositeAggregationSubjectBuilder = new CompositeAggregationBuilder(GROUP_BY_SUBJECT, compositeAggSubjectSources);
300                compositeAggregationSubjectBuilder.subAggregation(createObservationCodeAggregationBuilder(theMaxNumberObservationsPerCode, theTopHitsInclude));
301                compositeAggregationSubjectBuilder.size(10000);
302
303                return compositeAggregationSubjectBuilder;
304        }
305
306        private TermsAggregationBuilder createObservationCodeAggregationBuilder(int theMaxNumberObservationsPerCode, String[] theTopHitsInclude) {
307                TermsAggregationBuilder observationCodeCodeAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_CODE).field(OBSERVATION_CODEVALUE_FIELD_NAME);
308                observationCodeCodeAggregationBuilder.order(BucketOrder.key(true));
309                // Top Hits Aggregation
310                observationCodeCodeAggregationBuilder.subAggregation(AggregationBuilders.topHits(MOST_RECENT_EFFECTIVE)
311                        .sort(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, SortOrder.DESC)
312                        .fetchSource(theTopHitsInclude, null).size(theMaxNumberObservationsPerCode));
313                observationCodeCodeAggregationBuilder.size(10000);
314                TermsAggregationBuilder observationCodeSystemAggregationBuilder = new TermsAggregationBuilder(GROUP_BY_SYSTEM).field(OBSERVATION_CODESYSTEM_FIELD_NAME);
315                observationCodeSystemAggregationBuilder.order(BucketOrder.key(true));
316                observationCodeSystemAggregationBuilder.subAggregation(observationCodeCodeAggregationBuilder);
317                return observationCodeSystemAggregationBuilder;
318        }
319
320        private SearchResponse executeSearchRequest(SearchRequest searchRequest) throws IOException {
321                return myRestHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
322        }
323
324        private <T> List<T> buildObservationList(SearchResponse theSearchResponse, Function<ObservationJson, T> setValue,
325                                                                                                                  SearchParameterMap theSearchParameterMap, FhirContext theFhirContext,
326                                                                                                                  Integer theMaxResultsToFetch) throws IOException {
327                List<T> theObservationList = new ArrayList<>();
328                if (theSearchParameterMap.containsKey(LastNParameterHelper.getPatientParamName(theFhirContext))
329                        || theSearchParameterMap.containsKey(LastNParameterHelper.getSubjectParamName(theFhirContext))) {
330                        for (ParsedComposite.ParsedBucket subjectBucket : getSubjectBuckets(theSearchResponse)) {
331                                if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
332                                        break;
333                                }
334                                for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(subjectBucket)) {
335                                        if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
336                                                break;
337                                        }
338                                        for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) {
339                                                if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
340                                                        break;
341                                                }
342                                                String indexedObservation = lastNMatch.getSourceAsString();
343                                                ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class);
344                                                theObservationList.add(setValue.apply(observationJson));
345                                        }
346                                }
347                        }
348                } else {
349                        for (Terms.Bucket observationCodeBucket : getObservationCodeBuckets(theSearchResponse)) {
350                                if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
351                                        break;
352                                }
353                                for (SearchHit lastNMatch : getLastNMatches(observationCodeBucket)) {
354                                        if (theMaxResultsToFetch != null && theObservationList.size() >= theMaxResultsToFetch) {
355                                                break;
356                                        }
357                                        String indexedObservation = lastNMatch.getSourceAsString();
358                                        ObservationJson observationJson = objectMapper.readValue(indexedObservation, ObservationJson.class);
359                                        theObservationList.add(setValue.apply(observationJson));
360                                }
361                        }
362                }
363
364                return theObservationList;
365        }
366
367        private List<ParsedComposite.ParsedBucket> getSubjectBuckets(SearchResponse theSearchResponse) {
368                Aggregations responseAggregations = theSearchResponse.getAggregations();
369                ParsedComposite aggregatedSubjects = responseAggregations.get(GROUP_BY_SUBJECT);
370                return aggregatedSubjects.getBuckets();
371        }
372
373        private List<? extends Terms.Bucket> getObservationCodeBuckets(SearchResponse theSearchResponse) {
374                Aggregations responseAggregations = theSearchResponse.getAggregations();
375                return getObservationCodeBuckets(responseAggregations);
376        }
377
378        private List<? extends Terms.Bucket> getObservationCodeBuckets(ParsedComposite.ParsedBucket theSubjectBucket) {
379                Aggregations observationCodeSystemAggregations = theSubjectBucket.getAggregations();
380                return getObservationCodeBuckets(observationCodeSystemAggregations);
381        }
382
383        private List<? extends Terms.Bucket> getObservationCodeBuckets(Aggregations theObservationCodeSystemAggregations) {
384                List<Terms.Bucket> retVal = new ArrayList<>();
385                ParsedTerms aggregatedObservationCodeSystems = theObservationCodeSystemAggregations.get(GROUP_BY_SYSTEM);
386                for (Terms.Bucket observationCodeSystem : aggregatedObservationCodeSystems.getBuckets()) {
387                        Aggregations observationCodeCodeAggregations = observationCodeSystem.getAggregations();
388                        ParsedTerms aggregatedObservationCodeCodes = observationCodeCodeAggregations.get(GROUP_BY_CODE);
389                        retVal.addAll(aggregatedObservationCodeCodes.getBuckets());
390                }
391                return retVal;
392        }
393
394        private SearchHit[] getLastNMatches(Terms.Bucket theObservationCodeBucket) {
395                Aggregations topHitObservationCodes = theObservationCodeBucket.getAggregations();
396                ParsedTopHits parsedTopHits = topHitObservationCodes.get(MOST_RECENT_EFFECTIVE);
397                return parsedTopHits.getHits().getHits();
398        }
399
400        private SearchRequest buildObservationsSearchRequest(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext, AggregationBuilder theAggregationBuilder) {
401                SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX);
402                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
403                // Query
404                if (!searchParamsHaveLastNCriteria(theSearchParameterMap, theFhirContext)) {
405                        searchSourceBuilder.query(QueryBuilders.matchAllQuery());
406                } else {
407                        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
408                        addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext);
409                        addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext);
410                        addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext);
411                        searchSourceBuilder.query(boolQueryBuilder);
412                }
413                searchSourceBuilder.size(0);
414
415                // Aggregation by order codes
416                searchSourceBuilder.aggregation(theAggregationBuilder);
417                searchRequest.source(searchSourceBuilder);
418
419                return searchRequest;
420        }
421
422        private SearchRequest buildObservationsSearchRequest(String theSubjectParam, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext,
423                                                                                                                                                  AggregationBuilder theAggregationBuilder) {
424                SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX);
425                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
426                // Query
427                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
428                boolQueryBuilder.must(QueryBuilders.termQuery(OBSERVATION_SUBJECT_FIELD_NAME, theSubjectParam));
429                addCategoriesCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext);
430                addObservationCodeCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext);
431                addDateCriteria(boolQueryBuilder, theSearchParameterMap, theFhirContext);
432                searchSourceBuilder.query(boolQueryBuilder);
433                searchSourceBuilder.size(0);
434
435                // Aggregation by order codes
436                searchSourceBuilder.aggregation(theAggregationBuilder);
437                searchRequest.source(searchSourceBuilder);
438
439                return searchRequest;
440        }
441
442        private Boolean searchParamsHaveLastNCriteria(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
443                return theSearchParameterMap != null &&
444                        (theSearchParameterMap.containsKey(LastNParameterHelper.getPatientParamName(theFhirContext))
445                                || theSearchParameterMap.containsKey(LastNParameterHelper.getSubjectParamName(theFhirContext))
446                                || theSearchParameterMap.containsKey(LastNParameterHelper.getCategoryParamName(theFhirContext))
447                                || theSearchParameterMap.containsKey(LastNParameterHelper.getCodeParamName(theFhirContext)));
448        }
449
450        private void addCategoriesCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
451                String categoryParamName = LastNParameterHelper.getCategoryParamName(theFhirContext);
452                if (theSearchParameterMap.containsKey(categoryParamName)) {
453                        ArrayList<String> codeSystemHashList = new ArrayList<>();
454                        ArrayList<String> codeOnlyList = new ArrayList<>();
455                        ArrayList<String> systemOnlyList = new ArrayList<>();
456                        ArrayList<String> textOnlyList = new ArrayList<>();
457                        List<List<IQueryParameterType>> andOrParams = theSearchParameterMap.get(categoryParamName);
458                        for (List<? extends IQueryParameterType> nextAnd : andOrParams) {
459                                codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd));
460                                codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd));
461                                systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd));
462                                textOnlyList.addAll(getCodingTextOnlyValues(nextAnd));
463                        }
464                        if (codeSystemHashList.size() > 0) {
465                                theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYHASH_FIELD_NAME, codeSystemHashList));
466                        }
467                        if (codeOnlyList.size() > 0) {
468                                theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYVALUE_FIELD_NAME, codeOnlyList));
469                        }
470                        if (systemOnlyList.size() > 0) {
471                                theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CATEGORYSYSTEM_FIELD_NAME, systemOnlyList));
472                        }
473                        if (textOnlyList.size() > 0) {
474                                BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery();
475                                for (String textOnlyParam : textOnlyList) {
476                                        myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYDISPLAY_FIELD_NAME, textOnlyParam));
477                                        myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CATEGORYTEXT_FIELD_NAME, textOnlyParam));
478                                }
479                                theBoolQueryBuilder.must(myTextBoolQueryBuilder);
480                        }
481                }
482
483        }
484
485        private List<String> getCodingCodeSystemValues(List<? extends IQueryParameterType> codeParams) {
486                ArrayList<String> codeSystemHashList = new ArrayList<>();
487                for (IQueryParameterType nextOr : codeParams) {
488                        if (nextOr instanceof TokenParam) {
489                                TokenParam ref = (TokenParam) nextOr;
490                                if (ref.getSystem() != null && ref.getValue() != null) {
491                                        codeSystemHashList.add(String.valueOf(CodeSystemHash.hashCodeSystem(ref.getSystem(), ref.getValue())));
492                                }
493                        } else {
494                                throw new IllegalArgumentException(Msg.code(1181) + "Invalid token type (expecting TokenParam): " + nextOr.getClass());
495                        }
496                }
497                return codeSystemHashList;
498        }
499
500        private List<String> getCodingCodeOnlyValues(List<? extends IQueryParameterType> codeParams) {
501                ArrayList<String> codeOnlyList = new ArrayList<>();
502                for (IQueryParameterType nextOr : codeParams) {
503
504                        if (nextOr instanceof TokenParam) {
505                                TokenParam ref = (TokenParam) nextOr;
506                                if (ref.getValue() != null && ref.getSystem() == null && !ref.isText()) {
507                                        codeOnlyList.add(ref.getValue());
508                                }
509                        } else {
510                                throw new IllegalArgumentException(Msg.code(1182) + "Invalid token type (expecting TokenParam): " + nextOr.getClass());
511                        }
512                }
513                return codeOnlyList;
514        }
515
516        private List<String> getCodingSystemOnlyValues(List<? extends IQueryParameterType> codeParams) {
517                ArrayList<String> systemOnlyList = new ArrayList<>();
518                for (IQueryParameterType nextOr : codeParams) {
519
520                        if (nextOr instanceof TokenParam) {
521                                TokenParam ref = (TokenParam) nextOr;
522                                if (ref.getValue() == null && ref.getSystem() != null) {
523                                        systemOnlyList.add(ref.getSystem());
524                                }
525                        } else {
526                                throw new IllegalArgumentException(Msg.code(1183) + "Invalid token type (expecting TokenParam): " + nextOr.getClass());
527                        }
528                }
529                return systemOnlyList;
530        }
531
532        private List<String> getCodingTextOnlyValues(List<? extends IQueryParameterType> codeParams) {
533                ArrayList<String> textOnlyList = new ArrayList<>();
534                for (IQueryParameterType nextOr : codeParams) {
535
536                        if (nextOr instanceof TokenParam) {
537                                TokenParam ref = (TokenParam) nextOr;
538                                if (ref.isText() && ref.getValue() != null) {
539                                        textOnlyList.add(ref.getValue());
540                                }
541                        } else {
542                                throw new IllegalArgumentException(Msg.code(1184) + "Invalid token type (expecting TokenParam): " + nextOr.getClass());
543                        }
544                }
545                return textOnlyList;
546        }
547
548        private void addObservationCodeCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
549                String codeParamName = LastNParameterHelper.getCodeParamName(theFhirContext);
550                if (theSearchParameterMap.containsKey(codeParamName)) {
551                        ArrayList<String> codeSystemHashList = new ArrayList<>();
552                        ArrayList<String> codeOnlyList = new ArrayList<>();
553                        ArrayList<String> systemOnlyList = new ArrayList<>();
554                        ArrayList<String> textOnlyList = new ArrayList<>();
555                        List<List<IQueryParameterType>> andOrParams = theSearchParameterMap.get(codeParamName);
556                        for (List<? extends IQueryParameterType> nextAnd : andOrParams) {
557                                codeSystemHashList.addAll(getCodingCodeSystemValues(nextAnd));
558                                codeOnlyList.addAll(getCodingCodeOnlyValues(nextAnd));
559                                systemOnlyList.addAll(getCodingSystemOnlyValues(nextAnd));
560                                textOnlyList.addAll(getCodingTextOnlyValues(nextAnd));
561                        }
562                        if (codeSystemHashList.size() > 0) {
563                                theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEHASH_FIELD_NAME, codeSystemHashList));
564                        }
565                        if (codeOnlyList.size() > 0) {
566                                theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODEVALUE_FIELD_NAME, codeOnlyList));
567                        }
568                        if (systemOnlyList.size() > 0) {
569                                theBoolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_CODESYSTEM_FIELD_NAME, systemOnlyList));
570                        }
571                        if (textOnlyList.size() > 0) {
572                                BoolQueryBuilder myTextBoolQueryBuilder = QueryBuilders.boolQuery();
573                                for (String textOnlyParam : textOnlyList) {
574                                        myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODEDISPLAY_FIELD_NAME, textOnlyParam));
575                                        myTextBoolQueryBuilder.should(QueryBuilders.matchPhrasePrefixQuery(OBSERVATION_CODE_TEXT_FIELD_NAME, textOnlyParam));
576                                }
577                                theBoolQueryBuilder.must(myTextBoolQueryBuilder);
578                        }
579                }
580
581        }
582
583        private void addDateCriteria(BoolQueryBuilder theBoolQueryBuilder, SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
584                String dateParamName = LastNParameterHelper.getEffectiveParamName(theFhirContext);
585                if (theSearchParameterMap.containsKey(dateParamName)) {
586                        List<List<IQueryParameterType>> andOrParams = theSearchParameterMap.get(dateParamName);
587                        for (List<? extends IQueryParameterType> nextAnd : andOrParams) {
588                                BoolQueryBuilder myDateBoolQueryBuilder = new BoolQueryBuilder();
589                                for (IQueryParameterType nextOr : nextAnd) {
590                                        if (nextOr instanceof DateParam) {
591                                                DateParam myDate = (DateParam) nextOr;
592                                                createDateCriteria(myDate, myDateBoolQueryBuilder);
593                                        }
594                                }
595                                theBoolQueryBuilder.must(myDateBoolQueryBuilder);
596                        }
597                }
598        }
599
600        private void createDateCriteria(DateParam theDate, BoolQueryBuilder theBoolQueryBuilder) {
601                Long dateInstant = theDate.getValue().getTime();
602                RangeQueryBuilder myRangeQueryBuilder = new RangeQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME);
603
604                ParamPrefixEnum prefix = theDate.getPrefix();
605                if (prefix == ParamPrefixEnum.GREATERTHAN || prefix == ParamPrefixEnum.STARTS_AFTER) {
606                        theBoolQueryBuilder.should(myRangeQueryBuilder.gt(dateInstant));
607                } else if (prefix == ParamPrefixEnum.LESSTHAN || prefix == ParamPrefixEnum.ENDS_BEFORE) {
608                        theBoolQueryBuilder.should(myRangeQueryBuilder.lt(dateInstant));
609                } else if (prefix == ParamPrefixEnum.LESSTHAN_OR_EQUALS) {
610                        theBoolQueryBuilder.should(myRangeQueryBuilder.lte(dateInstant));
611                } else if (prefix == ParamPrefixEnum.GREATERTHAN_OR_EQUALS) {
612                        theBoolQueryBuilder.should(myRangeQueryBuilder.gte(dateInstant));
613                } else {
614                        theBoolQueryBuilder.should(new MatchQueryBuilder(OBSERVATION_EFFECTIVEDTM_FIELD_NAME, dateInstant));
615                }
616        }
617
618        @VisibleForTesting
619        public List<ObservationJson> executeLastNWithAllFieldsForTest(SearchParameterMap theSearchParameterMap, FhirContext theFhirContext) {
620                return buildAndExecuteSearch(theSearchParameterMap, theFhirContext, null, t -> t, 100);
621        }
622
623        @VisibleForTesting
624        List<CodeJson> queryAllIndexedObservationCodesForTest() throws IOException {
625                SearchRequest codeSearchRequest = new SearchRequest(OBSERVATION_CODE_INDEX);
626                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
627                // Query
628                searchSourceBuilder.query(QueryBuilders.matchAllQuery());
629                searchSourceBuilder.size(1000);
630                codeSearchRequest.source(searchSourceBuilder);
631                SearchResponse codeSearchResponse = executeSearchRequest(codeSearchRequest);
632                return buildCodeResult(codeSearchResponse);
633        }
634
635        private List<CodeJson> buildCodeResult(SearchResponse theSearchResponse) throws JsonProcessingException {
636                SearchHits codeHits = theSearchResponse.getHits();
637                List<CodeJson> codes = new ArrayList<>();
638                for (SearchHit codeHit : codeHits) {
639                        CodeJson code = objectMapper.readValue(codeHit.getSourceAsString(), CodeJson.class);
640                        codes.add(code);
641                }
642                return codes;
643        }
644
645        @Override
646        public ObservationJson getObservationDocument(String theDocumentID) {
647                if (theDocumentID == null) {
648                        throw new InvalidRequestException(Msg.code(1185) + "Require non-null document ID for observation document query");
649                }
650                SearchRequest theSearchRequest = buildSingleObservationSearchRequest(theDocumentID);
651                ObservationJson observationDocumentJson = null;
652                try {
653                        SearchResponse observationDocumentResponse = executeSearchRequest(theSearchRequest);
654                        SearchHit[] observationDocumentHits = observationDocumentResponse.getHits().getHits();
655                        if (observationDocumentHits.length > 0) {
656                                // There should be no more than one hit for the identifier
657                                String observationDocument = observationDocumentHits[0].getSourceAsString();
658                                observationDocumentJson = objectMapper.readValue(observationDocument, ObservationJson.class);
659                        }
660
661                } catch (IOException theE) {
662                        throw new InvalidRequestException(Msg.code(1186) + "Unable to execute observation document query for ID " + theDocumentID, theE);
663                }
664
665                return observationDocumentJson;
666        }
667
668        private SearchRequest buildSingleObservationSearchRequest(String theObservationIdentifier) {
669                SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX);
670                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
671                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
672                boolQueryBuilder.must(QueryBuilders.termQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theObservationIdentifier));
673                searchSourceBuilder.query(boolQueryBuilder);
674                searchSourceBuilder.size(1);
675
676                searchRequest.source(searchSourceBuilder);
677
678                return searchRequest;
679        }
680
681        @Override
682        public CodeJson getObservationCodeDocument(String theCodeSystemHash, String theText) {
683                if (theCodeSystemHash == null && theText == null) {
684                        throw new InvalidRequestException(Msg.code(1187) + "Require a non-null code system hash value or display value for observation code document query");
685                }
686                SearchRequest theSearchRequest = buildSingleObservationCodeSearchRequest(theCodeSystemHash, theText);
687                CodeJson observationCodeDocumentJson = null;
688                try {
689                        SearchResponse observationCodeDocumentResponse = executeSearchRequest(theSearchRequest);
690                        SearchHit[] observationCodeDocumentHits = observationCodeDocumentResponse.getHits().getHits();
691                        if (observationCodeDocumentHits.length > 0) {
692                                // There should be no more than one hit for the code lookup.
693                                String observationCodeDocument = observationCodeDocumentHits[0].getSourceAsString();
694                                observationCodeDocumentJson = objectMapper.readValue(observationCodeDocument, CodeJson.class);
695                        }
696
697                } catch (IOException theE) {
698                        throw new InvalidRequestException(Msg.code(1188) + "Unable to execute observation code document query hash code or display", theE);
699                }
700
701                return observationCodeDocumentJson;
702        }
703
704        private SearchRequest buildSingleObservationCodeSearchRequest(String theCodeSystemHash, String theText) {
705                SearchRequest searchRequest = new SearchRequest(OBSERVATION_CODE_INDEX);
706                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
707                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
708                if (theCodeSystemHash != null) {
709                        boolQueryBuilder.must(QueryBuilders.termQuery(CODE_HASH, theCodeSystemHash));
710                } else {
711                        boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(CODE_TEXT, theText));
712                }
713
714                searchSourceBuilder.query(boolQueryBuilder);
715                searchSourceBuilder.size(1);
716
717                searchRequest.source(searchSourceBuilder);
718
719                return searchRequest;
720        }
721
722        @Override
723        public Boolean createOrUpdateObservationIndex(String theDocumentId, ObservationJson theObservationDocument) {
724                try {
725                        String documentToIndex = objectMapper.writeValueAsString(theObservationDocument);
726                        return performIndex(OBSERVATION_INDEX, theDocumentId, documentToIndex, ElasticsearchSvcImpl.OBSERVATION_DOCUMENT_TYPE);
727                } catch (IOException theE) {
728                        throw new InvalidRequestException(Msg.code(1189) + "Unable to persist Observation document " + theDocumentId);
729                }
730        }
731
732        @Override
733        public Boolean createOrUpdateObservationCodeIndex(String theCodeableConceptID, CodeJson theObservationCodeDocument) {
734                try {
735                        String documentToIndex = objectMapper.writeValueAsString(theObservationCodeDocument);
736                        return performIndex(OBSERVATION_CODE_INDEX, theCodeableConceptID, documentToIndex, ElasticsearchSvcImpl.CODE_DOCUMENT_TYPE);
737                } catch (IOException theE) {
738                        throw new InvalidRequestException(Msg.code(1190) + "Unable to persist Observation Code document " + theCodeableConceptID);
739                }
740        }
741
742        private boolean performIndex(String theIndexName, String theDocumentId, String theIndexDocument, String theDocumentType) throws IOException {
743                IndexResponse indexResponse = myRestHighLevelClient.index(createIndexRequest(theIndexName, theDocumentId, theIndexDocument, theDocumentType),
744                        RequestOptions.DEFAULT);
745
746                return (indexResponse.getResult() == DocWriteResponse.Result.CREATED) || (indexResponse.getResult() == DocWriteResponse.Result.UPDATED);
747        }
748
749        @Override
750        public void close() throws IOException {
751                myRestHighLevelClient.close();
752        }
753
754        @Override
755        public List<IBaseResource> getObservationResources(Collection<ResourcePersistentId> thePids) {
756                SearchRequest searchRequest = buildObservationResourceSearchRequest(thePids);
757                try {
758                        SearchResponse observationDocumentResponse = executeSearchRequest(searchRequest);
759                        SearchHit[] observationDocumentHits = observationDocumentResponse.getHits().getHits();
760                        IParser parser = TolerantJsonParser.createWithLenientErrorHandling(myContext, null);
761                        Class<? extends IBaseResource> resourceType = myContext.getResourceDefinition(OBSERVATION_RESOURCE_NAME).getImplementingClass();
762                        /**
763                         * @see ca.uhn.fhir.jpa.dao.BaseHapiFhirDao#toResource(Class, IBaseResourceEntity, Collection, boolean) for
764                         * details about parsing raw json to BaseResource
765                         */
766                        return Arrays.stream(observationDocumentHits)
767                                .map(this::parseObservationJson)
768                                .map(observationJson -> parser.parseResource(resourceType, observationJson.getResource()))
769                                .collect(Collectors.toList());
770                } catch (IOException theE) {
771                        throw new InvalidRequestException(Msg.code(2003) + "Unable to execute observation document query for provided IDs " + thePids, theE);
772                }
773        }
774
775        private ObservationJson parseObservationJson(SearchHit theSearchHit) {
776                try {
777                        return objectMapper.readValue(theSearchHit.getSourceAsString(), ObservationJson.class);
778                } catch (JsonProcessingException exp) {
779                        throw new InvalidRequestException(Msg.code(2004) + "Unable to parse the observation resource json", exp);
780                }
781        }
782
783        private SearchRequest buildObservationResourceSearchRequest(Collection<ResourcePersistentId> thePids) {
784                SearchRequest searchRequest = new SearchRequest(OBSERVATION_INDEX);
785                SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
786                // Query
787                BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
788                List<String> pidParams = thePids.stream().map(Object::toString).collect(Collectors.toList());
789                boolQueryBuilder.must(QueryBuilders.termsQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, pidParams));
790                searchSourceBuilder.query(boolQueryBuilder);
791                searchSourceBuilder.size(thePids.size());
792                searchRequest.source(searchSourceBuilder);
793                return searchRequest;
794        }
795
796
797        private IndexRequest createIndexRequest(String theIndexName, String theDocumentId, String theObservationDocument, String theDocumentType) {
798                IndexRequest request = new IndexRequest(theIndexName);
799                request.id(theDocumentId);
800                request.source(theObservationDocument, XContentType.JSON);
801                return request;
802        }
803
804        @Override
805        public void deleteObservationDocument(String theDocumentId) {
806                DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(OBSERVATION_INDEX);
807                deleteByQueryRequest.setQuery(QueryBuilders.termQuery(OBSERVATION_IDENTIFIER_FIELD_NAME, theDocumentId));
808                try {
809                        myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
810                } catch (IOException theE) {
811                        throw new InvalidRequestException(Msg.code(1191) + "Unable to delete Observation " + theDocumentId);
812                }
813        }
814
815        @VisibleForTesting
816        public void deleteAllDocumentsForTest(String theIndexName) throws IOException {
817                DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(theIndexName);
818                deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery());
819                myRestHighLevelClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
820        }
821
822        @VisibleForTesting
823        public void refreshIndex(String theIndexName) throws IOException {
824                myRestHighLevelClient.indices().refresh(new RefreshRequest(theIndexName), RequestOptions.DEFAULT);
825        }
826
827}