Source code

001package ca.uhn.fhir.jpa.provider;
002
003/*
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.i18n.Msg;
024import ca.uhn.fhir.context.FhirContext;
025import ca.uhn.fhir.context.RuntimeResourceDefinition;
026import ca.uhn.fhir.jpa.model.util.JpaConstants;
027import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
028import ca.uhn.fhir.jpa.term.UploadStatistics;
029import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
030import ca.uhn.fhir.jpa.term.custom.ConceptHandler;
031import ca.uhn.fhir.jpa.term.custom.HierarchyHandler;
032import ca.uhn.fhir.jpa.term.custom.PropertyHandler;
033import ca.uhn.fhir.rest.annotation.Operation;
034import ca.uhn.fhir.rest.annotation.OperationParam;
035import ca.uhn.fhir.rest.api.server.RequestDetails;
036import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
037import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
038import ca.uhn.fhir.util.AttachmentUtil;
039import ca.uhn.fhir.util.ParametersUtil;
040import ca.uhn.fhir.util.ValidateUtil;
041import com.google.common.base.Charsets;
042import com.google.common.collect.ArrayListMultimap;
043import com.google.common.collect.Multimap;
044import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_30_40;
045import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50;
046import org.hl7.fhir.convertors.factory.VersionConvertorFactory_30_40;
047import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
048import org.hl7.fhir.instance.model.api.IBaseParameters;
049import org.hl7.fhir.instance.model.api.IBaseResource;
050import org.hl7.fhir.instance.model.api.ICompositeType;
051import org.hl7.fhir.instance.model.api.IPrimitiveType;
052import org.hl7.fhir.r4.model.CodeSystem;
053import org.springframework.beans.factory.annotation.Autowired;
054
055import javax.annotation.Nonnull;
056import javax.servlet.http.HttpServletRequest;
057import java.io.File;
058import java.io.FileInputStream;
059import java.io.FileNotFoundException;
060import java.io.InputStream;
061import java.util.ArrayList;
062import java.util.LinkedHashMap;
063import java.util.List;
064import java.util.Map;
065
066import static org.apache.commons.lang3.StringUtils.*;
067
068public class TerminologyUploaderProvider extends BaseJpaProvider {
069
070        public static final String PARAM_FILE = "file";
071        public static final String PARAM_CODESYSTEM = "codeSystem";
072        public static final String PARAM_SYSTEM = "system";
073        private static final String RESP_PARAM_CONCEPT_COUNT = "conceptCount";
074        private static final String RESP_PARAM_TARGET = "target";
075        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProvider.class);
076        private static final String RESP_PARAM_SUCCESS = "success";
077
078        @Autowired
079        private ITermLoaderSvc myTerminologyLoaderSvc;
080
081        /**
082         * Constructor
083         */
084        public TerminologyUploaderProvider() {
085                this(null, null);
086        }
087
088        /**
089         * Constructor
090         */
091        public TerminologyUploaderProvider(FhirContext theContext, ITermLoaderSvc theTerminologyLoaderSvc) {
092                setContext(theContext);
093                myTerminologyLoaderSvc = theTerminologyLoaderSvc;
094        }
095
096        /**
097         * <code>
098         * $upload-external-codesystem
099         * </code>
100         */
101        @Operation(typeName = "CodeSystem", name = JpaConstants.OPERATION_UPLOAD_EXTERNAL_CODE_SYSTEM, idempotent = false, returnParameters = {
102//              @OperationParam(name = "conceptCount", type = IntegerType.class, min = 1)
103        })
104        public IBaseParameters uploadSnapshot(
105                HttpServletRequest theServletRequest,
106                @OperationParam(name = PARAM_SYSTEM, min = 1, typeName = "uri") IPrimitiveType<String> theCodeSystemUrl,
107                @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles,
108                RequestDetails theRequestDetails
109        ) {
110
111                startRequest(theServletRequest);
112
113                if (theCodeSystemUrl == null || isBlank(theCodeSystemUrl.getValueAsString())) {
114                        throw new InvalidRequestException(Msg.code(1137) + "Missing mandatory parameter: " + PARAM_SYSTEM);
115                }
116
117                if (theFiles == null || theFiles.size() == 0) {
118                        throw new InvalidRequestException(Msg.code(1138) + "No '" + PARAM_FILE + "' parameter, or package had no data");
119                }
120                for (ICompositeType next : theFiles) {
121                        ValidateUtil.isTrueOrThrowInvalidRequest(getContext().getElementDefinition(next.getClass()).getName().equals("Attachment"), "Package must be of type Attachment");
122                }
123
124                try {
125                        List<ITermLoaderSvc.FileDescriptor> localFiles = convertAttachmentsToFileDescriptors(theFiles);
126
127                        String codeSystemUrl = theCodeSystemUrl.getValue();
128                        codeSystemUrl = trim(codeSystemUrl);
129
130                        UploadStatistics stats;
131                        switch (codeSystemUrl) {
132                                case ITermLoaderSvc.ICD10CM_URI:
133                                        stats = myTerminologyLoaderSvc.loadIcd10cm(localFiles, theRequestDetails);
134                                        break;
135                                case ITermLoaderSvc.IMGTHLA_URI:
136                                        stats = myTerminologyLoaderSvc.loadImgthla(localFiles, theRequestDetails);
137                                        break;
138                                case ITermLoaderSvc.LOINC_URI:
139                                        stats = myTerminologyLoaderSvc.loadLoinc(localFiles, theRequestDetails);
140                                        break;
141                                case ITermLoaderSvc.SCT_URI:
142                                        stats = myTerminologyLoaderSvc.loadSnomedCt(localFiles, theRequestDetails);
143                                        break;
144                                default:
145                                        stats = myTerminologyLoaderSvc.loadCustom(codeSystemUrl, localFiles, theRequestDetails);
146                                        break;
147                        }
148
149                        IBaseParameters retVal = ParametersUtil.newInstance(getContext());
150                        ParametersUtil.addParameterToParametersBoolean(getContext(), retVal, RESP_PARAM_SUCCESS, true);
151                        ParametersUtil.addParameterToParametersInteger(getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, stats.getUpdatedConceptCount());
152                        ParametersUtil.addParameterToParametersReference(getContext(), retVal, RESP_PARAM_TARGET, stats.getTarget().getValue());
153
154                        return retVal;
155                } finally {
156                        endRequest(theServletRequest);
157                }
158        }
159
160        /**
161         * <code>
162         * $apply-codesystem-delta-add
163         * </code>
164         */
165        @Operation(typeName = "CodeSystem", name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_ADD, idempotent = false, returnParameters = {
166        })
167        public IBaseParameters uploadDeltaAdd(
168                HttpServletRequest theServletRequest,
169                @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
170                @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles,
171                @OperationParam(name = PARAM_CODESYSTEM, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "CodeSystem") List<IBaseResource> theCodeSystems,
172                RequestDetails theRequestDetails
173        ) {
174
175                startRequest(theServletRequest);
176                try {
177                        validateHaveSystem(theSystem);
178                        validateHaveFiles(theFiles, theCodeSystems);
179
180                        List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
181                        convertCodeSystemsToFileDescriptors(files, theCodeSystems);
182                        UploadStatistics outcome = myTerminologyLoaderSvc.loadDeltaAdd(theSystem.getValue(), files, theRequestDetails);
183                        return toDeltaResponse(outcome);
184                } finally {
185                        endRequest(theServletRequest);
186                }
187
188        }
189
190
191        /**
192         * <code>
193         * $apply-codesystem-delta-remove
194         * </code>
195         */
196        @Operation(typeName = "CodeSystem", name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_REMOVE, idempotent = false, returnParameters = {
197        })
198        public IBaseParameters uploadDeltaRemove(
199                HttpServletRequest theServletRequest,
200                @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
201                @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles,
202                @OperationParam(name = PARAM_CODESYSTEM, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "CodeSystem") List<IBaseResource> theCodeSystems,
203                RequestDetails theRequestDetails
204        ) {
205
206                startRequest(theServletRequest);
207                try {
208                        validateHaveSystem(theSystem);
209                        validateHaveFiles(theFiles, theCodeSystems);
210
211                        List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
212                        convertCodeSystemsToFileDescriptors(files, theCodeSystems);
213                        UploadStatistics outcome = myTerminologyLoaderSvc.loadDeltaRemove(theSystem.getValue(), files, theRequestDetails);
214                        return toDeltaResponse(outcome);
215                } finally {
216                        endRequest(theServletRequest);
217                }
218
219        }
220
221        private void convertCodeSystemsToFileDescriptors(List<ITermLoaderSvc.FileDescriptor> theFiles, List<IBaseResource> theCodeSystems) {
222                Map<String, String> codes = new LinkedHashMap<>();
223                Map<String, List<CodeSystem.ConceptPropertyComponent>> codeToProperties = new LinkedHashMap<>();
224
225                Multimap<String, String> codeToParentCodes = ArrayListMultimap.create();
226
227                if (theCodeSystems != null) {
228                        for (IBaseResource nextCodeSystemUncast : theCodeSystems) {
229                                CodeSystem nextCodeSystem = canonicalizeCodeSystem(nextCodeSystemUncast);
230                                convertCodeSystemCodesToCsv(nextCodeSystem.getConcept(), codes, codeToProperties, null, codeToParentCodes);
231                        }
232                }
233
234                // Create concept file
235                if (codes.size() > 0) {
236                        StringBuilder b = new StringBuilder();
237                        b.append(ConceptHandler.CODE);
238                        b.append(",");
239                        b.append(ConceptHandler.DISPLAY);
240                        b.append("\n");
241                        for (Map.Entry<String, String> nextEntry : codes.entrySet()) {
242                                b.append(csvEscape(nextEntry.getKey()));
243                                b.append(",");
244                                b.append(csvEscape(nextEntry.getValue()));
245                                b.append("\n");
246                        }
247                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
248                        String fileName = TermLoaderSvcImpl.CUSTOM_CONCEPTS_FILE;
249                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
250                        theFiles.add(fileDescriptor);
251                }
252
253                // Create hierarchy file
254                if (codeToParentCodes.size() > 0) {
255                        StringBuilder b = new StringBuilder();
256                        b.append(HierarchyHandler.CHILD);
257                        b.append(",");
258                        b.append(HierarchyHandler.PARENT);
259                        b.append("\n");
260                        for (Map.Entry<String, String> nextEntry : codeToParentCodes.entries()) {
261                                b.append(csvEscape(nextEntry.getKey()));
262                                b.append(",");
263                                b.append(csvEscape(nextEntry.getValue()));
264                                b.append("\n");
265                        }
266                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
267                        String fileName = TermLoaderSvcImpl.CUSTOM_HIERARCHY_FILE;
268                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
269                        theFiles.add(fileDescriptor);
270                }
271                // Create codeToProperties file
272                if (codeToProperties.size() > 0) {
273                        StringBuilder b = new StringBuilder();
274                        b.append(PropertyHandler.CODE);
275                        b.append(",");
276                        b.append(PropertyHandler.KEY);
277                        b.append(",");
278                        b.append(PropertyHandler.VALUE);
279                        b.append(",");
280                        b.append(PropertyHandler.TYPE);
281                        b.append("\n");
282
283                        for (Map.Entry<String, List<CodeSystem.ConceptPropertyComponent>> nextEntry : codeToProperties.entrySet()) {
284                                for (CodeSystem.ConceptPropertyComponent propertyComponent : nextEntry.getValue()) {
285                                        b.append(csvEscape(nextEntry.getKey()));
286                                        b.append(",");
287                                        b.append(csvEscape(propertyComponent.getCode()));
288                                        b.append(",");
289                                        //TODO: check this for different types, other types should be added once TermConceptPropertyTypeEnum contain different types
290                                        b.append(csvEscape(propertyComponent.getValueStringType().getValue()));
291                                        b.append(",");
292                                        b.append(csvEscape(propertyComponent.getValue().primitiveValue()));
293                                        b.append("\n");
294                                }
295                        }
296                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
297                        String fileName = TermLoaderSvcImpl.CUSTOM_PROPERTIES_FILE;
298                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor = new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
299                        theFiles.add(fileDescriptor);
300                }
301
302        }
303
304        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
305        @Nonnull
306        CodeSystem canonicalizeCodeSystem(@Nonnull IBaseResource theCodeSystem) {
307                RuntimeResourceDefinition resourceDef = getContext().getResourceDefinition(theCodeSystem);
308                ValidateUtil.isTrueOrThrowInvalidRequest(resourceDef.getName().equals("CodeSystem"), "Resource '%s' is not a CodeSystem", resourceDef.getName());
309
310                CodeSystem nextCodeSystem;
311                switch (getContext().getVersion().getVersion()) {
312                        case DSTU3:
313                                nextCodeSystem = (CodeSystem) VersionConvertorFactory_30_40.convertResource((org.hl7.fhir.dstu3.model.CodeSystem) theCodeSystem, new BaseAdvisor_30_40(false));
314                                break;
315                        case R5:
316                                nextCodeSystem = (CodeSystem) VersionConvertorFactory_40_50.convertResource((org.hl7.fhir.r5.model.CodeSystem) theCodeSystem, new BaseAdvisor_40_50(false));
317                                break;
318                        default:
319                                nextCodeSystem = (CodeSystem) theCodeSystem;
320                }
321                return nextCodeSystem;
322        }
323
324        private void convertCodeSystemCodesToCsv(List<CodeSystem.ConceptDefinitionComponent> theConcept, Map<String, String> theCodes, Map<String, List<CodeSystem.ConceptPropertyComponent>> theProperties, String theParentCode, Multimap<String, String> theCodeToParentCodes) {
325                for (CodeSystem.ConceptDefinitionComponent nextConcept : theConcept) {
326                        if (isNotBlank(nextConcept.getCode())) {
327                                theCodes.put(nextConcept.getCode(), nextConcept.getDisplay());
328                                if (isNotBlank(theParentCode)) {
329                                        theCodeToParentCodes.put(nextConcept.getCode(), theParentCode);
330                                }
331                                if (nextConcept.getProperty() != null) {
332                                        theProperties.put(nextConcept.getCode(), nextConcept.getProperty());
333                                }
334                                convertCodeSystemCodesToCsv(nextConcept.getConcept(), theCodes, theProperties, nextConcept.getCode(), theCodeToParentCodes);
335                        }
336                }
337        }
338
339        private void validateHaveSystem(IPrimitiveType<String> theSystem) {
340                if (theSystem == null || isBlank(theSystem.getValueAsString())) {
341                        throw new InvalidRequestException(Msg.code(1139) + "Missing mandatory parameter: " + PARAM_SYSTEM);
342                }
343        }
344
345        private void validateHaveFiles(List<ICompositeType> theFiles, List<IBaseResource> theCodeSystems) {
346                if (theFiles != null) {
347                        for (ICompositeType nextFile : theFiles) {
348                                if (!nextFile.isEmpty()) {
349                                        return;
350                                }
351                        }
352                }
353                if (theCodeSystems != null) {
354                        for (IBaseResource next : theCodeSystems) {
355                                if (!next.isEmpty()) {
356                                        return;
357                                }
358                        }
359                }
360                throw new InvalidRequestException(Msg.code(1140) + "Missing mandatory parameter: " + PARAM_FILE);
361        }
362
363        @Nonnull
364        private List<ITermLoaderSvc.FileDescriptor> convertAttachmentsToFileDescriptors(@OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment") List<ICompositeType> theFiles) {
365                List<ITermLoaderSvc.FileDescriptor> files = new ArrayList<>();
366                if (theFiles != null) {
367                        for (ICompositeType next : theFiles) {
368
369                                String nextUrl = AttachmentUtil.getOrCreateUrl(getContext(), next).getValue();
370                                ValidateUtil.isNotBlankOrThrowUnprocessableEntity(nextUrl, "Missing Attachment.url value");
371
372                                byte[] nextData;
373                                if (nextUrl.startsWith("localfile:")) {
374                                        String nextLocalFile = nextUrl.substring("localfile:".length());
375
376
377                                        if (isNotBlank(nextLocalFile)) {
378                                                ourLog.info("Reading in local file: {}", nextLocalFile);
379                                                File nextFile = new File(nextLocalFile);
380                                                if (!nextFile.exists() || !nextFile.isFile()) {
381                                                        throw new InvalidRequestException(Msg.code(1141) + "Unknown file: " + nextFile.getName());
382                                                }
383                                                files.add(new FileBackedFileDescriptor(nextFile));
384                                        }
385
386                                } else {
387                                        nextData = AttachmentUtil.getOrCreateData(getContext(), next).getValue();
388                                        ValidateUtil.isTrueOrThrowInvalidRequest(nextData != null && nextData.length > 0, "Missing Attachment.data value");
389                                        files.add(new ITermLoaderSvc.ByteArrayFileDescriptor(nextUrl, nextData));
390                                }
391                        }
392                }
393                return files;
394        }
395
396        private IBaseParameters toDeltaResponse(UploadStatistics theOutcome) {
397                IBaseParameters retVal = ParametersUtil.newInstance(getContext());
398                ParametersUtil.addParameterToParametersInteger(getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, theOutcome.getUpdatedConceptCount());
399                ParametersUtil.addParameterToParametersReference(getContext(), retVal, RESP_PARAM_TARGET, theOutcome.getTarget().getValue());
400                return retVal;
401        }
402
403        public static class FileBackedFileDescriptor implements ITermLoaderSvc.FileDescriptor {
404                private final File myNextFile;
405
406                public FileBackedFileDescriptor(File theNextFile) {
407                        myNextFile = theNextFile;
408                }
409
410                @Override
411                public String getFilename() {
412                        return myNextFile.getAbsolutePath();
413                }
414
415                @Override
416                public InputStream getInputStream() {
417                        try {
418                                return new FileInputStream(myNextFile);
419                        } catch (FileNotFoundException theE) {
420                                throw new InternalErrorException(Msg.code(1142) + theE);
421                        }
422                }
423        }
424
425        private static String csvEscape(String theValue) {
426                return '"' +
427                        theValue
428                                .replace("\"", "\"\"")
429                                .replace("\n", "\\n")
430                                .replace("\r", "") +
431                        '"';
432        }
433}