Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.provider;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.context.RuntimeResourceDefinition;
024import ca.uhn.fhir.i18n.Msg;
025import ca.uhn.fhir.jpa.model.util.JpaConstants;
026import ca.uhn.fhir.jpa.term.TermLoaderSvcImpl;
027import ca.uhn.fhir.jpa.term.UploadStatistics;
028import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
029import ca.uhn.fhir.jpa.term.custom.ConceptHandler;
030import ca.uhn.fhir.jpa.term.custom.HierarchyHandler;
031import ca.uhn.fhir.jpa.term.custom.PropertyHandler;
032import ca.uhn.fhir.rest.annotation.Operation;
033import ca.uhn.fhir.rest.annotation.OperationParam;
034import ca.uhn.fhir.rest.api.server.RequestDetails;
035import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
036import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
037import ca.uhn.fhir.util.AttachmentUtil;
038import ca.uhn.fhir.util.ParametersUtil;
039import ca.uhn.fhir.util.ValidateUtil;
040import com.google.common.base.Charsets;
041import com.google.common.collect.ArrayListMultimap;
042import com.google.common.collect.Multimap;
043import jakarta.annotation.Nonnull;
044import jakarta.servlet.http.HttpServletRequest;
045import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_30_40;
046import org.hl7.fhir.convertors.advisors.impl.BaseAdvisor_40_50;
047import org.hl7.fhir.convertors.factory.VersionConvertorFactory_30_40;
048import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
049import org.hl7.fhir.instance.model.api.IBaseParameters;
050import org.hl7.fhir.instance.model.api.IBaseResource;
051import org.hl7.fhir.instance.model.api.ICompositeType;
052import org.hl7.fhir.instance.model.api.IPrimitiveType;
053import org.hl7.fhir.r4.model.CodeSystem;
054import org.springframework.beans.factory.annotation.Autowired;
055
056import java.io.File;
057import java.io.FileInputStream;
058import java.io.FileNotFoundException;
059import java.io.InputStream;
060import java.util.ArrayList;
061import java.util.LinkedHashMap;
062import java.util.List;
063import java.util.Map;
064
065import static org.apache.commons.lang3.StringUtils.*;
066
067public class TerminologyUploaderProvider extends BaseJpaProvider {
068
069        public static final String PARAM_FILE = "file";
070        public static final String PARAM_CODESYSTEM = "codeSystem";
071        public static final String PARAM_SYSTEM = "system";
072        private static final String RESP_PARAM_CONCEPT_COUNT = "conceptCount";
073        private static final String RESP_PARAM_TARGET = "target";
074        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyUploaderProvider.class);
075        private static final String RESP_PARAM_SUCCESS = "success";
076
077        @Autowired
078        private ITermLoaderSvc myTerminologyLoaderSvc;
079
080        /**
081         * Constructor
082         */
083        public TerminologyUploaderProvider() {
084                this(null, null);
085        }
086
087        /**
088         * Constructor
089         */
090        public TerminologyUploaderProvider(FhirContext theContext, ITermLoaderSvc theTerminologyLoaderSvc) {
091                setContext(theContext);
092                myTerminologyLoaderSvc = theTerminologyLoaderSvc;
093        }
094
095        /**
096         * <code>
097         * $upload-external-codesystem
098         * </code>
099         */
100        @Operation(
101                        typeName = "CodeSystem",
102                        name = JpaConstants.OPERATION_UPLOAD_EXTERNAL_CODE_SYSTEM,
103                        idempotent = false,
104                        returnParameters = {
105                                //              @OperationParam(name = "conceptCount", type = IntegerType.class, min = 1)
106                        })
107        public IBaseParameters uploadSnapshot(
108                        HttpServletRequest theServletRequest,
109                        @OperationParam(name = PARAM_SYSTEM, min = 1, typeName = "uri") IPrimitiveType<String> theCodeSystemUrl,
110                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
111                                        List<ICompositeType> theFiles,
112                        RequestDetails theRequestDetails) {
113
114                startRequest(theServletRequest);
115
116                if (theCodeSystemUrl == null || isBlank(theCodeSystemUrl.getValueAsString())) {
117                        throw new InvalidRequestException(Msg.code(1137) + "Missing mandatory parameter: " + PARAM_SYSTEM);
118                }
119
120                if (theFiles == null || theFiles.size() == 0) {
121                        throw new InvalidRequestException(
122                                        Msg.code(1138) + "No '" + PARAM_FILE + "' parameter, or package had no data");
123                }
124                for (ICompositeType next : theFiles) {
125                        ValidateUtil.isTrueOrThrowInvalidRequest(
126                                        getContext().getElementDefinition(next.getClass()).getName().equals("Attachment"),
127                                        "Package must be of type Attachment");
128                }
129
130                try {
131                        List<ITermLoaderSvc.FileDescriptor> localFiles = convertAttachmentsToFileDescriptors(theFiles);
132
133                        String codeSystemUrl = theCodeSystemUrl.getValue();
134                        codeSystemUrl = trim(codeSystemUrl);
135
136                        UploadStatistics stats;
137                        switch (codeSystemUrl) {
138                                case ITermLoaderSvc.ICD10_URI:
139                                        stats = myTerminologyLoaderSvc.loadIcd10(localFiles, theRequestDetails);
140                                        break;
141                                case ITermLoaderSvc.ICD10CM_URI:
142                                        stats = myTerminologyLoaderSvc.loadIcd10cm(localFiles, theRequestDetails);
143                                        break;
144                                case ITermLoaderSvc.IMGTHLA_URI:
145                                        stats = myTerminologyLoaderSvc.loadImgthla(localFiles, theRequestDetails);
146                                        break;
147                                case ITermLoaderSvc.LOINC_URI:
148                                        stats = myTerminologyLoaderSvc.loadLoinc(localFiles, theRequestDetails);
149                                        break;
150                                case ITermLoaderSvc.SCT_URI:
151                                        stats = myTerminologyLoaderSvc.loadSnomedCt(localFiles, theRequestDetails);
152                                        break;
153                                default:
154                                        stats = myTerminologyLoaderSvc.loadCustom(codeSystemUrl, localFiles, theRequestDetails);
155                                        break;
156                        }
157
158                        IBaseParameters retVal = ParametersUtil.newInstance(getContext());
159                        ParametersUtil.addParameterToParametersBoolean(getContext(), retVal, RESP_PARAM_SUCCESS, true);
160                        ParametersUtil.addParameterToParametersInteger(
161                                        getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, stats.getUpdatedConceptCount());
162                        ParametersUtil.addParameterToParametersReference(
163                                        getContext(), retVal, RESP_PARAM_TARGET, stats.getTarget().getValue());
164
165                        return retVal;
166                } finally {
167                        endRequest(theServletRequest);
168                }
169        }
170
171        /**
172         * <code>
173         * $apply-codesystem-delta-add
174         * </code>
175         */
176        @Operation(
177                        typeName = "CodeSystem",
178                        name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_ADD,
179                        idempotent = false,
180                        returnParameters = {})
181        public IBaseParameters uploadDeltaAdd(
182                        HttpServletRequest theServletRequest,
183                        @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
184                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
185                                        List<ICompositeType> theFiles,
186                        @OperationParam(
187                                                        name = PARAM_CODESYSTEM,
188                                                        min = 0,
189                                                        max = OperationParam.MAX_UNLIMITED,
190                                                        typeName = "CodeSystem")
191                                        List<IBaseResource> theCodeSystems,
192                        RequestDetails theRequestDetails) {
193
194                startRequest(theServletRequest);
195                try {
196                        validateHaveSystem(theSystem);
197                        validateHaveFiles(theFiles, theCodeSystems);
198
199                        List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
200                        convertCodeSystemsToFileDescriptors(files, theCodeSystems);
201                        UploadStatistics outcome =
202                                        myTerminologyLoaderSvc.loadDeltaAdd(theSystem.getValue(), files, theRequestDetails);
203                        return toDeltaResponse(outcome);
204                } finally {
205                        endRequest(theServletRequest);
206                }
207        }
208
209        /**
210         * <code>
211         * $apply-codesystem-delta-remove
212         * </code>
213         */
214        @Operation(
215                        typeName = "CodeSystem",
216                        name = JpaConstants.OPERATION_APPLY_CODESYSTEM_DELTA_REMOVE,
217                        idempotent = false,
218                        returnParameters = {})
219        public IBaseParameters uploadDeltaRemove(
220                        HttpServletRequest theServletRequest,
221                        @OperationParam(name = PARAM_SYSTEM, min = 1, max = 1, typeName = "uri") IPrimitiveType<String> theSystem,
222                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
223                                        List<ICompositeType> theFiles,
224                        @OperationParam(
225                                                        name = PARAM_CODESYSTEM,
226                                                        min = 0,
227                                                        max = OperationParam.MAX_UNLIMITED,
228                                                        typeName = "CodeSystem")
229                                        List<IBaseResource> theCodeSystems,
230                        RequestDetails theRequestDetails) {
231
232                startRequest(theServletRequest);
233                try {
234                        validateHaveSystem(theSystem);
235                        validateHaveFiles(theFiles, theCodeSystems);
236
237                        List<ITermLoaderSvc.FileDescriptor> files = convertAttachmentsToFileDescriptors(theFiles);
238                        convertCodeSystemsToFileDescriptors(files, theCodeSystems);
239                        UploadStatistics outcome =
240                                        myTerminologyLoaderSvc.loadDeltaRemove(theSystem.getValue(), files, theRequestDetails);
241                        return toDeltaResponse(outcome);
242                } finally {
243                        endRequest(theServletRequest);
244                }
245        }
246
247        private void convertCodeSystemsToFileDescriptors(
248                        List<ITermLoaderSvc.FileDescriptor> theFiles, List<IBaseResource> theCodeSystems) {
249                Map<String, String> codes = new LinkedHashMap<>();
250                Map<String, List<CodeSystem.ConceptPropertyComponent>> codeToProperties = new LinkedHashMap<>();
251
252                Multimap<String, String> codeToParentCodes = ArrayListMultimap.create();
253
254                if (theCodeSystems != null) {
255                        for (IBaseResource nextCodeSystemUncast : theCodeSystems) {
256                                CodeSystem nextCodeSystem = canonicalizeCodeSystem(nextCodeSystemUncast);
257                                convertCodeSystemCodesToCsv(
258                                                nextCodeSystem.getConcept(), codes, codeToProperties, null, codeToParentCodes);
259                        }
260                }
261
262                // Create concept file
263                if (codes.size() > 0) {
264                        StringBuilder b = new StringBuilder();
265                        b.append(ConceptHandler.CODE);
266                        b.append(",");
267                        b.append(ConceptHandler.DISPLAY);
268                        b.append("\n");
269                        for (Map.Entry<String, String> nextEntry : codes.entrySet()) {
270                                b.append(csvEscape(nextEntry.getKey()));
271                                b.append(",");
272                                b.append(csvEscape(nextEntry.getValue()));
273                                b.append("\n");
274                        }
275                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
276                        String fileName = TermLoaderSvcImpl.CUSTOM_CONCEPTS_FILE;
277                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor =
278                                        new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
279                        theFiles.add(fileDescriptor);
280                }
281
282                // Create hierarchy file
283                if (codeToParentCodes.size() > 0) {
284                        StringBuilder b = new StringBuilder();
285                        b.append(HierarchyHandler.CHILD);
286                        b.append(",");
287                        b.append(HierarchyHandler.PARENT);
288                        b.append("\n");
289                        for (Map.Entry<String, String> nextEntry : codeToParentCodes.entries()) {
290                                b.append(csvEscape(nextEntry.getKey()));
291                                b.append(",");
292                                b.append(csvEscape(nextEntry.getValue()));
293                                b.append("\n");
294                        }
295                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
296                        String fileName = TermLoaderSvcImpl.CUSTOM_HIERARCHY_FILE;
297                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor =
298                                        new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
299                        theFiles.add(fileDescriptor);
300                }
301                // Create codeToProperties file
302                if (codeToProperties.size() > 0) {
303                        StringBuilder b = new StringBuilder();
304                        b.append(PropertyHandler.CODE);
305                        b.append(",");
306                        b.append(PropertyHandler.KEY);
307                        b.append(",");
308                        b.append(PropertyHandler.VALUE);
309                        b.append(",");
310                        b.append(PropertyHandler.TYPE);
311                        b.append("\n");
312
313                        for (Map.Entry<String, List<CodeSystem.ConceptPropertyComponent>> nextEntry : codeToProperties.entrySet()) {
314                                for (CodeSystem.ConceptPropertyComponent propertyComponent : nextEntry.getValue()) {
315                                        b.append(csvEscape(nextEntry.getKey()));
316                                        b.append(",");
317                                        b.append(csvEscape(propertyComponent.getCode()));
318                                        b.append(",");
319                                        // TODO: check this for different types, other types should be added once
320                                        // TermConceptPropertyTypeEnum contain different types
321                                        b.append(csvEscape(propertyComponent.getValueStringType().getValue()));
322                                        b.append(",");
323                                        b.append(csvEscape(propertyComponent.getValue().primitiveValue()));
324                                        b.append("\n");
325                                }
326                        }
327                        byte[] bytes = b.toString().getBytes(Charsets.UTF_8);
328                        String fileName = TermLoaderSvcImpl.CUSTOM_PROPERTIES_FILE;
329                        ITermLoaderSvc.ByteArrayFileDescriptor fileDescriptor =
330                                        new ITermLoaderSvc.ByteArrayFileDescriptor(fileName, bytes);
331                        theFiles.add(fileDescriptor);
332                }
333        }
334
335        @SuppressWarnings("EnumSwitchStatementWhichMissesCases")
336        @Nonnull
337        CodeSystem canonicalizeCodeSystem(@Nonnull IBaseResource theCodeSystem) {
338                RuntimeResourceDefinition resourceDef = getContext().getResourceDefinition(theCodeSystem);
339                ValidateUtil.isTrueOrThrowInvalidRequest(
340                                resourceDef.getName().equals("CodeSystem"), "Resource '%s' is not a CodeSystem", resourceDef.getName());
341
342                CodeSystem nextCodeSystem;
343                switch (getContext().getVersion().getVersion()) {
344                        case DSTU3:
345                                nextCodeSystem = (CodeSystem) VersionConvertorFactory_30_40.convertResource(
346                                                (org.hl7.fhir.dstu3.model.CodeSystem) theCodeSystem, new BaseAdvisor_30_40(false));
347                                break;
348                        case R5:
349                                nextCodeSystem = (CodeSystem) VersionConvertorFactory_40_50.convertResource(
350                                                (org.hl7.fhir.r5.model.CodeSystem) theCodeSystem, new BaseAdvisor_40_50(false));
351                                break;
352                        default:
353                                nextCodeSystem = (CodeSystem) theCodeSystem;
354                }
355                return nextCodeSystem;
356        }
357
358        private void convertCodeSystemCodesToCsv(
359                        List<CodeSystem.ConceptDefinitionComponent> theConcept,
360                        Map<String, String> theCodes,
361                        Map<String, List<CodeSystem.ConceptPropertyComponent>> theProperties,
362                        String theParentCode,
363                        Multimap<String, String> theCodeToParentCodes) {
364                for (CodeSystem.ConceptDefinitionComponent nextConcept : theConcept) {
365                        if (isNotBlank(nextConcept.getCode())) {
366                                theCodes.put(nextConcept.getCode(), nextConcept.getDisplay());
367                                if (isNotBlank(theParentCode)) {
368                                        theCodeToParentCodes.put(nextConcept.getCode(), theParentCode);
369                                }
370                                if (nextConcept.getProperty() != null) {
371                                        theProperties.put(nextConcept.getCode(), nextConcept.getProperty());
372                                }
373                                convertCodeSystemCodesToCsv(
374                                                nextConcept.getConcept(), theCodes, theProperties, nextConcept.getCode(), theCodeToParentCodes);
375                        }
376                }
377        }
378
379        private void validateHaveSystem(IPrimitiveType<String> theSystem) {
380                if (theSystem == null || isBlank(theSystem.getValueAsString())) {
381                        throw new InvalidRequestException(Msg.code(1139) + "Missing mandatory parameter: " + PARAM_SYSTEM);
382                }
383        }
384
385        private void validateHaveFiles(List<ICompositeType> theFiles, List<IBaseResource> theCodeSystems) {
386                if (theFiles != null) {
387                        for (ICompositeType nextFile : theFiles) {
388                                if (!nextFile.isEmpty()) {
389                                        return;
390                                }
391                        }
392                }
393                if (theCodeSystems != null) {
394                        for (IBaseResource next : theCodeSystems) {
395                                if (!next.isEmpty()) {
396                                        return;
397                                }
398                        }
399                }
400                throw new InvalidRequestException(Msg.code(1140) + "Missing mandatory parameter: " + PARAM_FILE);
401        }
402
403        @Nonnull
404        private List<ITermLoaderSvc.FileDescriptor> convertAttachmentsToFileDescriptors(
405                        @OperationParam(name = PARAM_FILE, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "attachment")
406                                        List<ICompositeType> theFiles) {
407                List<ITermLoaderSvc.FileDescriptor> files = new ArrayList<>();
408                if (theFiles != null) {
409                        for (ICompositeType next : theFiles) {
410
411                                String nextUrl =
412                                                AttachmentUtil.getOrCreateUrl(getContext(), next).getValue();
413                                ValidateUtil.isNotBlankOrThrowUnprocessableEntity(nextUrl, "Missing Attachment.url value");
414
415                                byte[] nextData;
416                                if (nextUrl.startsWith("localfile:")) {
417                                        String nextLocalFile = nextUrl.substring("localfile:".length());
418
419                                        if (isNotBlank(nextLocalFile)) {
420                                                ourLog.info("Reading in local file: {}", nextLocalFile);
421                                                File nextFile = new File(nextLocalFile);
422                                                if (!nextFile.exists() || !nextFile.isFile()) {
423                                                        throw new InvalidRequestException(Msg.code(1141) + "Unknown file: " + nextFile.getName());
424                                                }
425                                                files.add(new FileBackedFileDescriptor(nextFile));
426                                        }
427
428                                } else {
429                                        nextData =
430                                                        AttachmentUtil.getOrCreateData(getContext(), next).getValue();
431                                        ValidateUtil.isTrueOrThrowInvalidRequest(
432                                                        nextData != null && nextData.length > 0, "Missing Attachment.data value");
433                                        files.add(new ITermLoaderSvc.ByteArrayFileDescriptor(nextUrl, nextData));
434                                }
435                        }
436                }
437                return files;
438        }
439
440        private IBaseParameters toDeltaResponse(UploadStatistics theOutcome) {
441                IBaseParameters retVal = ParametersUtil.newInstance(getContext());
442                ParametersUtil.addParameterToParametersInteger(
443                                getContext(), retVal, RESP_PARAM_CONCEPT_COUNT, theOutcome.getUpdatedConceptCount());
444                ParametersUtil.addParameterToParametersReference(
445                                getContext(), retVal, RESP_PARAM_TARGET, theOutcome.getTarget().getValue());
446                return retVal;
447        }
448
449        public void setTerminologyLoaderSvc(ITermLoaderSvc theTermLoaderSvc) {
450                myTerminologyLoaderSvc = theTermLoaderSvc;
451        }
452
453        public static class FileBackedFileDescriptor implements ITermLoaderSvc.FileDescriptor {
454                private final File myNextFile;
455
456                public FileBackedFileDescriptor(File theNextFile) {
457                        myNextFile = theNextFile;
458                }
459
460                @Override
461                public String getFilename() {
462                        return myNextFile.getAbsolutePath();
463                }
464
465                @Override
466                public InputStream getInputStream() {
467                        try {
468                                return new FileInputStream(myNextFile);
469                        } catch (FileNotFoundException theE) {
470                                throw new InternalErrorException(Msg.code(1142) + theE);
471                        }
472                }
473        }
474
475        private static String csvEscape(String theValue) {
476                return '"' + theValue.replace("\"", "\"\"").replace("\n", "\\n").replace("\r", "") + '"';
477        }
478}