Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2024 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.term;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.i18n.Msg;
024import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
025import ca.uhn.fhir.jpa.entity.TermConcept;
026import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
027import ca.uhn.fhir.jpa.entity.TermConceptProperty;
028import ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc;
029import ca.uhn.fhir.jpa.term.api.ITermDeferredStorageSvc;
030import ca.uhn.fhir.jpa.term.api.ITermLoaderSvc;
031import ca.uhn.fhir.jpa.term.custom.CustomTerminologySet;
032import ca.uhn.fhir.jpa.term.icd10.Icd10Loader;
033import ca.uhn.fhir.jpa.term.icd10cm.Icd10CmLoader;
034import ca.uhn.fhir.jpa.term.loinc.LoincAnswerListHandler;
035import ca.uhn.fhir.jpa.term.loinc.LoincAnswerListLinkHandler;
036import ca.uhn.fhir.jpa.term.loinc.LoincCodingPropertiesHandler;
037import ca.uhn.fhir.jpa.term.loinc.LoincConsumerNameHandler;
038import ca.uhn.fhir.jpa.term.loinc.LoincDocumentOntologyHandler;
039import ca.uhn.fhir.jpa.term.loinc.LoincGroupFileHandler;
040import ca.uhn.fhir.jpa.term.loinc.LoincGroupTermsFileHandler;
041import ca.uhn.fhir.jpa.term.loinc.LoincHandler;
042import ca.uhn.fhir.jpa.term.loinc.LoincHierarchyHandler;
043import ca.uhn.fhir.jpa.term.loinc.LoincIeeeMedicalDeviceCodeHandler;
044import ca.uhn.fhir.jpa.term.loinc.LoincImagingDocumentCodeHandler;
045import ca.uhn.fhir.jpa.term.loinc.LoincLinguisticVariantHandler;
046import ca.uhn.fhir.jpa.term.loinc.LoincLinguisticVariantsHandler;
047import ca.uhn.fhir.jpa.term.loinc.LoincMapToHandler;
048import ca.uhn.fhir.jpa.term.loinc.LoincParentGroupFileHandler;
049import ca.uhn.fhir.jpa.term.loinc.LoincPartHandler;
050import ca.uhn.fhir.jpa.term.loinc.LoincPartLinkHandler;
051import ca.uhn.fhir.jpa.term.loinc.LoincPartRelatedCodeMappingHandler;
052import ca.uhn.fhir.jpa.term.loinc.LoincRsnaPlaybookHandler;
053import ca.uhn.fhir.jpa.term.loinc.LoincTop2000LabResultsSiHandler;
054import ca.uhn.fhir.jpa.term.loinc.LoincTop2000LabResultsUsHandler;
055import ca.uhn.fhir.jpa.term.loinc.LoincUniversalOrderSetHandler;
056import ca.uhn.fhir.jpa.term.loinc.LoincXmlFileZipContentsHandler;
057import ca.uhn.fhir.jpa.term.loinc.PartTypeAndPartName;
058import ca.uhn.fhir.jpa.term.snomedct.SctHandlerConcept;
059import ca.uhn.fhir.jpa.term.snomedct.SctHandlerDescription;
060import ca.uhn.fhir.jpa.term.snomedct.SctHandlerRelationship;
061import ca.uhn.fhir.jpa.util.Counter;
062import ca.uhn.fhir.rest.api.EncodingEnum;
063import ca.uhn.fhir.rest.api.server.RequestDetails;
064import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
065import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
066import ca.uhn.fhir.util.ValidateUtil;
067import com.google.common.annotations.VisibleForTesting;
068import com.google.common.base.Charsets;
069import jakarta.annotation.Nonnull;
070import org.apache.commons.csv.CSVFormat;
071import org.apache.commons.csv.CSVParser;
072import org.apache.commons.csv.CSVRecord;
073import org.apache.commons.csv.QuoteMode;
074import org.apache.commons.io.IOUtils;
075import org.apache.commons.lang3.ObjectUtils;
076import org.apache.commons.lang3.StringUtils;
077import org.apache.commons.lang3.Validate;
078import org.hl7.fhir.instance.model.api.IIdType;
079import org.hl7.fhir.r4.model.CodeSystem;
080import org.hl7.fhir.r4.model.ConceptMap;
081import org.hl7.fhir.r4.model.Enumerations;
082import org.hl7.fhir.r4.model.ValueSet;
083import org.springframework.aop.support.AopUtils;
084import org.springframework.beans.factory.annotation.Autowired;
085import org.xml.sax.SAXException;
086
087import java.io.IOException;
088import java.io.InputStream;
089import java.io.InputStreamReader;
090import java.io.LineNumberReader;
091import java.io.Reader;
092import java.util.ArrayList;
093import java.util.Arrays;
094import java.util.Collections;
095import java.util.Date;
096import java.util.HashMap;
097import java.util.HashSet;
098import java.util.Iterator;
099import java.util.List;
100import java.util.Locale;
101import java.util.Map;
102import java.util.Map.Entry;
103import java.util.Optional;
104import java.util.Properties;
105import java.util.Set;
106import java.util.stream.Collectors;
107
108import static ca.uhn.fhir.jpa.term.api.ITermCodeSystemStorageSvc.MAKE_LOADING_VERSION_CURRENT;
109import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_FILE;
110import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_FILE_DEFAULT;
111import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_FILE;
112import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_ANSWERLIST_LINK_FILE_DEFAULT;
113import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYSTEM_MAKE_CURRENT;
114import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CODESYSTEM_VERSION;
115import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CONSUMER_NAME_FILE;
116import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_CONSUMER_NAME_FILE_DEFAULT;
117import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_DOCUMENT_ONTOLOGY_FILE;
118import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT;
119import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_FILE;
120import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_FILE_DEFAULT;
121import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_FILE;
122import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_FILE_DEFAULT;
123import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_TERMS_FILE;
124import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_GROUP_TERMS_FILE_DEFAULT;
125import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_HIERARCHY_FILE;
126import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_HIERARCHY_FILE_DEFAULT;
127import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE;
128import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT;
129import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IMAGING_DOCUMENT_CODES_FILE;
130import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT;
131import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_LINGUISTIC_VARIANTS_FILE;
132import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_LINGUISTIC_VARIANTS_FILE_DEFAULT;
133import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_LINGUISTIC_VARIANTS_PATH;
134import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_LINGUISTIC_VARIANTS_PATH_DEFAULT;
135import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_MAPTO_FILE;
136import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_MAPTO_FILE_DEFAULT;
137import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PARENT_GROUP_FILE;
138import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PARENT_GROUP_FILE_DEFAULT;
139import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_FILE;
140import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_FILE_DEFAULT;
141import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE;
142import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_DEFAULT;
143import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_PRIMARY;
144import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_PRIMARY_DEFAULT;
145import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_SUPPLEMENTARY;
146import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT;
147import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_RELATED_CODE_MAPPING_FILE;
148import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT;
149import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_RSNA_PLAYBOOK_FILE;
150import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_RSNA_PLAYBOOK_FILE_DEFAULT;
151import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE;
152import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT;
153import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE;
154import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT;
155import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE;
156import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT;
157import static ca.uhn.fhir.jpa.term.loinc.LoincUploadPropertiesEnum.LOINC_UPLOAD_PROPERTIES_FILE;
158import static org.apache.commons.lang3.StringUtils.isBlank;
159import static org.apache.commons.lang3.StringUtils.isNotBlank;
160import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_ALL_VALUESET_ID;
161import static org.hl7.fhir.common.hapi.validation.support.ValidationConstants.LOINC_GENERIC_VALUESET_URL;
162
163public class TermLoaderSvcImpl implements ITermLoaderSvc {
164        public static final String CUSTOM_CONCEPTS_FILE = "concepts.csv";
165        public static final String CUSTOM_HIERARCHY_FILE = "hierarchy.csv";
166        public static final String CUSTOM_PROPERTIES_FILE = "properties.csv";
167        static final String IMGTHLA_HLA_NOM_TXT = "hla_nom.txt";
168        static final String IMGTHLA_HLA_XML = "hla.xml";
169        static final String CUSTOM_CODESYSTEM_JSON = "codesystem.json";
170        private static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full_";
171        private static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full";
172        private static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full";
173        private static final String CUSTOM_CODESYSTEM_XML = "codesystem.xml";
174
175        private static final int LOG_INCREMENT = 1000;
176        private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TermLoaderSvcImpl.class);
177        // FYI: Hardcoded to R4 because that's what the term svc uses internally
178        private final FhirContext myCtx = FhirContext.forR4Cached();
179        private final ITermDeferredStorageSvc myDeferredStorageSvc;
180        private final ITermCodeSystemStorageSvc myCodeSystemStorageSvc;
181
182        @Autowired
183        public TermLoaderSvcImpl(
184                        ITermDeferredStorageSvc theDeferredStorageSvc, ITermCodeSystemStorageSvc theCodeSystemStorageSvc) {
185                this(theDeferredStorageSvc, theCodeSystemStorageSvc, true);
186        }
187
188        private TermLoaderSvcImpl(
189                        ITermDeferredStorageSvc theDeferredStorageSvc,
190                        ITermCodeSystemStorageSvc theCodeSystemStorageSvc,
191                        boolean theProxyCheck) {
192                if (theProxyCheck) {
193                        // If these validations start failing, it likely means a cyclic dependency has been introduced into the
194                        // Spring Application
195                        // Context that is preventing the Spring auto-proxy bean post-processor from being able to proxy these
196                        // beans.  Check
197                        // for recent changes to the Spring @Configuration that may have caused this.
198                        Validate.isTrue(
199                                        AopUtils.isAopProxy(theDeferredStorageSvc),
200                                        theDeferredStorageSvc.getClass().getName()
201                                                        + " is not a proxy.  @Transactional annotations will be ignored.");
202                        Validate.isTrue(
203                                        AopUtils.isAopProxy(theCodeSystemStorageSvc),
204                                        theCodeSystemStorageSvc.getClass().getName()
205                                                        + " is not a proxy.  @Transactional annotations will be ignored.");
206                }
207                myDeferredStorageSvc = theDeferredStorageSvc;
208                myCodeSystemStorageSvc = theCodeSystemStorageSvc;
209        }
210
211        @VisibleForTesting
212        public static TermLoaderSvcImpl withoutProxyCheck(
213                        ITermDeferredStorageSvc theTermDeferredStorageSvc, ITermCodeSystemStorageSvc theTermCodeSystemStorageSvc) {
214                return new TermLoaderSvcImpl(theTermDeferredStorageSvc, theTermCodeSystemStorageSvc, false);
215        }
216
217        @Override
218        public UploadStatistics loadImgthla(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
219                try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
220                        List<String> mandatoryFilenameFragments = Arrays.asList(IMGTHLA_HLA_NOM_TXT, IMGTHLA_HLA_XML);
221                        descriptors.verifyMandatoryFilesExist(mandatoryFilenameFragments);
222
223                        ourLog.info("Beginning IMGTHLA processing");
224
225                        return processImgthlaFiles(descriptors, theRequestDetails);
226                }
227        }
228
229        @VisibleForTesting
230        LoadedFileDescriptors getLoadedFileDescriptors(List<FileDescriptor> theFiles) {
231                return new LoadedFileDescriptors(theFiles);
232        }
233
234        @Override
235        public UploadStatistics loadLoinc(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
236                try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
237                        Properties uploadProperties = getProperties(descriptors, LOINC_UPLOAD_PROPERTIES_FILE.getCode());
238
239                        String codeSystemVersionId = uploadProperties.getProperty(LOINC_CODESYSTEM_VERSION.getCode());
240                        boolean isMakeCurrentVersion =
241                                        Boolean.parseBoolean(uploadProperties.getProperty(LOINC_CODESYSTEM_MAKE_CURRENT.getCode(), "true"));
242
243                        if (StringUtils.isBlank(codeSystemVersionId) && !isMakeCurrentVersion) {
244                                throw new InvalidRequestException(
245                                                Msg.code(864) + "'" + LOINC_CODESYSTEM_VERSION.getCode() + "' property is required when '"
246                                                                + LOINC_CODESYSTEM_MAKE_CURRENT.getCode() + "' property is 'false'");
247                        }
248
249                        List<String> mandatoryFilenameFragments = Arrays.asList(
250                                        uploadProperties.getProperty(
251                                                        LOINC_ANSWERLIST_FILE.getCode(), LOINC_ANSWERLIST_FILE_DEFAULT.getCode()),
252                                        uploadProperties.getProperty(
253                                                        LOINC_ANSWERLIST_LINK_FILE.getCode(), LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode()),
254                                        uploadProperties.getProperty(
255                                                        LOINC_DOCUMENT_ONTOLOGY_FILE.getCode(), LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode()),
256                                        uploadProperties.getProperty(LOINC_FILE.getCode(), LOINC_FILE_DEFAULT.getCode()),
257                                        uploadProperties.getProperty(
258                                                        LOINC_HIERARCHY_FILE.getCode(), LOINC_HIERARCHY_FILE_DEFAULT.getCode()),
259                                        uploadProperties.getProperty(
260                                                        LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE.getCode(),
261                                                        LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT.getCode()),
262                                        uploadProperties.getProperty(
263                                                        LOINC_IMAGING_DOCUMENT_CODES_FILE.getCode(),
264                                                        LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT.getCode()),
265                                        uploadProperties.getProperty(LOINC_PART_FILE.getCode(), LOINC_PART_FILE_DEFAULT.getCode()),
266                                        uploadProperties.getProperty(
267                                                        LOINC_PART_RELATED_CODE_MAPPING_FILE.getCode(),
268                                                        LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()),
269                                        uploadProperties.getProperty(
270                                                        LOINC_RSNA_PLAYBOOK_FILE.getCode(), LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()),
271                                        uploadProperties.getProperty(
272                                                        LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE.getCode(),
273                                                        LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT.getCode()));
274                        descriptors.verifyMandatoryFilesExist(mandatoryFilenameFragments);
275
276                        List<String> splitPartLinkFilenameFragments = Arrays.asList(
277                                        uploadProperties.getProperty(
278                                                        LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()),
279                                        uploadProperties.getProperty(
280                                                        LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(),
281                                                        LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()));
282                        descriptors.verifyPartLinkFilesExist(
283                                        splitPartLinkFilenameFragments,
284                                        uploadProperties.getProperty(
285                                                        LOINC_PART_LINK_FILE.getCode(), LOINC_PART_LINK_FILE_DEFAULT.getCode()));
286
287                        List<String> optionalFilenameFragments = Arrays.asList(
288                                        uploadProperties.getProperty(LOINC_GROUP_FILE.getCode(), LOINC_GROUP_FILE_DEFAULT.getCode()),
289                                        uploadProperties.getProperty(
290                                                        LOINC_GROUP_TERMS_FILE.getCode(), LOINC_GROUP_TERMS_FILE_DEFAULT.getCode()),
291                                        uploadProperties.getProperty(
292                                                        LOINC_PARENT_GROUP_FILE.getCode(), LOINC_PARENT_GROUP_FILE_DEFAULT.getCode()),
293                                        uploadProperties.getProperty(
294                                                        LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE.getCode(),
295                                                        LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT.getCode()),
296                                        uploadProperties.getProperty(
297                                                        LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE.getCode(),
298                                                        LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT.getCode()),
299                                        uploadProperties.getProperty(LOINC_MAPTO_FILE.getCode(), LOINC_MAPTO_FILE_DEFAULT.getCode()),
300
301                                        // -- optional consumer name
302                                        uploadProperties.getProperty(
303                                                        LOINC_CONSUMER_NAME_FILE.getCode(), LOINC_CONSUMER_NAME_FILE_DEFAULT.getCode()),
304                                        uploadProperties.getProperty(
305                                                        LOINC_LINGUISTIC_VARIANTS_FILE.getCode(),
306                                                        LOINC_LINGUISTIC_VARIANTS_FILE_DEFAULT.getCode()));
307
308                        descriptors.verifyOptionalFilesExist(optionalFilenameFragments);
309
310                        ourLog.info("Beginning LOINC processing");
311
312                        if (isMakeCurrentVersion) {
313                                if (codeSystemVersionId != null) {
314                                        processLoincFiles(descriptors, theRequestDetails, uploadProperties, false);
315                                        uploadProperties.remove(LOINC_CODESYSTEM_VERSION.getCode());
316                                }
317                                ourLog.info("Uploading CodeSystem and making it current version");
318
319                        } else {
320                                ourLog.info("Uploading CodeSystem without updating current version");
321                        }
322
323                        theRequestDetails.getUserData().put(MAKE_LOADING_VERSION_CURRENT, isMakeCurrentVersion);
324                        return processLoincFiles(descriptors, theRequestDetails, uploadProperties, true);
325                }
326        }
327
328        @Override
329        public UploadStatistics loadSnomedCt(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
330                try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
331
332                        List<String> expectedFilenameFragments =
333                                        Arrays.asList(SCT_FILE_DESCRIPTION, SCT_FILE_RELATIONSHIP, SCT_FILE_CONCEPT);
334                        descriptors.verifyMandatoryFilesExist(expectedFilenameFragments);
335
336                        ourLog.info("Beginning SNOMED CT processing");
337
338                        return processSnomedCtFiles(descriptors, theRequestDetails);
339                }
340        }
341
342        @Override
343        public UploadStatistics loadIcd10(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
344                ourLog.info("Beginning ICD-10 processing");
345
346                CodeSystem codeSystem = new CodeSystem();
347                codeSystem.setUrl(ICD10_URI);
348                codeSystem.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
349                codeSystem.setStatus(Enumerations.PublicationStatus.ACTIVE);
350
351                TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
352                int count = 0;
353
354                try (LoadedFileDescriptors compressedDescriptors = getLoadedFileDescriptors(theFiles)) {
355                        for (FileDescriptor nextDescriptor : compressedDescriptors.getUncompressedFileDescriptors()) {
356                                if (nextDescriptor.getFilename().toLowerCase(Locale.US).endsWith(".xml")) {
357                                        try (InputStream inputStream = nextDescriptor.getInputStream();
358                                                        InputStreamReader reader = new InputStreamReader(inputStream, Charsets.UTF_8)) {
359                                                Icd10Loader loader = new Icd10Loader(codeSystem, codeSystemVersion);
360                                                loader.load(reader);
361                                                count += loader.getConceptCount();
362                                        }
363                                }
364                        }
365                } catch (IOException | SAXException e) {
366                        throw new InternalErrorException(Msg.code(2135) + e);
367                }
368
369                codeSystem.setVersion(codeSystemVersion.getCodeSystemVersionId());
370
371                IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, codeSystem, null, null);
372                return new UploadStatistics(count, target);
373        }
374
375        @Override
376        public UploadStatistics loadIcd10cm(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
377                ourLog.info("Beginning ICD-10-cm processing");
378
379                CodeSystem cs = new CodeSystem();
380                cs.setUrl(ICD10CM_URI);
381                cs.setName("ICD-10-CM");
382                cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
383                cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
384
385                TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
386                int count = 0;
387
388                try (LoadedFileDescriptors compressedDescriptors = getLoadedFileDescriptors(theFiles)) {
389                        for (FileDescriptor nextDescriptor : compressedDescriptors.getUncompressedFileDescriptors()) {
390                                if (nextDescriptor.getFilename().toLowerCase(Locale.US).endsWith(".xml")) {
391                                        try (InputStream inputStream = nextDescriptor.getInputStream();
392                                                        InputStreamReader reader = new InputStreamReader(inputStream, Charsets.UTF_8)) {
393                                                Icd10CmLoader loader = new Icd10CmLoader(codeSystemVersion);
394                                                loader.load(reader);
395                                                count += loader.getConceptCount();
396                                        }
397                                }
398                        }
399                } catch (IOException | SAXException e) {
400                        throw new InternalErrorException(Msg.code(865) + e);
401                }
402
403                cs.setVersion(codeSystemVersion.getCodeSystemVersionId());
404
405                IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
406                return new UploadStatistics(count, target);
407        }
408
409        @Override
410        public UploadStatistics loadCustom(
411                        String theSystem, List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
412                try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
413                        Optional<String> codeSystemContent = loadFile(descriptors, CUSTOM_CODESYSTEM_JSON, CUSTOM_CODESYSTEM_XML);
414                        CodeSystem codeSystem;
415                        if (codeSystemContent.isPresent()) {
416                                codeSystem = EncodingEnum.detectEncoding(codeSystemContent.get())
417                                                .newParser(myCtx)
418                                                .parseResource(CodeSystem.class, codeSystemContent.get());
419                                ValidateUtil.isTrueOrThrowInvalidRequest(
420                                                theSystem.equalsIgnoreCase(codeSystem.getUrl()),
421                                                "CodeSystem.url does not match the supplied system: %s",
422                                                theSystem);
423                                ValidateUtil.isTrueOrThrowInvalidRequest(
424                                                CodeSystem.CodeSystemContentMode.NOTPRESENT.equals(codeSystem.getContent()),
425                                                "CodeSystem.content does not match the expected value: %s",
426                                                CodeSystem.CodeSystemContentMode.NOTPRESENT.toCode());
427                        } else {
428                                codeSystem = new CodeSystem();
429                                codeSystem.setUrl(theSystem);
430                                codeSystem.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
431                        }
432
433                        CustomTerminologySet terminologySet = CustomTerminologySet.load(descriptors, false);
434                        TermCodeSystemVersion csv = terminologySet.toCodeSystemVersion();
435
436                        IIdType target = storeCodeSystem(theRequestDetails, csv, codeSystem, null, null);
437                        return new UploadStatistics(terminologySet.getSize(), target);
438                }
439        }
440
441        @Override
442        public UploadStatistics loadDeltaAdd(
443                        String theSystem, List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
444                ourLog.info(
445                                "Processing terminology delta ADD for system[{}] with files: {}",
446                                theSystem,
447                                theFiles.stream().map(FileDescriptor::getFilename).collect(Collectors.toList()));
448                try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
449                        CustomTerminologySet terminologySet = CustomTerminologySet.load(descriptors, false);
450                        return myCodeSystemStorageSvc.applyDeltaCodeSystemsAdd(theSystem, terminologySet);
451                }
452        }
453
454        @Override
455        public UploadStatistics loadDeltaRemove(
456                        String theSystem, List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
457                ourLog.info(
458                                "Processing terminology delta REMOVE for system[{}] with files: {}",
459                                theSystem,
460                                theFiles.stream().map(FileDescriptor::getFilename).collect(Collectors.toList()));
461                try (LoadedFileDescriptors descriptors = getLoadedFileDescriptors(theFiles)) {
462                        CustomTerminologySet terminologySet = CustomTerminologySet.load(descriptors, true);
463                        return myCodeSystemStorageSvc.applyDeltaCodeSystemsRemove(theSystem, terminologySet);
464                }
465        }
466
467        private void dropCircularRefs(
468                        TermConcept theConcept, ArrayList<String> theChain, Map<String, TermConcept> theCode2concept) {
469
470                theChain.add(theConcept.getCode());
471                for (Iterator<TermConceptParentChildLink> childIter =
472                                                theConcept.getChildren().iterator();
473                                childIter.hasNext(); ) {
474                        TermConceptParentChildLink next = childIter.next();
475                        TermConcept nextChild = next.getChild();
476                        if (theChain.contains(nextChild.getCode())) {
477
478                                StringBuilder b = new StringBuilder();
479                                b.append("Removing circular reference code ");
480                                b.append(nextChild.getCode());
481                                b.append(" from parent ");
482                                b.append(next.getParent().getCode());
483                                b.append(". Chain was: ");
484                                for (String nextInChain : theChain) {
485                                        TermConcept nextCode = theCode2concept.get(nextInChain);
486                                        b.append(nextCode.getCode());
487                                        b.append('[');
488                                        b.append(StringUtils.substring(nextCode.getDisplay(), 0, 20)
489                                                        .replace("[", "")
490                                                        .replace("]", "")
491                                                        .trim());
492                                        b.append("] ");
493                                }
494                                ourLog.info(b.toString(), theConcept.getCode());
495                                childIter.remove();
496                                nextChild.getParents().remove(next);
497
498                        } else {
499                                dropCircularRefs(nextChild, theChain, theCode2concept);
500                        }
501                }
502                theChain.remove(theChain.size() - 1);
503        }
504
505        @VisibleForTesting
506        @Nonnull
507        Properties getProperties(LoadedFileDescriptors theDescriptors, String thePropertiesFile) {
508                Properties retVal = new Properties();
509
510                try (InputStream propertyStream = ca.uhn.fhir.jpa.term.TermLoaderSvcImpl.class.getResourceAsStream(
511                                "/ca/uhn/fhir/jpa/term/loinc/loincupload.properties")) {
512                        retVal.load(propertyStream);
513                } catch (IOException e) {
514                        throw new InternalErrorException(Msg.code(866) + "Failed to process loinc.properties", e);
515                }
516
517                for (FileDescriptor next : theDescriptors.getUncompressedFileDescriptors()) {
518                        if (next.getFilename().endsWith(thePropertiesFile)) {
519                                try {
520                                        try (InputStream inputStream = next.getInputStream()) {
521                                                retVal.load(inputStream);
522                                        }
523                                } catch (IOException e) {
524                                        throw new InternalErrorException(Msg.code(867) + "Failed to read " + thePropertiesFile, e);
525                                }
526                        }
527                }
528                return retVal;
529        }
530
531        private Optional<String> loadFile(LoadedFileDescriptors theDescriptors, String... theFilenames) {
532                for (FileDescriptor next : theDescriptors.getUncompressedFileDescriptors()) {
533                        for (String nextFilename : theFilenames) {
534                                if (next.getFilename().endsWith(nextFilename)) {
535                                        try {
536                                                String contents = IOUtils.toString(next.getInputStream(), Charsets.UTF_8);
537                                                return Optional.of(contents);
538                                        } catch (IOException e) {
539                                                throw new InternalErrorException(Msg.code(868) + e);
540                                        }
541                                }
542                        }
543                }
544                return Optional.empty();
545        }
546
547        private UploadStatistics processImgthlaFiles(
548                        LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
549                final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
550                final List<ValueSet> valueSets = new ArrayList<>();
551                final List<ConceptMap> conceptMaps = new ArrayList<>();
552
553                CodeSystem imgthlaCs;
554                try {
555                        String imgthlaCsString = IOUtils.toString(
556                                        TermReadSvcImpl.class.getResourceAsStream("/ca/uhn/fhir/jpa/term/imgthla/imgthla.xml"),
557                                        Charsets.UTF_8);
558                        imgthlaCs = FhirContext.forR4Cached().newXmlParser().parseResource(CodeSystem.class, imgthlaCsString);
559                } catch (IOException e) {
560                        throw new InternalErrorException(Msg.code(869) + "Failed to load imgthla.xml", e);
561                }
562
563                boolean foundHlaNom = false;
564                boolean foundHlaXml = false;
565                for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) {
566                        String nextFilename = nextZipBytes.getFilename();
567
568                        if (!IMGTHLA_HLA_NOM_TXT.equals(nextFilename)
569                                        && !nextFilename.endsWith("/" + IMGTHLA_HLA_NOM_TXT)
570                                        && !IMGTHLA_HLA_XML.equals(nextFilename)
571                                        && !nextFilename.endsWith("/" + IMGTHLA_HLA_XML)) {
572                                ourLog.info("Skipping unexpected file {}", nextFilename);
573                                continue;
574                        }
575
576                        if (IMGTHLA_HLA_NOM_TXT.equals(nextFilename) || nextFilename.endsWith("/" + IMGTHLA_HLA_NOM_TXT)) {
577                                // process colon-delimited hla_nom.txt file
578                                ourLog.info("Processing file {}", nextFilename);
579
580                                //                              IRecordHandler handler = new HlaNomTxtHandler(codeSystemVersion, code2concept,
581                                // propertyNamesToTypes);
582                                //                              AntigenSource antigenSource = new WmdaAntigenSource(hlaNomFilename, relSerSerFilename,
583                                // relDnaSerFilename);
584
585                                Reader reader = null;
586                                try {
587                                        reader = new InputStreamReader(nextZipBytes.getInputStream(), Charsets.UTF_8);
588
589                                        LineNumberReader lnr = new LineNumberReader(reader);
590                                        while (lnr.readLine() != null) {}
591                                        ourLog.warn("Lines read from {}:  {}", nextFilename, lnr.getLineNumber());
592
593                                } catch (IOException e) {
594                                        throw new InternalErrorException(Msg.code(870) + e);
595                                } finally {
596                                        IOUtils.closeQuietly(reader);
597                                }
598
599                                foundHlaNom = true;
600                        }
601
602                        if (IMGTHLA_HLA_XML.equals(nextFilename) || nextFilename.endsWith("/" + IMGTHLA_HLA_XML)) {
603                                // process hla.xml file
604                                ourLog.info("Processing file {}", nextFilename);
605
606                                //                              IRecordHandler handler = new HlaXmlHandler(codeSystemVersion, code2concept, propertyNamesToTypes);
607                                //                              AlleleSource alleleSource = new HlaXmlAlleleSource(hlaXmlFilename);
608
609                                Reader reader = null;
610                                try {
611                                        reader = new InputStreamReader(nextZipBytes.getInputStream(), Charsets.UTF_8);
612
613                                        LineNumberReader lnr = new LineNumberReader(reader);
614                                        while (lnr.readLine() != null) {}
615                                        ourLog.warn("Lines read from {}:  {}", nextFilename, lnr.getLineNumber());
616
617                                } catch (IOException e) {
618                                        throw new InternalErrorException(Msg.code(871) + e);
619                                } finally {
620                                        IOUtils.closeQuietly(reader);
621                                }
622
623                                foundHlaXml = true;
624                        }
625                }
626
627                if (!foundHlaNom) {
628                        throw new InvalidRequestException(Msg.code(872) + "Did not find file matching " + IMGTHLA_HLA_NOM_TXT);
629                }
630
631                if (!foundHlaXml) {
632                        throw new InvalidRequestException(Msg.code(873) + "Did not find file matching " + IMGTHLA_HLA_XML);
633                }
634
635                int valueSetCount = valueSets.size();
636                int rootConceptCount = codeSystemVersion.getConcepts().size();
637                ourLog.info(
638                                "Have {} total concepts, {} root concepts, {} ValueSets",
639                                rootConceptCount,
640                                rootConceptCount,
641                                valueSetCount);
642
643                // remove this when fully implemented ...
644                throw new InternalErrorException(
645                                Msg.code(874) + "HLA nomenclature terminology upload not yet fully implemented.");
646
647                //              IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, imgthlaCs, valueSets, conceptMaps);
648                //
649                //              return new UploadStatistics(conceptCount, target);
650        }
651
652        UploadStatistics processLoincFiles(
653                        LoadedFileDescriptors theDescriptors,
654                        RequestDetails theRequestDetails,
655                        Properties theUploadProperties,
656                        Boolean theCloseFiles) {
657                final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
658                final Map<String, TermConcept> code2concept = new HashMap<>();
659                final List<ValueSet> valueSets = new ArrayList<>();
660                final List<ConceptMap> conceptMaps = new ArrayList<>();
661
662                final List<LoincLinguisticVariantsHandler.LinguisticVariant> linguisticVariants = new ArrayList<>();
663
664                LoincXmlFileZipContentsHandler loincXmlHandler = getLoincXmlFileZipContentsHandler();
665                iterateOverZipFile(theDescriptors, "loinc.xml", false, false, loincXmlHandler);
666                String loincCsString = loincXmlHandler.getContents();
667                if (isBlank(loincCsString)) {
668                        throw new InvalidRequestException(Msg.code(875) + "Did not find loinc.xml in the ZIP distribution.");
669                }
670
671                CodeSystem loincCs = FhirContext.forR4Cached().newXmlParser().parseResource(CodeSystem.class, loincCsString);
672                if (isNotBlank(loincCs.getVersion())) {
673                        throw new InvalidRequestException(
674                                        Msg.code(876) + "'loinc.xml' file must not have a version defined. To define a version use '"
675                                                        + LOINC_CODESYSTEM_VERSION.getCode() + "' property of 'loincupload.properties' file");
676                }
677
678                String codeSystemVersionId = theUploadProperties.getProperty(LOINC_CODESYSTEM_VERSION.getCode());
679                if (codeSystemVersionId != null) {
680                        loincCs.setVersion(codeSystemVersionId);
681                        loincCs.setId(loincCs.getId() + "-" + codeSystemVersionId);
682                }
683
684                Map<String, CodeSystem.PropertyType> propertyNamesToTypes = new HashMap<>();
685                for (CodeSystem.PropertyComponent nextProperty : loincCs.getProperty()) {
686                        String nextPropertyCode = nextProperty.getCode();
687                        CodeSystem.PropertyType nextPropertyType = nextProperty.getType();
688                        if (isNotBlank(nextPropertyCode)) {
689                                propertyNamesToTypes.put(nextPropertyCode, nextPropertyType);
690                        }
691                }
692
693                // TODO: DM 2019-09-13 - Manually add EXTERNAL_COPYRIGHT_NOTICE property until Regenstrief adds this to
694                // loinc.xml
695                if (!propertyNamesToTypes.containsKey("EXTERNAL_COPYRIGHT_NOTICE")) {
696                        String externalCopyRightNoticeCode = "EXTERNAL_COPYRIGHT_NOTICE";
697                        CodeSystem.PropertyType externalCopyRightNoticeType = CodeSystem.PropertyType.STRING;
698                        propertyNamesToTypes.put(externalCopyRightNoticeCode, externalCopyRightNoticeType);
699                }
700
701                IZipContentsHandlerCsv handler;
702
703                // Part
704                handler = new LoincPartHandler(codeSystemVersion, code2concept);
705                iterateOverZipFileCsv(
706                                theDescriptors,
707                                theUploadProperties.getProperty(LOINC_PART_FILE.getCode(), LOINC_PART_FILE_DEFAULT.getCode()),
708                                handler,
709                                ',',
710                                QuoteMode.NON_NUMERIC,
711                                false);
712                Map<PartTypeAndPartName, String> partTypeAndPartNameToPartNumber =
713                                ((LoincPartHandler) handler).getPartTypeAndPartNameToPartNumber();
714
715                // LOINC string properties
716                handler = new LoincHandler(
717                                codeSystemVersion, code2concept, propertyNamesToTypes, partTypeAndPartNameToPartNumber);
718                iterateOverZipFileCsv(
719                                theDescriptors,
720                                theUploadProperties.getProperty(LOINC_FILE.getCode(), LOINC_FILE_DEFAULT.getCode()),
721                                handler,
722                                ',',
723                                QuoteMode.NON_NUMERIC,
724                                false);
725
726                // LOINC hierarchy
727                handler = new LoincHierarchyHandler(codeSystemVersion, code2concept);
728                iterateOverZipFileCsv(
729                                theDescriptors,
730                                theUploadProperties.getProperty(LOINC_HIERARCHY_FILE.getCode(), LOINC_HIERARCHY_FILE_DEFAULT.getCode()),
731                                handler,
732                                ',',
733                                QuoteMode.NON_NUMERIC,
734                                false);
735
736                // Answer lists (ValueSets of potential answers/values for LOINC "questions")
737                handler = new LoincAnswerListHandler(
738                                codeSystemVersion, code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
739                iterateOverZipFileCsv(
740                                theDescriptors,
741                                theUploadProperties.getProperty(
742                                                LOINC_ANSWERLIST_FILE.getCode(), LOINC_ANSWERLIST_FILE_DEFAULT.getCode()),
743                                handler,
744                                ',',
745                                QuoteMode.NON_NUMERIC,
746                                false);
747
748                // Answer list links (connects LOINC observation codes to answer list codes)
749                handler = new LoincAnswerListLinkHandler(code2concept);
750                iterateOverZipFileCsv(
751                                theDescriptors,
752                                theUploadProperties.getProperty(
753                                                LOINC_ANSWERLIST_LINK_FILE.getCode(), LOINC_ANSWERLIST_LINK_FILE_DEFAULT.getCode()),
754                                handler,
755                                ',',
756                                QuoteMode.NON_NUMERIC,
757                                false);
758
759                // RSNA playbook
760                // Note that this should come before the "Part Related Code Mapping"
761                // file because there are some duplicate mappings between these
762                // two files, and the RSNA Playbook file has more metadata
763                handler = new LoincRsnaPlaybookHandler(
764                                code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
765                iterateOverZipFileCsv(
766                                theDescriptors,
767                                theUploadProperties.getProperty(
768                                                LOINC_RSNA_PLAYBOOK_FILE.getCode(), LOINC_RSNA_PLAYBOOK_FILE_DEFAULT.getCode()),
769                                handler,
770                                ',',
771                                QuoteMode.NON_NUMERIC,
772                                false);
773
774                // Part related code mapping
775                handler = new LoincPartRelatedCodeMappingHandler(
776                                code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
777                iterateOverZipFileCsv(
778                                theDescriptors,
779                                theUploadProperties.getProperty(
780                                                LOINC_PART_RELATED_CODE_MAPPING_FILE.getCode(),
781                                                LOINC_PART_RELATED_CODE_MAPPING_FILE_DEFAULT.getCode()),
782                                handler,
783                                ',',
784                                QuoteMode.NON_NUMERIC,
785                                false);
786
787                // Document ontology
788                handler = new LoincDocumentOntologyHandler(
789                                code2concept,
790                                propertyNamesToTypes,
791                                valueSets,
792                                conceptMaps,
793                                theUploadProperties,
794                                loincCs.getCopyright());
795                iterateOverZipFileCsv(
796                                theDescriptors,
797                                theUploadProperties.getProperty(
798                                                LOINC_DOCUMENT_ONTOLOGY_FILE.getCode(), LOINC_DOCUMENT_ONTOLOGY_FILE_DEFAULT.getCode()),
799                                handler,
800                                ',',
801                                QuoteMode.NON_NUMERIC,
802                                false);
803
804                // Top 2000 codes - US
805                handler = new LoincTop2000LabResultsUsHandler(
806                                code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
807                iterateOverZipFileCsvOptional(
808                                theDescriptors,
809                                theUploadProperties.getProperty(
810                                                LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE.getCode(),
811                                                LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE_DEFAULT.getCode()),
812                                handler,
813                                ',',
814                                QuoteMode.NON_NUMERIC,
815                                false);
816
817                // Top 2000 codes - SI
818                handler = new LoincTop2000LabResultsSiHandler(
819                                code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
820                iterateOverZipFileCsvOptional(
821                                theDescriptors,
822                                theUploadProperties.getProperty(
823                                                LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE.getCode(),
824                                                LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE_DEFAULT.getCode()),
825                                handler,
826                                ',',
827                                QuoteMode.NON_NUMERIC,
828                                false);
829
830                // Universal lab order ValueSet
831                handler = new LoincUniversalOrderSetHandler(code2concept, valueSets, conceptMaps, theUploadProperties);
832                iterateOverZipFileCsv(
833                                theDescriptors,
834                                theUploadProperties.getProperty(
835                                                LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE.getCode(),
836                                                LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE_DEFAULT.getCode()),
837                                handler,
838                                ',',
839                                QuoteMode.NON_NUMERIC,
840                                false);
841
842                // IEEE medical device codes
843                handler = new LoincIeeeMedicalDeviceCodeHandler(
844                                code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
845                iterateOverZipFileCsv(
846                                theDescriptors,
847                                theUploadProperties.getProperty(
848                                                LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE.getCode(),
849                                                LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_FILE_DEFAULT.getCode()),
850                                handler,
851                                ',',
852                                QuoteMode.NON_NUMERIC,
853                                false);
854
855                // Imaging document codes
856                handler = new LoincImagingDocumentCodeHandler(code2concept, valueSets, conceptMaps, theUploadProperties);
857                iterateOverZipFileCsv(
858                                theDescriptors,
859                                theUploadProperties.getProperty(
860                                                LOINC_IMAGING_DOCUMENT_CODES_FILE.getCode(),
861                                                LOINC_IMAGING_DOCUMENT_CODES_FILE_DEFAULT.getCode()),
862                                handler,
863                                ',',
864                                QuoteMode.NON_NUMERIC,
865                                false);
866
867                // Group
868                handler = new LoincGroupFileHandler(
869                                code2concept, valueSets, conceptMaps, theUploadProperties, loincCs.getCopyright());
870                iterateOverZipFileCsv(
871                                theDescriptors,
872                                theUploadProperties.getProperty(LOINC_GROUP_FILE.getCode(), LOINC_GROUP_FILE_DEFAULT.getCode()),
873                                handler,
874                                ',',
875                                QuoteMode.NON_NUMERIC,
876                                false);
877
878                // Group terms
879                handler = new LoincGroupTermsFileHandler(code2concept, valueSets, conceptMaps, theUploadProperties);
880                iterateOverZipFileCsv(
881                                theDescriptors,
882                                theUploadProperties.getProperty(
883                                                LOINC_GROUP_TERMS_FILE.getCode(), LOINC_GROUP_TERMS_FILE_DEFAULT.getCode()),
884                                handler,
885                                ',',
886                                QuoteMode.NON_NUMERIC,
887                                false);
888
889                // Parent group
890                handler = new LoincParentGroupFileHandler(code2concept, valueSets, conceptMaps, theUploadProperties);
891                iterateOverZipFileCsv(
892                                theDescriptors,
893                                theUploadProperties.getProperty(
894                                                LOINC_PARENT_GROUP_FILE.getCode(), LOINC_PARENT_GROUP_FILE_DEFAULT.getCode()),
895                                handler,
896                                ',',
897                                QuoteMode.NON_NUMERIC,
898                                false);
899
900                // Part link
901                handler = new LoincPartLinkHandler(codeSystemVersion, code2concept, propertyNamesToTypes);
902                iterateOverZipFileCsvOptional(
903                                theDescriptors,
904                                theUploadProperties.getProperty(LOINC_PART_LINK_FILE.getCode(), LOINC_PART_LINK_FILE_DEFAULT.getCode()),
905                                handler,
906                                ',',
907                                QuoteMode.NON_NUMERIC,
908                                false);
909                iterateOverZipFileCsvOptional(
910                                theDescriptors,
911                                theUploadProperties.getProperty(
912                                                LOINC_PART_LINK_FILE_PRIMARY.getCode(), LOINC_PART_LINK_FILE_PRIMARY_DEFAULT.getCode()),
913                                handler,
914                                ',',
915                                QuoteMode.NON_NUMERIC,
916                                false);
917                iterateOverZipFileCsvOptional(
918                                theDescriptors,
919                                theUploadProperties.getProperty(
920                                                LOINC_PART_LINK_FILE_SUPPLEMENTARY.getCode(),
921                                                LOINC_PART_LINK_FILE_SUPPLEMENTARY_DEFAULT.getCode()),
922                                handler,
923                                ',',
924                                QuoteMode.NON_NUMERIC,
925                                false);
926
927                // Consumer Name
928                handler = new LoincConsumerNameHandler(code2concept);
929                iterateOverZipFileCsvOptional(
930                                theDescriptors,
931                                theUploadProperties.getProperty(
932                                                LOINC_CONSUMER_NAME_FILE.getCode(), LOINC_CONSUMER_NAME_FILE_DEFAULT.getCode()),
933                                handler,
934                                ',',
935                                QuoteMode.NON_NUMERIC,
936                                false);
937
938                // LOINC coding properties (must run after all TermConcepts were created)
939                handler = new LoincCodingPropertiesHandler(code2concept, propertyNamesToTypes);
940                iterateOverZipFileCsv(
941                                theDescriptors,
942                                theUploadProperties.getProperty(LOINC_FILE.getCode(), LOINC_FILE_DEFAULT.getCode()),
943                                handler,
944                                ',',
945                                QuoteMode.NON_NUMERIC,
946                                false);
947
948                // Linguistic Variants
949                handler = new LoincLinguisticVariantsHandler(linguisticVariants);
950                iterateOverZipFileCsvOptional(
951                                theDescriptors,
952                                theUploadProperties.getProperty(
953                                                LOINC_LINGUISTIC_VARIANTS_FILE.getCode(), LOINC_LINGUISTIC_VARIANTS_FILE_DEFAULT.getCode()),
954                                handler,
955                                ',',
956                                QuoteMode.NON_NUMERIC,
957                                false);
958
959                String langFileName;
960                for (LoincLinguisticVariantsHandler.LinguisticVariant linguisticVariant : linguisticVariants) {
961                        handler = new LoincLinguisticVariantHandler(code2concept, linguisticVariant.getLanguageCode());
962                        langFileName = linguisticVariant.getLinguisticVariantFileName();
963                        iterateOverZipFileCsvOptional(
964                                        theDescriptors,
965                                        theUploadProperties.getProperty(
966                                                        LOINC_LINGUISTIC_VARIANTS_PATH.getCode() + langFileName,
967                                                        LOINC_LINGUISTIC_VARIANTS_PATH_DEFAULT.getCode() + langFileName),
968                                        handler,
969                                        ',',
970                                        QuoteMode.NON_NUMERIC,
971                                        false);
972                }
973
974                if (theDescriptors.isOptionalFilesExist(List.of(
975                                theUploadProperties.getProperty(LOINC_MAPTO_FILE.getCode(), LOINC_MAPTO_FILE_DEFAULT.getCode())))) {
976                        // LOINC MapTo codes (last to make sure that all concepts were added to code2concept map)
977                        handler = new LoincMapToHandler(code2concept);
978                        iterateOverZipFileCsv(
979                                        theDescriptors,
980                                        theUploadProperties.getProperty(LOINC_MAPTO_FILE.getCode(), LOINC_MAPTO_FILE_DEFAULT.getCode()),
981                                        handler,
982                                        ',',
983                                        QuoteMode.NON_NUMERIC,
984                                        false);
985                }
986
987                if (theCloseFiles) {
988                        IOUtils.closeQuietly(theDescriptors);
989                }
990
991                valueSets.add(getValueSetLoincAll(theUploadProperties, loincCs.getCopyright()));
992
993                for (Entry<String, TermConcept> next : code2concept.entrySet()) {
994                        TermConcept nextConcept = next.getValue();
995                        if (nextConcept.getParents().isEmpty()) {
996                                codeSystemVersion.getConcepts().add(nextConcept);
997                        }
998                }
999
1000                int valueSetCount = valueSets.size();
1001                int rootConceptCount = codeSystemVersion.getConcepts().size();
1002                int conceptCount = code2concept.size();
1003                ourLog.info(
1004                                "Have {} total concepts, {} root concepts, {} ValueSets",
1005                                conceptCount,
1006                                rootConceptCount,
1007                                valueSetCount);
1008
1009                IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, loincCs, valueSets, conceptMaps);
1010
1011                return new UploadStatistics(conceptCount, target);
1012        }
1013
1014        @VisibleForTesting
1015        protected LoincXmlFileZipContentsHandler getLoincXmlFileZipContentsHandler() {
1016                return new LoincXmlFileZipContentsHandler();
1017        }
1018
1019        private ValueSet getValueSetLoincAll(Properties theUploadProperties, String theCopyrightStatement) {
1020                ValueSet retVal = new ValueSet();
1021
1022                String codeSystemVersionId = theUploadProperties.getProperty(LOINC_CODESYSTEM_VERSION.getCode());
1023                String valueSetId;
1024                if (codeSystemVersionId != null) {
1025                        valueSetId = LOINC_ALL_VALUESET_ID + "-" + codeSystemVersionId;
1026                } else {
1027                        valueSetId = LOINC_ALL_VALUESET_ID;
1028                }
1029                retVal.setId(valueSetId);
1030                retVal.setUrl(LOINC_GENERIC_VALUESET_URL);
1031                retVal.setVersion(codeSystemVersionId);
1032                retVal.setName("All LOINC codes");
1033                retVal.setStatus(Enumerations.PublicationStatus.ACTIVE);
1034                retVal.setDate(new Date());
1035                retVal.setPublisher("Regenstrief Institute, Inc.");
1036                retVal.setDescription("A value set that includes all LOINC codes");
1037                retVal.setCopyright(theCopyrightStatement);
1038                retVal.getCompose().addInclude().setSystem(ITermLoaderSvc.LOINC_URI).setVersion(codeSystemVersionId);
1039
1040                return retVal;
1041        }
1042
1043        private UploadStatistics processSnomedCtFiles(
1044                        LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
1045                final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
1046                final Map<String, TermConcept> id2concept = new HashMap<>();
1047                final Map<String, TermConcept> code2concept = new HashMap<>();
1048                final Set<String> validConceptIds = new HashSet<>();
1049
1050                IZipContentsHandlerCsv handler = new SctHandlerConcept(validConceptIds);
1051                iterateOverZipFileCsv(theDescriptors, SCT_FILE_CONCEPT, handler, '\t', null, true);
1052
1053                ourLog.info("Have {} valid concept IDs", validConceptIds.size());
1054
1055                handler = new SctHandlerDescription(validConceptIds, code2concept, id2concept, codeSystemVersion);
1056                iterateOverZipFileCsv(theDescriptors, SCT_FILE_DESCRIPTION, handler, '\t', null, true);
1057
1058                ourLog.info("Got {} concepts, cloning map", code2concept.size());
1059                final HashMap<String, TermConcept> rootConcepts = new HashMap<>(code2concept);
1060
1061                handler = new SctHandlerRelationship(codeSystemVersion, rootConcepts, code2concept);
1062                iterateOverZipFileCsv(theDescriptors, SCT_FILE_RELATIONSHIP, handler, '\t', null, true);
1063
1064                IOUtils.closeQuietly(theDescriptors);
1065
1066                ourLog.info("Looking for root codes");
1067                rootConcepts
1068                                .entrySet()
1069                                .removeIf(theStringTermConceptEntry ->
1070                                                !theStringTermConceptEntry.getValue().getParents().isEmpty());
1071
1072                ourLog.info(
1073                                "Done loading SNOMED CT files - {} root codes, {} total codes",
1074                                rootConcepts.size(),
1075                                code2concept.size());
1076
1077                Counter circularCounter = new Counter();
1078                for (TermConcept next : rootConcepts.values()) {
1079                        long count = circularCounter.getThenAdd();
1080                        float pct = ((float) count / rootConcepts.size()) * 100.0f;
1081                        ourLog.info(
1082                                        " * Scanning for circular refs - have scanned {} / {} codes ({}%)",
1083                                        count, rootConcepts.size(), pct);
1084                        dropCircularRefs(next, new ArrayList<>(), code2concept);
1085                }
1086
1087                codeSystemVersion.getConcepts().addAll(rootConcepts.values());
1088
1089                CodeSystem cs = new org.hl7.fhir.r4.model.CodeSystem();
1090                cs.setUrl(SCT_URI);
1091                cs.setName("SNOMED CT");
1092                cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
1093                cs.setStatus(Enumerations.PublicationStatus.ACTIVE);
1094                IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
1095
1096                return new UploadStatistics(code2concept.size(), target);
1097        }
1098
1099        private IIdType storeCodeSystem(
1100                        RequestDetails theRequestDetails,
1101                        final TermCodeSystemVersion theCodeSystemVersion,
1102                        CodeSystem theCodeSystem,
1103                        List<ValueSet> theValueSets,
1104                        List<ConceptMap> theConceptMaps) {
1105                Validate.isTrue(theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT);
1106
1107                List<ValueSet> valueSets = ObjectUtils.defaultIfNull(theValueSets, Collections.emptyList());
1108                List<ConceptMap> conceptMaps = ObjectUtils.defaultIfNull(theConceptMaps, Collections.emptyList());
1109
1110                IIdType retVal;
1111                myDeferredStorageSvc.setProcessDeferred(false);
1112                retVal = myCodeSystemStorageSvc.storeNewCodeSystemVersion(
1113                                theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
1114                myDeferredStorageSvc.setProcessDeferred(true);
1115
1116                return retVal;
1117        }
1118
1119        public static void iterateOverZipFileCsv(
1120                        LoadedFileDescriptors theDescriptors,
1121                        String theFileNamePart,
1122                        IZipContentsHandlerCsv theHandler,
1123                        char theDelimiter,
1124                        QuoteMode theQuoteMode,
1125                        boolean theIsPartialFilename) {
1126                iterateOverZipFileCsv(
1127                                theDescriptors, theFileNamePart, theHandler, theDelimiter, theQuoteMode, theIsPartialFilename, true);
1128        }
1129
1130        public static void iterateOverZipFileCsvOptional(
1131                        LoadedFileDescriptors theDescriptors,
1132                        String theFileNamePart,
1133                        IZipContentsHandlerCsv theHandler,
1134                        char theDelimiter,
1135                        QuoteMode theQuoteMode,
1136                        boolean theIsPartialFilename) {
1137                iterateOverZipFileCsv(
1138                                theDescriptors, theFileNamePart, theHandler, theDelimiter, theQuoteMode, theIsPartialFilename, false);
1139        }
1140
1141        private static void iterateOverZipFileCsv(
1142                        LoadedFileDescriptors theDescriptors,
1143                        String theFileNamePart,
1144                        IZipContentsHandlerCsv theHandler,
1145                        char theDelimiter,
1146                        QuoteMode theQuoteMode,
1147                        boolean theIsPartialFilename,
1148                        boolean theRequireMatch) {
1149                IZipContentsHandler handler = (reader, filename) -> {
1150                        CSVParser parsed = newCsvRecords(theDelimiter, theQuoteMode, reader);
1151                        Iterator<CSVRecord> iter = parsed.iterator();
1152                        ourLog.debug("Header map: {}", parsed.getHeaderMap());
1153
1154                        int count = 0;
1155                        int nextLoggedCount = 0;
1156                        while (iter.hasNext()) {
1157                                CSVRecord nextRecord = iter.next();
1158                                if (!nextRecord.isConsistent()) {
1159                                        continue;
1160                                }
1161                                theHandler.accept(nextRecord);
1162                                count++;
1163                                if (count >= nextLoggedCount) {
1164                                        ourLog.info(" * Processed {} records in {}", count, filename);
1165                                        nextLoggedCount += LOG_INCREMENT;
1166                                }
1167                        }
1168                };
1169
1170                iterateOverZipFile(theDescriptors, theFileNamePart, theIsPartialFilename, theRequireMatch, handler);
1171        }
1172
1173        private static void iterateOverZipFile(
1174                        LoadedFileDescriptors theDescriptors,
1175                        String theFileNamePart,
1176                        boolean theIsPartialFilename,
1177                        boolean theRequireMatch,
1178                        IZipContentsHandler theHandler) {
1179                boolean foundMatch = false;
1180                for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) {
1181                        String nextFilename = nextZipBytes.getFilename();
1182                        boolean matches;
1183                        if (theIsPartialFilename) {
1184                                matches = nextFilename.contains(theFileNamePart);
1185                        } else {
1186                                matches = nextFilename.endsWith("/" + theFileNamePart) || nextFilename.equals(theFileNamePart);
1187                        }
1188
1189                        if (matches) {
1190                                ourLog.info("Processing file {}", nextFilename);
1191                                foundMatch = true;
1192
1193                                try {
1194
1195                                        Reader reader = new InputStreamReader(nextZipBytes.getInputStream(), Charsets.UTF_8);
1196                                        theHandler.handle(reader, nextFilename);
1197
1198                                } catch (IOException e) {
1199                                        throw new InternalErrorException(Msg.code(877) + e);
1200                                }
1201                        }
1202                }
1203
1204                if (!foundMatch && theRequireMatch) {
1205                        throw new InvalidRequestException(Msg.code(878) + "Did not find file matching " + theFileNamePart);
1206                }
1207        }
1208
1209        @Nonnull
1210        private static CSVParser newCsvRecords(char theDelimiter, QuoteMode theQuoteMode, Reader theReader)
1211                        throws IOException {
1212                CSVParser parsed;
1213                CSVFormat format =
1214                                CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader().withTrim();
1215                if (theQuoteMode != null) {
1216                        format = format.withQuote('"').withQuoteMode(theQuoteMode);
1217                }
1218                parsed = new CSVParser(theReader, format);
1219                return parsed;
1220        }
1221
1222        public static String firstNonBlank(String... theStrings) {
1223                String retVal = "";
1224                for (String nextString : theStrings) {
1225                        if (isNotBlank(nextString)) {
1226                                retVal = nextString;
1227                                break;
1228                        }
1229                }
1230                return retVal;
1231        }
1232
1233        public static TermConcept getOrCreateConcept(Map<String, TermConcept> id2concept, String id) {
1234                TermConcept concept = id2concept.get(id);
1235                if (concept == null) {
1236                        concept = new TermConcept();
1237                        id2concept.put(id, concept);
1238                }
1239                return concept;
1240        }
1241
1242        public static TermConceptProperty getOrCreateConceptProperty(
1243                        Map<String, List<TermConceptProperty>> code2Properties, String code, String key) {
1244                List<TermConceptProperty> termConceptProperties = code2Properties.get(code);
1245                if (termConceptProperties == null) return new TermConceptProperty();
1246                Optional<TermConceptProperty> termConceptProperty = termConceptProperties.stream()
1247                                .filter(property -> key.equals(property.getKey()))
1248                                .findFirst();
1249                return termConceptProperty.orElseGet(TermConceptProperty::new);
1250        }
1251}