View Javadoc
1   package ca.uhn.fhir.jpa.term;
2   
3   import ca.uhn.fhir.context.FhirContext;
4   import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
5   import ca.uhn.fhir.jpa.entity.TermConcept;
6   import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
7   import ca.uhn.fhir.jpa.term.loinc.*;
8   import ca.uhn.fhir.jpa.term.snomedct.SctHandlerConcept;
9   import ca.uhn.fhir.jpa.term.snomedct.SctHandlerDescription;
10  import ca.uhn.fhir.jpa.term.snomedct.SctHandlerRelationship;
11  import ca.uhn.fhir.jpa.util.Counter;
12  import ca.uhn.fhir.rest.api.server.RequestDetails;
13  import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
14  import ca.uhn.fhir.rest.server.exceptions.UnprocessableEntityException;
15  import com.google.common.annotations.VisibleForTesting;
16  import com.google.common.base.Charsets;
17  import org.apache.commons.csv.CSVFormat;
18  import org.apache.commons.csv.CSVParser;
19  import org.apache.commons.csv.CSVRecord;
20  import org.apache.commons.csv.QuoteMode;
21  import org.apache.commons.io.FileUtils;
22  import org.apache.commons.io.IOUtils;
23  import org.apache.commons.io.input.BOMInputStream;
24  import org.apache.commons.lang3.ObjectUtils;
25  import org.apache.commons.lang3.StringUtils;
26  import org.apache.commons.lang3.Validate;
27  import org.hl7.fhir.instance.model.api.IIdType;
28  import org.hl7.fhir.r4.model.CodeSystem;
29  import org.hl7.fhir.r4.model.ConceptMap;
30  import org.hl7.fhir.r4.model.ValueSet;
31  import org.springframework.beans.factory.annotation.Autowired;
32  
33  import java.io.*;
34  import java.util.*;
35  import java.util.Map.Entry;
36  import java.util.zip.ZipEntry;
37  import java.util.zip.ZipInputStream;
38  
39  import static org.apache.commons.lang3.StringUtils.isNotBlank;
40  
41  /*
42   * #%L
43   * HAPI FHIR JPA Server
44   * %%
45   * Copyright (C) 2014 - 2018 University Health Network
46   * %%
47   * Licensed under the Apache License, Version 2.0 (the "License");
48   * you may not use this file except in compliance with the License.
49   * You may obtain a copy of the License at
50   * 
51   *      http://www.apache.org/licenses/LICENSE-2.0
52   * 
53   * Unless required by applicable law or agreed to in writing, software
54   * distributed under the License is distributed on an "AS IS" BASIS,
55   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
56   * See the License for the specific language governing permissions and
57   * limitations under the License.
58   * #L%
59   */
60  
61  public class TerminologyLoaderSvcImpl implements IHapiTerminologyLoaderSvc {
62  	public static final String SCT_FILE_CONCEPT = "Terminology/sct2_Concept_Full_";
63  	public static final String SCT_FILE_DESCRIPTION = "Terminology/sct2_Description_Full-en";
64  	public static final String SCT_FILE_RELATIONSHIP = "Terminology/sct2_Relationship_Full";
65  	public static final String LOINC_ANSWERLIST_FILE = "AnswerList_Beta_1.csv";
66  	public static final String LOINC_ANSWERLIST_LINK_FILE = "LoincAnswerListLink_Beta_1.csv";
67  	public static final String LOINC_DOCUMENT_ONTOLOGY_FILE = "DocumentOntology.csv";
68  	public static final String LOINC_UPLOAD_PROPERTIES_FILE = "loincupload.properties";
69  	public static final String LOINC_FILE = "loinc.csv";
70  	public static final String LOINC_HIERARCHY_FILE = "MULTI-AXIAL_HIERARCHY.CSV";
71  	public static final String LOINC_PART_FILE = "Part_Beta_1.csv";
72  	public static final String LOINC_PART_LINK_FILE = "LoincPartLink_Beta_1.csv";
73  	public static final String LOINC_PART_RELATED_CODE_MAPPING_FILE = "PartRelatedCodeMapping_Beta_1.csv";
74  	public static final String LOINC_RSNA_PLAYBOOK_FILE = "LoincRsnaRadiologyPlaybook.csv";
75  	public static final String LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE = "Top2000CommonLabResultsUS.csv";
76  	public static final String LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE = "Top2000CommonLabResultsSI.csv";
77  	public static final String LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE = "LoincUniversalLabOrdersValueSet.csv";
78  	public static final String LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV = "LoincIeeeMedicalDeviceCodeMappingTable.csv";
79  	public static final String LOINC_IMAGING_DOCUMENT_CODES_FILE = "ImagingDocumentCodes.csv";
80  	private static final int LOG_INCREMENT = 100000;
81  	private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(TerminologyLoaderSvcImpl.class);
82  	@Autowired
83  	private IHapiTerminologySvc myTermSvc;
84  	@Autowired(required = false)
85  	private IHapiTerminologySvcDstu3 myTermSvcDstu3;
86  	@Autowired(required = false)
87  	private IHapiTerminologySvcR4 myTermSvcR4;
88  
89  	private void dropCircularRefs(TermConcept theConcept, ArrayList<String> theChain, Map<String, TermConcept> theCode2concept, Counter theCircularCounter) {
90  
91  		theChain.add(theConcept.getCode());
92  		for (Iterator<TermConceptParentChildLink> childIter = theConcept.getChildren().iterator(); childIter.hasNext(); ) {
93  			TermConceptParentChildLink next = childIter.next();
94  			TermConcept nextChild = next.getChild();
95  			if (theChain.contains(nextChild.getCode())) {
96  
97  				StringBuilder b = new StringBuilder();
98  				b.append("Removing circular reference code ");
99  				b.append(nextChild.getCode());
100 				b.append(" from parent ");
101 				b.append(next.getParent().getCode());
102 				b.append(". Chain was: ");
103 				for (String nextInChain : theChain) {
104 					TermConcept nextCode = theCode2concept.get(nextInChain);
105 					b.append(nextCode.getCode());
106 					b.append('[');
107 					b.append(StringUtils.substring(nextCode.getDisplay(), 0, 20).replace("[", "").replace("]", "").trim());
108 					b.append("] ");
109 				}
110 				ourLog.info(b.toString(), theConcept.getCode());
111 				childIter.remove();
112 				nextChild.getParents().remove(next);
113 
114 			} else {
115 				dropCircularRefs(nextChild, theChain, theCode2concept, theCircularCounter);
116 			}
117 		}
118 		theChain.remove(theChain.size() - 1);
119 
120 	}
121 
122 	private void iterateOverZipFile(LoadedFileDescriptors theDescriptors, String theFileNamePart, IRecordHandler theHandler, char theDelimiter, QuoteMode theQuoteMode) {
123 
124 		for (FileDescriptor nextZipBytes : theDescriptors.getUncompressedFileDescriptors()) {
125 			String nextFilename = nextZipBytes.getFilename();
126 			if (nextFilename.contains(theFileNamePart)) {
127 				ourLog.info("Processing file {}", nextFilename);
128 
129 				Reader reader;
130 				CSVParser parsed;
131 				try {
132 					reader = new InputStreamReader(nextZipBytes.getInputStream(), Charsets.UTF_8);
133 
134 					if (ourLog.isTraceEnabled()) {
135 						String contents = IOUtils.toString(reader);
136 						ourLog.info("File contents for: {}\n{}", nextFilename, contents);
137 						reader = new StringReader(contents);
138 					}
139 
140 					CSVFormat format = CSVFormat.newFormat(theDelimiter).withFirstRecordAsHeader();
141 					if (theQuoteMode != null) {
142 						format = format.withQuote('"').withQuoteMode(theQuoteMode);
143 					}
144 					parsed = new CSVParser(reader, format);
145 					Iterator<CSVRecord> iter = parsed.iterator();
146 					ourLog.debug("Header map: {}", parsed.getHeaderMap());
147 
148 					int count = 0;
149 					int nextLoggedCount = 0;
150 					while (iter.hasNext()) {
151 						CSVRecord nextRecord = iter.next();
152 						theHandler.accept(nextRecord);
153 						count++;
154 						if (count >= nextLoggedCount) {
155 							ourLog.info(" * Processed {} records in {}", count, nextFilename);
156 							nextLoggedCount += LOG_INCREMENT;
157 						}
158 					}
159 
160 				} catch (IOException e) {
161 					throw new InternalErrorException(e);
162 				}
163 			}
164 
165 		}
166 
167 	}
168 
169 	@Override
170 	public UploadStatistics loadLoinc(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
171 		LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles);
172 		List<String> mandatoryFilenameFragments = Arrays.asList(
173 			LOINC_FILE,
174 			LOINC_HIERARCHY_FILE);
175 		descriptors.verifyMandatoryFilesExist(mandatoryFilenameFragments);
176 
177 		List<String> optionalFilenameFragments = Arrays.asList(
178 			LOINC_UPLOAD_PROPERTIES_FILE,
179 			LOINC_ANSWERLIST_FILE,
180 			LOINC_ANSWERLIST_LINK_FILE,
181 			LOINC_PART_FILE,
182 			LOINC_PART_LINK_FILE,
183 			LOINC_PART_RELATED_CODE_MAPPING_FILE,
184 			LOINC_DOCUMENT_ONTOLOGY_FILE,
185 			LOINC_RSNA_PLAYBOOK_FILE,
186 			LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE,
187 			LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE,
188 			LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE,
189 			LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV,
190 			LOINC_IMAGING_DOCUMENT_CODES_FILE
191 		);
192 		descriptors.verifyOptionalFilesExist(optionalFilenameFragments);
193 
194 		ourLog.info("Beginning LOINC processing");
195 
196 		return processLoincFiles(descriptors, theRequestDetails);
197 	}
198 
199 	@Override
200 	public UploadStatistics loadSnomedCt(List<FileDescriptor> theFiles, RequestDetails theRequestDetails) {
201 		LoadedFileDescriptors descriptors = new LoadedFileDescriptors(theFiles);
202 
203 		List<String> expectedFilenameFragments = Arrays.asList(
204 			SCT_FILE_DESCRIPTION,
205 			SCT_FILE_RELATIONSHIP,
206 			SCT_FILE_CONCEPT);
207 		descriptors.verifyMandatoryFilesExist(expectedFilenameFragments);
208 
209 		ourLog.info("Beginning SNOMED CT processing");
210 
211 		return processSnomedCtFiles(descriptors, theRequestDetails);
212 	}
213 
214 	UploadStatistics processLoincFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
215 		final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
216 		final Map<String, TermConcept> code2concept = new HashMap<>();
217 		final List<ValueSet> valueSets = new ArrayList<>();
218 		final List<ConceptMap> conceptMaps = new ArrayList<>();
219 
220 		CodeSystem loincCs;
221 		try {
222 			String loincCsString = IOUtils.toString(BaseHapiTerminologySvcImpl.class.getResourceAsStream("/ca/uhn/fhir/jpa/term/loinc/loinc.xml"), Charsets.UTF_8);
223 			loincCs = FhirContext.forR4().newXmlParser().parseResource(CodeSystem.class, loincCsString);
224 		} catch (IOException e) {
225 			throw new InternalErrorException("Failed to load loinc.xml", e);
226 		}
227 
228 		Map<String, CodeSystem.PropertyType> propertyNamesToTypes = new HashMap<>();
229 		for (CodeSystem.PropertyComponent nextProperty : loincCs.getProperty()) {
230 			String nextPropertyCode = nextProperty.getCode();
231 			CodeSystem.PropertyType nextPropertyType = nextProperty.getType();
232 			if (isNotBlank(nextPropertyCode)) {
233 				propertyNamesToTypes.put(nextPropertyCode, nextPropertyType);
234 			}
235 		}
236 
237 		IRecordHandler handler;
238 
239 		Properties uploadProperties = new Properties();
240 		for (FileDescriptor next : theDescriptors.getUncompressedFileDescriptors()) {
241 			if (next.getFilename().endsWith("loincupload.properties")) {
242 				try {
243 					try (InputStream inputStream = next.getInputStream()) {
244 						uploadProperties.load(inputStream);
245 					}
246 				} catch (IOException e) {
247 					throw new InternalErrorException("Failed to read loincupload.properties", e);
248 				}
249 			}
250 		}
251 
252 		// Part file
253 		handler = new LoincPartHandler(codeSystemVersion, code2concept);
254 		iterateOverZipFile(theDescriptors, LOINC_PART_FILE, handler, ',', QuoteMode.NON_NUMERIC);
255 		Map<PartTypeAndPartName, String> partTypeAndPartNameToPartNumber = ((LoincPartHandler) handler).getPartTypeAndPartNameToPartNumber();
256 
257 		// Loinc Codes
258 		handler = new LoincHandler(codeSystemVersion, code2concept, propertyNamesToTypes, partTypeAndPartNameToPartNumber);
259 		iterateOverZipFile(theDescriptors, LOINC_FILE, handler, ',', QuoteMode.NON_NUMERIC);
260 
261 		// Loinc Hierarchy
262 		handler = new LoincHierarchyHandler(codeSystemVersion, code2concept);
263 		iterateOverZipFile(theDescriptors, LOINC_HIERARCHY_FILE, handler, ',', QuoteMode.NON_NUMERIC);
264 
265 		// Answer lists (ValueSets of potential answers/values for loinc "questions")
266 		handler = new LoincAnswerListHandler(codeSystemVersion, code2concept, valueSets, conceptMaps, uploadProperties);
267 		iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_FILE, handler, ',', QuoteMode.NON_NUMERIC);
268 
269 		// Answer list links (connects loinc observation codes to answerlist codes)
270 		handler = new LoincAnswerListLinkHandler(code2concept, valueSets);
271 		iterateOverZipFile(theDescriptors, LOINC_ANSWERLIST_LINK_FILE, handler, ',', QuoteMode.NON_NUMERIC);
272 
273 		// Part link file
274 		handler = new LoincPartLinkHandler(codeSystemVersion, code2concept);
275 		iterateOverZipFile(theDescriptors, LOINC_PART_LINK_FILE, handler, ',', QuoteMode.NON_NUMERIC);
276 
277 		// Part related code mapping
278 		handler = new LoincPartRelatedCodeMappingHandler(codeSystemVersion, code2concept, valueSets, conceptMaps, uploadProperties);
279 		iterateOverZipFile(theDescriptors, LOINC_PART_RELATED_CODE_MAPPING_FILE, handler, ',', QuoteMode.NON_NUMERIC);
280 
281 		// Document Ontology File
282 		handler = new LoincDocumentOntologyHandler(code2concept, propertyNamesToTypes, valueSets, conceptMaps, uploadProperties);
283 		iterateOverZipFile(theDescriptors, LOINC_DOCUMENT_ONTOLOGY_FILE, handler, ',', QuoteMode.NON_NUMERIC);
284 
285 		// RSNA Playbook file
286 		handler = new LoincRsnaPlaybookHandler(code2concept, valueSets, conceptMaps, uploadProperties);
287 		iterateOverZipFile(theDescriptors, LOINC_RSNA_PLAYBOOK_FILE, handler, ',', QuoteMode.NON_NUMERIC);
288 
289 		// Top 2000 Codes - US
290 		handler = new LoincTop2000LabResultsUsHandler(code2concept, valueSets, conceptMaps, uploadProperties);
291 		iterateOverZipFile(theDescriptors, LOINC_TOP2000_COMMON_LAB_RESULTS_US_FILE, handler, ',', QuoteMode.NON_NUMERIC);
292 
293 		// Top 2000 Codes - SI
294 		handler = new LoincTop2000LabResultsSiHandler(code2concept, valueSets, conceptMaps, uploadProperties);
295 		iterateOverZipFile(theDescriptors, LOINC_TOP2000_COMMON_LAB_RESULTS_SI_FILE, handler, ',', QuoteMode.NON_NUMERIC);
296 
297 		// Universal Lab Order ValueSet
298 		handler = new LoincUniversalOrderSetHandler(code2concept, valueSets, conceptMaps, uploadProperties);
299 		iterateOverZipFile(theDescriptors, LOINC_UNIVERSAL_LAB_ORDER_VALUESET_FILE, handler, ',', QuoteMode.NON_NUMERIC);
300 
301 		// IEEE Medical Device Codes
302 		handler = new LoincIeeeMedicalDeviceCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
303 		iterateOverZipFile(theDescriptors, LOINC_IEEE_MEDICAL_DEVICE_CODE_MAPPING_TABLE_CSV, handler, ',', QuoteMode.NON_NUMERIC);
304 
305 		// Imaging Document Codes
306 		handler = new LoincImagingDocumentCodeHandler(code2concept, valueSets, conceptMaps, uploadProperties);
307 		iterateOverZipFile(theDescriptors, LOINC_IMAGING_DOCUMENT_CODES_FILE, handler, ',', QuoteMode.NON_NUMERIC);
308 
309 		IOUtils.closeQuietly(theDescriptors);
310 
311 		for (Entry<String, TermConcept> next : code2concept.entrySet()) {
312 			TermConcept nextConcept = next.getValue();
313 			if (nextConcept.getParents().isEmpty()) {
314 				codeSystemVersion.getConcepts().add(nextConcept);
315 			}
316 		}
317 
318 		int valueSetCount = valueSets.size();
319 		int rootConceptCount = codeSystemVersion.getConcepts().size();
320 		int conceptCount = code2concept.size();
321 		ourLog.info("Have {} total concepts, {} root concepts, {} ValueSets", conceptCount, rootConceptCount, valueSetCount);
322 
323 		IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, loincCs, valueSets, conceptMaps);
324 
325 		return new UploadStatistics(conceptCount, target);
326 	}
327 
328 	private UploadStatistics processSnomedCtFiles(LoadedFileDescriptors theDescriptors, RequestDetails theRequestDetails) {
329 		final TermCodeSystemVersion codeSystemVersion = new TermCodeSystemVersion();
330 		final Map<String, TermConcept> id2concept = new HashMap<>();
331 		final Map<String, TermConcept> code2concept = new HashMap<>();
332 		final Set<String> validConceptIds = new HashSet<>();
333 
334 		IRecordHandler handler = new SctHandlerConcept(validConceptIds);
335 		iterateOverZipFile(theDescriptors, SCT_FILE_CONCEPT, handler, '\t', null);
336 
337 		ourLog.info("Have {} valid concept IDs", validConceptIds.size());
338 
339 		handler = new SctHandlerDescription(validConceptIds, code2concept, id2concept, codeSystemVersion);
340 		iterateOverZipFile(theDescriptors, SCT_FILE_DESCRIPTION, handler, '\t', null);
341 
342 		ourLog.info("Got {} concepts, cloning map", code2concept.size());
343 		final HashMap<String, TermConcept> rootConcepts = new HashMap<>(code2concept);
344 
345 		handler = new SctHandlerRelationship(codeSystemVersion, rootConcepts, code2concept);
346 		iterateOverZipFile(theDescriptors, SCT_FILE_RELATIONSHIP, handler, '\t', null);
347 
348 		IOUtils.closeQuietly(theDescriptors);
349 
350 		ourLog.info("Looking for root codes");
351 		rootConcepts
352 			.entrySet()
353 			.removeIf(theStringTermConceptEntry -> theStringTermConceptEntry.getValue().getParents().isEmpty() == false);
354 
355 		ourLog.info("Done loading SNOMED CT files - {} root codes, {} total codes", rootConcepts.size(), code2concept.size());
356 
357 		Counter circularCounter = new Counter();
358 		for (TermConcept next : rootConcepts.values()) {
359 			long count = circularCounter.getThenAdd();
360 			float pct = ((float) count / rootConcepts.size()) * 100.0f;
361 			ourLog.info(" * Scanning for circular refs - have scanned {} / {} codes ({}%)", count, rootConcepts.size(), pct);
362 			dropCircularRefs(next, new ArrayList<>(), code2concept, circularCounter);
363 		}
364 
365 		codeSystemVersion.getConcepts().addAll(rootConcepts.values());
366 
367 		CodeSystem cs = new org.hl7.fhir.r4.model.CodeSystem();
368 		cs.setUrl(SCT_URI);
369 		cs.setName("SNOMED CT");
370 		cs.setContent(CodeSystem.CodeSystemContentMode.NOTPRESENT);
371 		IIdType target = storeCodeSystem(theRequestDetails, codeSystemVersion, cs, null, null);
372 
373 		return new UploadStatistics(code2concept.size(), target);
374 	}
375 
376 	@VisibleForTesting
377 	void setTermSvcDstu3ForUnitTest(IHapiTerminologySvcDstu3 theTermSvcDstu3) {
378 		myTermSvcDstu3 = theTermSvcDstu3;
379 	}
380 
381 	@VisibleForTesting
382 	void setTermSvcForUnitTests(IHapiTerminologySvc theTermSvc) {
383 		myTermSvc = theTermSvc;
384 	}
385 
386 	private IIdType storeCodeSystem(RequestDetails theRequestDetails, final TermCodeSystemVersion theCodeSystemVersion, CodeSystem theCodeSystem, List<ValueSet> theValueSets, List<ConceptMap> theConceptMaps) {
387 		Validate.isTrue(theCodeSystem.getContent() == CodeSystem.CodeSystemContentMode.NOTPRESENT);
388 
389 		List<ValueSet> valueSets = ObjectUtils.defaultIfNull(theValueSets, Collections.emptyList());
390 		List<ConceptMap> conceptMaps = ObjectUtils.defaultIfNull(theConceptMaps, Collections.emptyList());
391 
392 		IIdType retVal;
393 		myTermSvc.setProcessDeferred(false);
394 		if (myTermSvcDstu3 != null) {
395 			retVal = myTermSvcDstu3.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
396 		} else {
397 			retVal = myTermSvcR4.storeNewCodeSystemVersion(theCodeSystem, theCodeSystemVersion, theRequestDetails, valueSets, conceptMaps);
398 		}
399 		myTermSvc.setProcessDeferred(true);
400 
401 		return retVal;
402 	}
403 
404 
405 	public static String firstNonBlank(String... theStrings) {
406 		String retVal = "";
407 		for (String nextString : theStrings) {
408 			if (isNotBlank(nextString)) {
409 				retVal = nextString;
410 				break;
411 			}
412 		}
413 		return retVal;
414 	}
415 
416 	public static TermConcept getOrCreateConcept(TermCodeSystemVersion codeSystemVersion, Map<String, TermConcept> id2concept, String id) {
417 		TermConcept concept = id2concept.get(id);
418 		if (concept == null) {
419 			concept = new TermConcept();
420 			id2concept.put(id, concept);
421 			concept.setCodeSystemVersion(codeSystemVersion);
422 		}
423 		return concept;
424 	}
425 
426 	static class LoadedFileDescriptors implements Closeable {
427 
428 		private List<File> myTemporaryFiles = new ArrayList<>();
429 		private List<IHapiTerminologyLoaderSvc.FileDescriptor> myUncompressedFileDescriptors = new ArrayList<>();
430 
431 		LoadedFileDescriptors(List<IHapiTerminologyLoaderSvc.FileDescriptor> theFileDescriptors) {
432 			try {
433 				for (FileDescriptor next : theFileDescriptors) {
434 					if (next.getFilename().toLowerCase().endsWith(".zip")) {
435 						ourLog.info("Uncompressing {} into temporary files", next.getFilename());
436 						try (InputStream inputStream = next.getInputStream()) {
437 							ZipInputStream zis = new ZipInputStream(new BufferedInputStream(inputStream));
438 							for (ZipEntry nextEntry; (nextEntry = zis.getNextEntry()) != null; ) {
439 								BOMInputStream fis = new BOMInputStream(zis);
440 								File nextTemporaryFile = File.createTempFile("hapifhir", ".tmp");
441 								nextTemporaryFile.deleteOnExit();
442 								FileOutputStream fos = new FileOutputStream(nextTemporaryFile, false);
443 								IOUtils.copy(fis, fos);
444 								String nextEntryFileName = nextEntry.getName();
445 								myUncompressedFileDescriptors.add(new FileDescriptor() {
446 									@Override
447 									public String getFilename() {
448 										return nextEntryFileName;
449 									}
450 
451 									@Override
452 									public InputStream getInputStream() {
453 										try {
454 											return new FileInputStream(nextTemporaryFile);
455 										} catch (FileNotFoundException e) {
456 											throw new InternalErrorException(e);
457 										}
458 									}
459 								});
460 								myTemporaryFiles.add(nextTemporaryFile);
461 							}
462 						}
463 					} else {
464 						myUncompressedFileDescriptors.add(next);
465 					}
466 
467 				}
468 			} catch (Exception e) {
469 				close();
470 				throw new InternalErrorException(e);
471 			}
472 		}
473 
474 		@Override
475 		public void close() {
476 			for (File next : myTemporaryFiles) {
477 				FileUtils.deleteQuietly(next);
478 			}
479 		}
480 
481 		List<IHapiTerminologyLoaderSvc.FileDescriptor> getUncompressedFileDescriptors() {
482 			return myUncompressedFileDescriptors;
483 		}
484 
485 		private List<String> notFound(List<String> theExpectedFilenameFragments) {
486 			Set<String> foundFragments = new HashSet<>();
487 			for (String nextExpected : theExpectedFilenameFragments) {
488 				for (FileDescriptor next : myUncompressedFileDescriptors) {
489 					if (next.getFilename().contains(nextExpected)) {
490 						foundFragments.add(nextExpected);
491 						break;
492 					}
493 				}
494 			}
495 
496 			ArrayList<String> notFoundFileNameFragments = new ArrayList<>(theExpectedFilenameFragments);
497 			notFoundFileNameFragments.removeAll(foundFragments);
498 			return notFoundFileNameFragments;
499 		}
500 
501 		private void verifyMandatoryFilesExist(List<String> theExpectedFilenameFragments) {
502 			List<String> notFound = notFound(theExpectedFilenameFragments);
503 			if (!notFound.isEmpty()) {
504 				throw new UnprocessableEntityException("Could not find the following mandatory files in input: " + notFound);
505 			}
506 		}
507 
508 		private void verifyOptionalFilesExist(List<String> theExpectedFilenameFragments) {
509 			List<String> notFound = notFound(theExpectedFilenameFragments);
510 			if (!notFound.isEmpty()) {
511 				ourLog.warn("Could not find the following optional file: " + notFound);
512 			}
513 		}
514 
515 
516 	}
517 }