Source code

001package ca.uhn.fhir.jpa.packages;
002
003/*-
004 * #%L
005 * HAPI FHIR JPA Server
006 * %%
007 * Copyright (C) 2014 - 2022 Smile CDR, Inc.
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 *
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 *
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import ca.uhn.fhir.i18n.Msg;
024import ca.uhn.fhir.context.BaseRuntimeChildDefinition;
025import ca.uhn.fhir.context.FhirContext;
026import ca.uhn.fhir.context.FhirVersionEnum;
027import ca.uhn.fhir.interceptor.model.RequestPartitionId;
028import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
029import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
030import ca.uhn.fhir.jpa.api.model.ExpungeOptions;
031import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
032import ca.uhn.fhir.jpa.binary.svc.NullBinaryStorageSvcImpl;
033import ca.uhn.fhir.jpa.dao.data.INpmPackageDao;
034import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao;
035import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionResourceDao;
036import ca.uhn.fhir.jpa.model.config.PartitionSettings;
037import ca.uhn.fhir.jpa.model.entity.NpmPackageEntity;
038import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionEntity;
039import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionResourceEntity;
040import ca.uhn.fhir.jpa.model.entity.ResourceTable;
041import ca.uhn.fhir.jpa.model.util.JpaConstants;
042import ca.uhn.fhir.jpa.partition.SystemRequestDetails;
043import ca.uhn.fhir.rest.api.Constants;
044import ca.uhn.fhir.rest.api.EncodingEnum;
045import ca.uhn.fhir.rest.api.server.storage.ResourcePersistentId;
046import ca.uhn.fhir.rest.server.exceptions.InternalErrorException;
047import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
048import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
049import ca.uhn.fhir.util.BinaryUtil;
050import ca.uhn.fhir.util.ClasspathUtil;
051import ca.uhn.fhir.util.ResourceUtil;
052import ca.uhn.fhir.util.StringUtil;
053import org.apache.commons.collections4.comparators.ReverseComparator;
054import org.apache.commons.io.IOUtils;
055import org.apache.commons.lang3.Validate;
056import org.apache.http.client.methods.CloseableHttpResponse;
057import org.apache.http.client.methods.HttpGet;
058import org.apache.http.conn.HttpClientConnectionManager;
059import org.apache.http.impl.client.HttpClientBuilder;
060import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
061import org.hl7.fhir.exceptions.FHIRException;
062import org.hl7.fhir.instance.model.api.IBaseBinary;
063import org.hl7.fhir.instance.model.api.IBaseResource;
064import org.hl7.fhir.instance.model.api.IIdType;
065import org.hl7.fhir.instance.model.api.IPrimitiveType;
066import org.hl7.fhir.utilities.npm.BasePackageCacheManager;
067import org.hl7.fhir.utilities.npm.NpmPackage;
068import org.slf4j.Logger;
069import org.slf4j.LoggerFactory;
070import org.springframework.beans.factory.annotation.Autowired;
071import org.springframework.data.domain.PageRequest;
072import org.springframework.data.domain.Slice;
073import org.springframework.transaction.PlatformTransactionManager;
074import org.springframework.transaction.support.TransactionTemplate;
075
076import javax.annotation.Nonnull;
077import javax.annotation.Nullable;
078import javax.persistence.EntityManager;
079import javax.persistence.PersistenceContext;
080import javax.persistence.TypedQuery;
081import javax.persistence.criteria.CriteriaBuilder;
082import javax.persistence.criteria.CriteriaQuery;
083import javax.persistence.criteria.Join;
084import javax.persistence.criteria.JoinType;
085import javax.persistence.criteria.Predicate;
086import javax.persistence.criteria.Root;
087import javax.transaction.Transactional;
088import java.io.ByteArrayInputStream;
089import java.io.IOException;
090import java.io.InputStream;
091import java.net.URI;
092import java.net.URISyntaxException;
093import java.nio.charset.StandardCharsets;
094import java.nio.file.Files;
095import java.nio.file.Paths;
096import java.util.ArrayList;
097import java.util.Collection;
098import java.util.Collections;
099import java.util.Date;
100import java.util.HashMap;
101import java.util.List;
102import java.util.Map;
103import java.util.Optional;
104import java.util.stream.Collectors;
105
106import static ca.uhn.fhir.jpa.dao.LegacySearchBuilder.toPredicateArray;
107import static ca.uhn.fhir.util.StringUtil.toUtf8String;
108import static org.apache.commons.lang3.StringUtils.defaultString;
109import static org.apache.commons.lang3.StringUtils.isNotBlank;
110
111public class JpaPackageCache extends BasePackageCacheManager implements IHapiPackageCacheManager {
112
113        public static final String UTF8_BOM = "\uFEFF";
114        private static final Logger ourLog = LoggerFactory.getLogger(JpaPackageCache.class);
115        private final Map<FhirVersionEnum, FhirContext> myVersionToContext = Collections.synchronizedMap(new HashMap<>());
116        @PersistenceContext
117        protected EntityManager myEntityManager;
118        @Autowired
119        private INpmPackageDao myPackageDao;
120        @Autowired
121        private INpmPackageVersionDao myPackageVersionDao;
122        @Autowired
123        private INpmPackageVersionResourceDao myPackageVersionResourceDao;
124        @Autowired
125        private DaoRegistry myDaoRegistry;
126        @Autowired
127        private FhirContext myCtx;
128        @Autowired
129        private PlatformTransactionManager myTxManager;
130        @Autowired
131        private PartitionSettings myPartitionSettings;
132
133        @Autowired(required = false)//It is possible that some implementers will not create such a bean.
134        private IBinaryStorageSvc myBinaryStorageSvc;
135
136        @Override
137        @Transactional
138        public NpmPackage loadPackageFromCacheOnly(String theId, @Nullable String theVersion) {
139                Optional<NpmPackageVersionEntity> packageVersion = loadPackageVersionEntity(theId, theVersion);
140                if (!packageVersion.isPresent() && theVersion.endsWith(".x")) {
141                        String lookupVersion = theVersion;
142                        do {
143                                lookupVersion = lookupVersion.substring(0, lookupVersion.length() - 2);
144                        } while (lookupVersion.endsWith(".x"));
145
146                        List<String> candidateVersionIds = myPackageVersionDao.findVersionIdsByPackageIdAndLikeVersion(theId, lookupVersion + ".%");
147                        if (candidateVersionIds.size() > 0) {
148                                candidateVersionIds.sort(PackageVersionComparator.INSTANCE);
149                                packageVersion = loadPackageVersionEntity(theId, candidateVersionIds.get(candidateVersionIds.size() - 1));
150                        }
151
152                }
153
154                return packageVersion.map(t -> loadPackage(t)).orElse(null);
155        }
156
157        private Optional<NpmPackageVersionEntity> loadPackageVersionEntity(String theId, @Nullable String theVersion) {
158                Validate.notBlank(theId, "theId must be populated");
159
160                Optional<NpmPackageVersionEntity> packageVersion = Optional.empty();
161                if (isNotBlank(theVersion) && !"latest".equals(theVersion)) {
162                        packageVersion = myPackageVersionDao.findByPackageIdAndVersion(theId, theVersion);
163                } else {
164                        Optional<NpmPackageEntity> pkg = myPackageDao.findByPackageId(theId);
165                        if (pkg.isPresent()) {
166                                packageVersion = myPackageVersionDao.findByPackageIdAndVersion(theId, pkg.get().getCurrentVersionId());
167                        }
168                }
169                return packageVersion;
170        }
171
172        private NpmPackage loadPackage(NpmPackageVersionEntity thePackageVersion) {
173                PackageContents content = loadPackageContents(thePackageVersion);
174                ByteArrayInputStream inputStream = new ByteArrayInputStream(content.getBytes());
175                try {
176                        return NpmPackage.fromPackage(inputStream);
177                } catch (IOException e) {
178                        throw new InternalErrorException(Msg.code(1294) + e);
179                }
180        }
181
182        private IHapiPackageCacheManager.PackageContents loadPackageContents(NpmPackageVersionEntity thePackageVersion) {
183                IFhirResourceDao<? extends IBaseBinary> binaryDao = getBinaryDao();
184                IBaseBinary binary = binaryDao.readByPid(new ResourcePersistentId(thePackageVersion.getPackageBinary().getId()));
185                try {
186                        byte[] content = fetchBlobFromBinary(binary);
187                        PackageContents retVal = new PackageContents()
188                                .setBytes(content)
189                                .setPackageId(thePackageVersion.getPackageId())
190                                .setVersion(thePackageVersion.getVersionId())
191                                .setLastModified(thePackageVersion.getUpdatedTime());
192                        return retVal;
193                } catch (IOException e) {
194                        throw new InternalErrorException(Msg.code(1295) + "Failed to load package. There was a problem reading binaries", e);
195                }
196        }
197
198        /**
199         * Helper method which will attempt to use the IBinaryStorageSvc to resolve the binary blob if available. If
200         * the bean is unavailable, fallback to assuming we are using an embedded base64 in the data element.
201         * @param theBinary the Binary who's `data` blob you want to retrieve
202         * @return a byte array containing the blob.
203         *
204         * @throws IOException
205         */
206        private byte[] fetchBlobFromBinary(IBaseBinary theBinary) throws IOException {
207                if (myBinaryStorageSvc != null && !(myBinaryStorageSvc instanceof NullBinaryStorageSvcImpl)) {
208                        return myBinaryStorageSvc.fetchDataBlobFromBinary(theBinary);
209                } else {
210                        byte[] value = BinaryUtil.getOrCreateData(myCtx, theBinary).getValue();
211                        if (value == null) {
212                                throw new InternalErrorException(Msg.code(1296) + "Failed to fetch blob from Binary/" + theBinary.getIdElement());
213                        }
214                        return value;
215                }
216        }
217
218        @SuppressWarnings("unchecked")
219        private IFhirResourceDao<IBaseBinary> getBinaryDao() {
220                return myDaoRegistry.getResourceDao("Binary");
221        }
222
223        @Override
224        public NpmPackage addPackageToCache(String thePackageId, String thePackageVersionId, InputStream thePackageTgzInputStream, String theSourceDesc) throws IOException {
225                Validate.notBlank(thePackageId, "thePackageId must not be null");
226                Validate.notBlank(thePackageVersionId, "thePackageVersionId must not be null");
227                Validate.notNull(thePackageTgzInputStream, "thePackageTgzInputStream must not be null");
228
229                byte[] bytes = IOUtils.toByteArray(thePackageTgzInputStream);
230
231                ourLog.info("Parsing package .tar.gz ({} bytes) from {}", bytes.length, theSourceDesc);
232
233                NpmPackage npmPackage = NpmPackage.fromPackage(new ByteArrayInputStream(bytes));
234                if (!npmPackage.id().equalsIgnoreCase(thePackageId)) {
235                        throw new InvalidRequestException(Msg.code(1297) + "Package ID " + npmPackage.id() + " doesn't match expected: " + thePackageId);
236                }
237                if (!PackageVersionComparator.isEquivalent(thePackageVersionId, npmPackage.version())) {
238                        throw new InvalidRequestException(Msg.code(1298) + "Package ID " + npmPackage.version() + " doesn't match expected: " + thePackageVersionId);
239                }
240
241                String packageVersionId = npmPackage.version();
242                FhirVersionEnum fhirVersion = FhirVersionEnum.forVersionString(npmPackage.fhirVersion());
243                if (fhirVersion == null) {
244                        throw new InvalidRequestException(Msg.code(1299) + "Unknown FHIR version: " + npmPackage.fhirVersion());
245                }
246                FhirContext packageContext = getFhirContext(fhirVersion);
247
248                IBaseBinary binary = createPackageBinary(bytes);
249
250                return newTxTemplate().execute(tx -> {
251
252                        ResourceTable persistedPackage = createResourceBinary(binary);
253                        NpmPackageEntity pkg = myPackageDao.findByPackageId(thePackageId).orElseGet(() -> createPackage(npmPackage));
254                        NpmPackageVersionEntity packageVersion = myPackageVersionDao.findByPackageIdAndVersion(thePackageId, packageVersionId).orElse(null);
255                        if (packageVersion != null) {
256                                NpmPackage existingPackage = loadPackageFromCacheOnly(packageVersion.getPackageId(), packageVersion.getVersionId());
257                                String msg = "Package version already exists in local storage, no action taken: " + thePackageId + "#" + packageVersionId;
258                                getProcessingMessages(existingPackage).add(msg);
259                                ourLog.info(msg);
260                                return existingPackage;
261                        }
262
263                        boolean currentVersion = updateCurrentVersionFlagForAllPackagesBasedOnNewIncomingVersion(thePackageId, packageVersionId);
264                        String packageDesc = null;
265                        if (npmPackage.description() != null) {
266                                if (npmPackage.description().length() > NpmPackageVersionEntity.PACKAGE_DESC_LENGTH) {
267                                        packageDesc = npmPackage.description().substring(0, NpmPackageVersionEntity.PACKAGE_DESC_LENGTH - 4) + "...";
268                                } else {
269                                        packageDesc = npmPackage.description();
270                                }
271                        }
272                        if (currentVersion) {
273                                getProcessingMessages(npmPackage).add("Marking package " + thePackageId + "#" + thePackageVersionId + " as current version");
274                                pkg.setCurrentVersionId(packageVersionId);
275                                pkg.setDescription(packageDesc);
276                                myPackageDao.save(pkg);
277                        } else {
278                                getProcessingMessages(npmPackage).add("Package " + thePackageId + "#" + thePackageVersionId + " is not the newest version");
279                        }
280
281                        packageVersion = new NpmPackageVersionEntity();
282                        packageVersion.setPackageId(thePackageId);
283                        packageVersion.setVersionId(packageVersionId);
284                        packageVersion.setPackage(pkg);
285                        packageVersion.setPackageBinary(persistedPackage);
286                        packageVersion.setSavedTime(new Date());
287                        packageVersion.setDescription(packageDesc);
288                        packageVersion.setFhirVersionId(npmPackage.fhirVersion());
289                        packageVersion.setFhirVersion(fhirVersion);
290                        packageVersion.setCurrentVersion(currentVersion);
291                        packageVersion.setPackageSizeBytes(bytes.length);
292                        packageVersion = myPackageVersionDao.save(packageVersion);
293
294                        String dirName = "package";
295                        NpmPackage.NpmPackageFolder packageFolder = npmPackage.getFolders().get(dirName);
296                        for (Map.Entry<String, List<String>> nextTypeToFiles : packageFolder.getTypes().entrySet()) {
297                                String nextType = nextTypeToFiles.getKey();
298                                for (String nextFile : nextTypeToFiles.getValue()) {
299
300                                        byte[] contents;
301                                        String contentsString;
302                                        try {
303                                                contents = packageFolder.fetchFile(nextFile);
304                                                contentsString = toUtf8String(contents);
305                                        } catch (IOException e) {
306                                                throw new InternalErrorException(Msg.code(1300) + e);
307                                        }
308
309                                        IBaseResource resource;
310                                        if (nextFile.toLowerCase().endsWith(".xml")) {
311                                                resource = packageContext.newXmlParser().parseResource(contentsString);
312                                        } else if (nextFile.toLowerCase().endsWith(".json")) {
313                                                resource = packageContext.newJsonParser().parseResource(contentsString);
314                                        } else {
315                                                getProcessingMessages(npmPackage).add("Not indexing file: " + nextFile);
316                                                continue;
317                                        }
318
319                                        /*
320                                         * Re-encode the resource as JSON with the narrative removed in order to reduce the footprint.
321                                         * This is useful since we'll be loading these resources back and hopefully keeping lots of
322                                         * them in memory in order to speed up validation activities.
323                                         */
324                                        String contentType = Constants.CT_FHIR_JSON_NEW;
325                                        ResourceUtil.removeNarrative(packageContext, resource);
326                                        byte[] minimizedContents = packageContext.newJsonParser().encodeResourceToString(resource).getBytes(StandardCharsets.UTF_8);
327
328                                        IBaseBinary resourceBinary = createPackageResourceBinary(nextFile, minimizedContents, contentType);
329                                        ResourceTable persistedResource = createResourceBinary(resourceBinary);
330
331                                        NpmPackageVersionResourceEntity resourceEntity = new NpmPackageVersionResourceEntity();
332                                        resourceEntity.setPackageVersion(packageVersion);
333                                        resourceEntity.setResourceBinary(persistedResource);
334                                        resourceEntity.setDirectory(dirName);
335                                        resourceEntity.setFhirVersionId(npmPackage.fhirVersion());
336                                        resourceEntity.setFhirVersion(fhirVersion);
337                                        resourceEntity.setFilename(nextFile);
338                                        resourceEntity.setResourceType(nextType);
339                                        resourceEntity.setResSizeBytes(contents.length);
340                                        BaseRuntimeChildDefinition urlChild = packageContext.getResourceDefinition(nextType).getChildByName("url");
341                                        BaseRuntimeChildDefinition versionChild = packageContext.getResourceDefinition(nextType).getChildByName("version");
342                                        String url = null;
343                                        String version = null;
344                                        if (urlChild != null) {
345                                                url = urlChild.getAccessor().getFirstValueOrNull(resource).map(t -> ((IPrimitiveType<?>) t).getValueAsString()).orElse(null);
346                                                resourceEntity.setCanonicalUrl(url);
347                                                version = versionChild.getAccessor().getFirstValueOrNull(resource).map(t -> ((IPrimitiveType<?>) t).getValueAsString()).orElse(null);
348                                                resourceEntity.setCanonicalVersion(version);
349                                        }
350                                        myPackageVersionResourceDao.save(resourceEntity);
351
352                                        String resType = packageContext.getResourceType(resource);
353                                        String msg = "Indexing " + resType + " Resource[" + dirName + '/' + nextFile + "] with URL: " + defaultString(url) + "|" + defaultString(version);
354                                        getProcessingMessages(npmPackage).add(msg);
355                                        ourLog.info("Package[{}#{}] " + msg, thePackageId, packageVersionId);
356                                }
357                        }
358
359                        getProcessingMessages(npmPackage).add("Successfully added package " + npmPackage.id() + "#" + npmPackage.version() + " to registry");
360
361                        return npmPackage;
362                });
363
364        }
365
366        private ResourceTable createResourceBinary(IBaseBinary theResourceBinary) {
367
368                if (myPartitionSettings.isPartitioningEnabled()) {
369                        SystemRequestDetails requestDetails = new SystemRequestDetails();
370                        if (myPartitionSettings.isUnnamedPartitionMode() && myPartitionSettings.getDefaultPartitionId() != null) {
371                                requestDetails.setRequestPartitionId(RequestPartitionId.fromPartitionId(myPartitionSettings.getDefaultPartitionId()));
372                        } else {
373                                requestDetails.setTenantId(JpaConstants.DEFAULT_PARTITION_NAME);
374                        }
375                        return (ResourceTable) getBinaryDao().create(theResourceBinary, requestDetails).getEntity();
376                } else {
377                        return (ResourceTable) getBinaryDao().create(theResourceBinary).getEntity();
378                }
379        }
380
381        private boolean updateCurrentVersionFlagForAllPackagesBasedOnNewIncomingVersion(String thePackageId, String thePackageVersion) {
382                Collection<NpmPackageVersionEntity> existingVersions = myPackageVersionDao.findByPackageId(thePackageId);
383                boolean retVal = true;
384
385                for (NpmPackageVersionEntity next : existingVersions) {
386                        int cmp = PackageVersionComparator.INSTANCE.compare(next.getVersionId(), thePackageVersion);
387                        assert cmp != 0;
388                        if (cmp < 0) {
389                                if (next.isCurrentVersion()) {
390                                        next.setCurrentVersion(false);
391                                        myPackageVersionDao.save(next);
392                                }
393                        } else {
394                                retVal = false;
395                        }
396                }
397
398                return retVal;
399        }
400
401        @Nonnull
402        public FhirContext getFhirContext(FhirVersionEnum theFhirVersion) {
403                return myVersionToContext.computeIfAbsent(theFhirVersion, v -> new FhirContext(v));
404        }
405
406        private IBaseBinary createPackageBinary(byte[] theBytes) {
407                IBaseBinary binary = BinaryUtil.newBinary(myCtx);
408                BinaryUtil.setData(myCtx, binary, theBytes, Constants.CT_APPLICATION_GZIP);
409                return binary;
410        }
411
412        private IBaseBinary createPackageResourceBinary(String theFileName, byte[] theBytes, String theContentType) {
413                IBaseBinary binary = BinaryUtil.newBinary(myCtx);
414                BinaryUtil.setData(myCtx, binary, theBytes, theContentType);
415                return binary;
416        }
417
418        private NpmPackageEntity createPackage(NpmPackage theNpmPackage) {
419                NpmPackageEntity entity = new NpmPackageEntity();
420                entity.setPackageId(theNpmPackage.id());
421                entity.setCurrentVersionId(theNpmPackage.version());
422                return myPackageDao.save(entity);
423        }
424
425        @Override
426        @Transactional
427        public NpmPackage loadPackage(String thePackageId, String thePackageVersion) throws FHIRException, IOException {
428                NpmPackage cachedPackage = loadPackageFromCacheOnly(thePackageId, thePackageVersion);
429                if (cachedPackage != null) {
430                        return cachedPackage;
431                }
432
433                InputStreamWithSrc pkg = super.loadFromPackageServer(thePackageId, thePackageVersion);
434                if (pkg == null) {
435                        throw new ResourceNotFoundException(Msg.code(1301) + "Unable to locate package " + thePackageId + "#" + thePackageVersion);
436                }
437
438                try {
439                        NpmPackage retVal = addPackageToCache(thePackageId, thePackageVersion == null ? pkg.version : thePackageVersion, pkg.stream, pkg.url);
440                        getProcessingMessages(retVal).add(0, "Package fetched from server at: " + pkg.url);
441                        return retVal;
442                } finally {
443                        pkg.stream.close();
444                }
445
446        }
447
448        private TransactionTemplate newTxTemplate() {
449                return new TransactionTemplate(myTxManager);
450        }
451
452        @Override
453        @Transactional(Transactional.TxType.NEVER)
454        public NpmPackage installPackage(PackageInstallationSpec theInstallationSpec) throws IOException {
455                Validate.notBlank(theInstallationSpec.getName(), "thePackageId must not be blank");
456                Validate.notBlank(theInstallationSpec.getVersion(), "thePackageVersion must not be blank");
457
458                String sourceDescription = "Embedded content";
459                if (isNotBlank(theInstallationSpec.getPackageUrl())) {
460                        byte[] contents = loadPackageUrlContents(theInstallationSpec.getPackageUrl());
461                        theInstallationSpec.setPackageContents(contents);
462                        sourceDescription = theInstallationSpec.getPackageUrl();
463                }
464
465                if (theInstallationSpec.getPackageContents() != null) {
466                        return addPackageToCache(theInstallationSpec.getName(), theInstallationSpec.getVersion(), new ByteArrayInputStream(theInstallationSpec.getPackageContents()), sourceDescription);
467                }
468
469                return newTxTemplate().execute(tx -> {
470                        try {
471                                return loadPackage(theInstallationSpec.getName(), theInstallationSpec.getVersion());
472                        } catch (IOException e) {
473                                throw new InternalErrorException(Msg.code(1302) + e);
474                        }
475                });
476        }
477
478        protected byte[] loadPackageUrlContents(String thePackageUrl) {
479                if (thePackageUrl.startsWith("classpath:")) {
480                        return ClasspathUtil.loadResourceAsByteArray(thePackageUrl.substring("classpath:" .length()));
481                } else if (thePackageUrl.startsWith("file:")) {
482                        try {
483                                byte[] bytes = Files.readAllBytes(Paths.get(new URI(thePackageUrl)));
484                                return bytes;
485                        } catch (IOException | URISyntaxException e) {
486                                throw new InternalErrorException(Msg.code(2031) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
487                        }
488                } else {
489                        HttpClientConnectionManager connManager = new BasicHttpClientConnectionManager();
490                        try (CloseableHttpResponse request = HttpClientBuilder
491                                .create()
492                                .setConnectionManager(connManager)
493                                .build()
494                                .execute(new HttpGet(thePackageUrl))) {
495                                if (request.getStatusLine().getStatusCode() != 200) {
496                                        throw new ResourceNotFoundException(Msg.code(1303) + "Received HTTP " + request.getStatusLine().getStatusCode() + " from URL: " + thePackageUrl);
497                                }
498                                return IOUtils.toByteArray(request.getEntity().getContent());
499                        } catch (IOException e) {
500                                throw new InternalErrorException(Msg.code(1304) + "Error loading \"" + thePackageUrl + "\": " + e.getMessage());
501                        }
502                }
503        }
504
505        @Override
506        @Transactional
507        public IBaseResource loadPackageAssetByUrl(FhirVersionEnum theFhirVersion, String theCanonicalUrl) {
508
509                String canonicalUrl = theCanonicalUrl;
510
511                int versionSeparator = canonicalUrl.lastIndexOf('|');
512                Slice<NpmPackageVersionResourceEntity> slice;
513                if (versionSeparator != -1) {
514                        String canonicalVersion = canonicalUrl.substring(versionSeparator + 1);
515                        canonicalUrl = canonicalUrl.substring(0, versionSeparator);
516                        slice = myPackageVersionResourceDao.findCurrentVersionByCanonicalUrlAndVersion(PageRequest.of(0, 1), theFhirVersion, canonicalUrl, canonicalVersion);
517                } else {
518                        slice = myPackageVersionResourceDao.findCurrentVersionByCanonicalUrl(PageRequest.of(0, 1), theFhirVersion, canonicalUrl);
519                }
520
521                if (slice.isEmpty()) {
522                        return null;
523                } else {
524                        NpmPackageVersionResourceEntity contents = slice.getContent().get(0);
525                        return loadPackageEntity(contents);
526                }
527        }
528
529        private IBaseResource loadPackageEntity(NpmPackageVersionResourceEntity contents) {
530                try {
531                        ResourcePersistentId binaryPid = new ResourcePersistentId(contents.getResourceBinary().getId());
532                        IBaseBinary binary = getBinaryDao().readByPid(binaryPid);
533                        byte[] resourceContentsBytes= fetchBlobFromBinary(binary);
534                        String resourceContents = new String(resourceContentsBytes, StandardCharsets.UTF_8);
535                        FhirContext packageContext = getFhirContext(contents.getFhirVersion());
536                        return EncodingEnum.detectEncoding(resourceContents).newParser(packageContext).parseResource(resourceContents);
537                } catch (Exception e) {
538                        throw new RuntimeException(Msg.code(1305) + "Failed to load package resource " + contents, e);
539                }
540        }
541
542        @Override
543        @Transactional
544        public NpmPackageMetadataJson loadPackageMetadata(String thePackageId) {
545                NpmPackageMetadataJson retVal = new NpmPackageMetadataJson();
546
547                Optional<NpmPackageEntity> pkg = myPackageDao.findByPackageId(thePackageId);
548                if (!pkg.isPresent()) {
549                        throw new ResourceNotFoundException(Msg.code(1306) + "Unknown package ID: " + thePackageId);
550                }
551
552                List<NpmPackageVersionEntity> packageVersions = new ArrayList<>(myPackageVersionDao.findByPackageId(thePackageId));
553                packageVersions.sort(new ReverseComparator<>((o1, o2) -> PackageVersionComparator.INSTANCE.compare(o1.getVersionId(), o2.getVersionId())));
554
555                for (NpmPackageVersionEntity next : packageVersions) {
556                        if (next.isCurrentVersion()) {
557                                retVal.setDistTags(new NpmPackageMetadataJson.DistTags().setLatest(next.getVersionId()));
558                        }
559
560                        NpmPackageMetadataJson.Version version = new NpmPackageMetadataJson.Version();
561                        version.setFhirVersion(next.getFhirVersionId());
562                        version.setDescription(next.getDescription());
563                        version.setName(next.getPackageId());
564                        version.setVersion(next.getVersionId());
565                        version.setBytes(next.getPackageSizeBytes());
566                        retVal.addVersion(version);
567
568                }
569
570                return retVal;
571        }
572
573        @Override
574        @Transactional
575        public PackageContents loadPackageContents(String thePackageId, String theVersion) {
576                Optional<NpmPackageVersionEntity> entity = loadPackageVersionEntity(thePackageId, theVersion);
577                return entity.map(t -> loadPackageContents(t)).orElse(null);
578        }
579
580        @Override
581        @Transactional
582        public NpmPackageSearchResultJson search(PackageSearchSpec thePackageSearchSpec) {
583                NpmPackageSearchResultJson retVal = new NpmPackageSearchResultJson();
584
585                CriteriaBuilder cb = myEntityManager.getCriteriaBuilder();
586
587                // Query for total
588                {
589                        CriteriaQuery<Long> countCriteriaQuery = cb.createQuery(Long.class);
590                        Root<NpmPackageVersionEntity> countCriteriaRoot = countCriteriaQuery.from(NpmPackageVersionEntity.class);
591                        countCriteriaQuery.multiselect(cb.countDistinct(countCriteriaRoot.get("myPackageId")));
592
593                        List<Predicate> predicates = createSearchPredicates(thePackageSearchSpec, cb, countCriteriaRoot);
594
595                        countCriteriaQuery.where(toPredicateArray(predicates));
596                        Long total = myEntityManager.createQuery(countCriteriaQuery).getSingleResult();
597                        retVal.setTotal(Math.toIntExact(total));
598                }
599
600                // Query for results
601                {
602                        CriteriaQuery<NpmPackageVersionEntity> criteriaQuery = cb.createQuery(NpmPackageVersionEntity.class);
603                        Root<NpmPackageVersionEntity> root = criteriaQuery.from(NpmPackageVersionEntity.class);
604
605                        List<Predicate> predicates = createSearchPredicates(thePackageSearchSpec, cb, root);
606
607                        criteriaQuery.where(toPredicateArray(predicates));
608                        criteriaQuery.orderBy(cb.asc(root.get("myPackageId")));
609                        TypedQuery<NpmPackageVersionEntity> query = myEntityManager.createQuery(criteriaQuery);
610                        query.setFirstResult(thePackageSearchSpec.getStart());
611                        query.setMaxResults(thePackageSearchSpec.getSize());
612
613                        List<NpmPackageVersionEntity> resultList = query.getResultList();
614                        for (NpmPackageVersionEntity next : resultList) {
615
616                                if (!retVal.hasPackageWithId(next.getPackageId())) {
617                                        retVal
618                                                .addObject()
619                                                .getPackage()
620                                                .setName(next.getPackageId())
621                                                .setDescription(next.getPackage().getDescription())
622                                                .setVersion(next.getVersionId())
623                                                .addFhirVersion(next.getFhirVersionId())
624                                                .setBytes(next.getPackageSizeBytes());
625                                } else {
626                                        NpmPackageSearchResultJson.Package retPackage = retVal.getPackageWithId(next.getPackageId());
627                                        retPackage.addFhirVersion(next.getFhirVersionId());
628
629                                        int cmp = PackageVersionComparator.INSTANCE.compare(next.getVersionId(), retPackage.getVersion());
630                                        if (cmp > 0) {
631                                                retPackage.setVersion(next.getVersionId());
632                                        }
633
634                                }
635                        }
636
637                }
638
639
640                return retVal;
641        }
642
643        @Override
644        @Transactional
645        public PackageDeleteOutcomeJson uninstallPackage(String thePackageId, String theVersion) {
646                PackageDeleteOutcomeJson retVal = new PackageDeleteOutcomeJson();
647
648                Optional<NpmPackageVersionEntity> packageVersion = myPackageVersionDao.findByPackageIdAndVersion(thePackageId, theVersion);
649                if (packageVersion.isPresent()) {
650
651                        String msg = "Deleting package " + thePackageId + "#" + theVersion;
652                        ourLog.info(msg);
653                        retVal.getMessage().add(msg);
654
655                        for (NpmPackageVersionResourceEntity next : packageVersion.get().getResources()) {
656                                msg = "Deleting package +" + thePackageId + "#" + theVersion + "resource: " + next.getCanonicalUrl();
657                                ourLog.info(msg);
658                                retVal.getMessage().add(msg);
659
660                                myPackageVersionResourceDao.delete(next);
661
662                                ExpungeOptions options = new ExpungeOptions();
663                                options.setExpungeDeletedResources(true).setExpungeOldVersions(true);
664                                deleteAndExpungeResourceBinary(next.getResourceBinary().getIdDt().toVersionless(), options);
665                        }
666
667                        myPackageVersionDao.delete(packageVersion.get());
668
669                        ExpungeOptions options = new ExpungeOptions();
670                        options.setExpungeDeletedResources(true).setExpungeOldVersions(true);
671                        deleteAndExpungeResourceBinary(packageVersion.get().getPackageBinary().getIdDt().toVersionless(), options);
672
673                        Collection<NpmPackageVersionEntity> remainingVersions = myPackageVersionDao.findByPackageId(thePackageId);
674                        if (remainingVersions.size() == 0) {
675                                msg = "No versions of package " + thePackageId + " remain";
676                                ourLog.info(msg);
677                                retVal.getMessage().add(msg);
678                                Optional<NpmPackageEntity> pkgEntity = myPackageDao.findByPackageId(thePackageId);
679                                myPackageDao.delete(pkgEntity.get());
680                        } else {
681
682                                List<NpmPackageVersionEntity> versions = remainingVersions
683                                        .stream()
684                                        .sorted((o1, o2) -> PackageVersionComparator.INSTANCE.compare(o1.getVersionId(), o2.getVersionId()))
685                                        .collect(Collectors.toList());
686                                for (int i = 0; i < versions.size(); i++) {
687                                        boolean isCurrent = i == versions.size() - 1;
688                                        if (isCurrent != versions.get(i).isCurrentVersion()) {
689                                                versions.get(i).setCurrentVersion(isCurrent);
690                                                myPackageVersionDao.save(versions.get(i));
691                                        }
692                                }
693
694                        }
695
696                } else {
697
698                        String msg = "No package found with the given ID";
699                        retVal.getMessage().add(msg);
700
701                }
702
703                return retVal;
704        }
705
706        @Override
707        @Transactional
708        public List<IBaseResource> loadPackageAssetsByType(FhirVersionEnum theFhirVersion, String theResourceType) {
709//              List<NpmPackageVersionResourceEntity> outcome = myPackageVersionResourceDao.findAll();
710                Slice<NpmPackageVersionResourceEntity> outcome = myPackageVersionResourceDao.findCurrentVersionByResourceType(PageRequest.of(0, 1000), theFhirVersion, theResourceType);
711                return outcome.stream().map(t -> loadPackageEntity(t)).collect(Collectors.toList());
712        }
713
714        private void deleteAndExpungeResourceBinary(IIdType theResourceBinaryId, ExpungeOptions theOptions) {
715                getBinaryDao().delete(theResourceBinaryId, new SystemRequestDetails()).getEntity();
716                getBinaryDao().forceExpungeInExistingTransaction(theResourceBinaryId, theOptions, new SystemRequestDetails());
717        }
718
719
720        @Nonnull
721        public List<Predicate> createSearchPredicates(PackageSearchSpec thePackageSearchSpec, CriteriaBuilder theCb, Root<NpmPackageVersionEntity> theRoot) {
722                List<Predicate> predicates = new ArrayList<>();
723
724                if (isNotBlank(thePackageSearchSpec.getResourceUrl())) {
725                        Join<NpmPackageVersionEntity, NpmPackageVersionResourceEntity> resources = theRoot.join("myResources", JoinType.LEFT);
726
727                        predicates.add(theCb.equal(resources.get("myCanonicalUrl").as(String.class), thePackageSearchSpec.getResourceUrl()));
728                }
729
730                if (isNotBlank(thePackageSearchSpec.getDescription())) {
731                        String searchTerm = "%" + thePackageSearchSpec.getDescription() + "%";
732                        searchTerm = StringUtil.normalizeStringForSearchIndexing(searchTerm);
733                        predicates.add(theCb.like(theRoot.get("myDescriptionUpper").as(String.class), searchTerm));
734                }
735
736                if (isNotBlank(thePackageSearchSpec.getFhirVersion())) {
737                        if (!thePackageSearchSpec.getFhirVersion().matches("([0-9]+\\.)+[0-9]+")) {
738                                FhirVersionEnum versionEnum = FhirVersionEnum.forVersionString(thePackageSearchSpec.getFhirVersion());
739                                if (versionEnum != null) {
740                                        predicates.add(theCb.equal(theRoot.get("myFhirVersion").as(FhirVersionEnum.class), versionEnum));
741                                }
742                        } else {
743                                predicates.add(theCb.like(theRoot.get("myFhirVersionId").as(String.class), thePackageSearchSpec.getFhirVersion() + "%"));
744                        }
745                }
746
747                return predicates;
748        }
749
750        @SuppressWarnings("unchecked")
751        public static List<String> getProcessingMessages(NpmPackage thePackage) {
752                return (List<String>) thePackage.getUserData().computeIfAbsent("JpPackageCache_ProcessingMessages", t -> new ArrayList<>());
753        }
754
755
756}