Source code

001/*
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2023 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 * http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.search;
021
022import ca.uhn.fhir.context.FhirContext;
023import ca.uhn.fhir.interceptor.api.HookParams;
024import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
025import ca.uhn.fhir.interceptor.api.Pointcut;
026import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
027import ca.uhn.fhir.interceptor.model.RequestPartitionId;
028import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
029import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
030import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
031import ca.uhn.fhir.jpa.api.svc.ISearchCoordinatorSvc;
032import ca.uhn.fhir.jpa.dao.HistoryBuilder;
033import ca.uhn.fhir.jpa.dao.HistoryBuilderFactory;
034import ca.uhn.fhir.jpa.dao.IJpaStorageResourceParser;
035import ca.uhn.fhir.jpa.dao.ISearchBuilder;
036import ca.uhn.fhir.jpa.dao.SearchBuilderFactory;
037import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
038import ca.uhn.fhir.jpa.entity.Search;
039import ca.uhn.fhir.jpa.entity.SearchTypeEnum;
040import ca.uhn.fhir.jpa.model.dao.JpaPid;
041import ca.uhn.fhir.jpa.model.entity.BaseHasResource;
042import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
043import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
044import ca.uhn.fhir.jpa.search.cache.ISearchCacheSvc;
045import ca.uhn.fhir.jpa.search.cache.SearchCacheStatusEnum;
046import ca.uhn.fhir.jpa.searchparam.SearchParameterMap;
047import ca.uhn.fhir.jpa.util.MemoryCacheService;
048import ca.uhn.fhir.jpa.util.QueryParameterUtils;
049import ca.uhn.fhir.model.api.Include;
050import ca.uhn.fhir.model.primitive.InstantDt;
051import ca.uhn.fhir.rest.api.server.IBundleProvider;
052import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails;
053import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails;
054import ca.uhn.fhir.rest.api.server.RequestDetails;
055import ca.uhn.fhir.rest.api.server.SimplePreResourceAccessDetails;
056import ca.uhn.fhir.rest.api.server.SimplePreResourceShowDetails;
057import ca.uhn.fhir.rest.server.interceptor.ServerInterceptorUtil;
058import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
059import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
060import com.google.common.annotations.VisibleForTesting;
061import org.hl7.fhir.instance.model.api.IBaseResource;
062import org.slf4j.Logger;
063import org.slf4j.LoggerFactory;
064import org.springframework.beans.factory.annotation.Autowired;
065
066import java.util.ArrayList;
067import java.util.Collections;
068import java.util.List;
069import java.util.Optional;
070import java.util.Set;
071import java.util.function.Function;
072import javax.annotation.Nonnull;
073import javax.persistence.EntityManager;
074import javax.persistence.PersistenceContext;
075
076public class PersistedJpaBundleProvider implements IBundleProvider {
077
078        private static final Logger ourLog = LoggerFactory.getLogger(PersistedJpaBundleProvider.class);
079
080        /*
081         * Autowired fields
082         */
083        protected final RequestDetails myRequest;
084
085        @Autowired
086        protected HapiTransactionService myTxService;
087
088        @PersistenceContext
089        private EntityManager myEntityManager;
090
091        @Autowired
092        private IInterceptorBroadcaster myInterceptorBroadcaster;
093
094        @Autowired
095        private SearchBuilderFactory<JpaPid> mySearchBuilderFactory;
096
097        @Autowired
098        private HistoryBuilderFactory myHistoryBuilderFactory;
099
100        @Autowired
101        private DaoRegistry myDaoRegistry;
102
103        @Autowired
104        private FhirContext myContext;
105
106        @Autowired
107        private ISearchCoordinatorSvc<JpaPid> mySearchCoordinatorSvc;
108
109        @Autowired
110        private ISearchCacheSvc mySearchCacheSvc;
111
112        @Autowired
113        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
114
115        @Autowired
116        private JpaStorageSettings myStorageSettings;
117
118        @Autowired
119        private MemoryCacheService myMemoryCacheService;
120
121        @Autowired
122        private IJpaStorageResourceParser myJpaStorageResourceParser;
123        /*
124         * Non autowired fields (will be different for every instance
125         * of this class, since it's a prototype
126         */
127        private Search mySearchEntity;
128        private String myUuid;
129        private SearchCacheStatusEnum myCacheStatus;
130        private RequestPartitionId myRequestPartitionId;
131        /**
132         * Constructor
133         */
134        public PersistedJpaBundleProvider(RequestDetails theRequest, String theSearchUuid) {
135                myRequest = theRequest;
136                myUuid = theSearchUuid;
137        }
138
139        /**
140         * Constructor
141         */
142        public PersistedJpaBundleProvider(RequestDetails theRequest, Search theSearch) {
143                myRequest = theRequest;
144                mySearchEntity = theSearch;
145                myUuid = theSearch.getUuid();
146        }
147
148        @VisibleForTesting
149        public void setRequestPartitionHelperSvcForUnitTest(IRequestPartitionHelperSvc theRequestPartitionHelperSvc) {
150                myRequestPartitionHelperSvc = theRequestPartitionHelperSvc;
151        }
152
153        protected Search getSearchEntity() {
154                return mySearchEntity;
155        }
156
157        // Note: Leave as protected, HSPC depends on this
158        @SuppressWarnings("WeakerAccess")
159        protected void setSearchEntity(Search theSearchEntity) {
160                mySearchEntity = theSearchEntity;
161        }
162
163        /**
164         * Perform a history search
165         */
166        private List<IBaseResource> doHistoryInTransaction(Integer theOffset, int theFromIndex, int theToIndex) {
167
168                HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
169                                mySearchEntity.getResourceType(),
170                                mySearchEntity.getResourceId(),
171                                mySearchEntity.getLastUpdatedLow(),
172                                mySearchEntity.getLastUpdatedHigh());
173
174                RequestPartitionId partitionId = getRequestPartitionId();
175                List<ResourceHistoryTable> results = historyBuilder.fetchEntities(
176                                partitionId, theOffset, theFromIndex, theToIndex, mySearchEntity.getHistorySearchStyle());
177
178                List<IBaseResource> retVal = new ArrayList<>();
179                for (ResourceHistoryTable next : results) {
180                        BaseHasResource resource;
181                        resource = next;
182
183                        IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(next.getResourceType());
184                        retVal.add(myJpaStorageResourceParser.toResource(resource, true));
185                }
186
187                // Interceptor call: STORAGE_PREACCESS_RESOURCES
188                {
189                        SimplePreResourceAccessDetails accessDetails = new SimplePreResourceAccessDetails(retVal);
190                        HookParams params = new HookParams()
191                                        .add(IPreResourceAccessDetails.class, accessDetails)
192                                        .add(RequestDetails.class, myRequest)
193                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
194                        CompositeInterceptorBroadcaster.doCallHooks(
195                                        myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PREACCESS_RESOURCES, params);
196
197                        for (int i = retVal.size() - 1; i >= 0; i--) {
198                                if (accessDetails.isDontReturnResourceAtIndex(i)) {
199                                        retVal.remove(i);
200                                }
201                        }
202                }
203
204                // Interceptor broadcast: STORAGE_PRESHOW_RESOURCES
205                {
206                        SimplePreResourceShowDetails showDetails = new SimplePreResourceShowDetails(retVal);
207                        HookParams params = new HookParams()
208                                        .add(IPreResourceShowDetails.class, showDetails)
209                                        .add(RequestDetails.class, myRequest)
210                                        .addIfMatchesType(ServletRequestDetails.class, myRequest);
211                        CompositeInterceptorBroadcaster.doCallHooks(
212                                        myInterceptorBroadcaster, myRequest, Pointcut.STORAGE_PRESHOW_RESOURCES, params);
213                        retVal = showDetails.toList();
214                }
215
216                return retVal;
217        }
218
219        @Nonnull
220        protected final RequestPartitionId getRequestPartitionId() {
221                if (myRequestPartitionId == null) {
222                        ReadPartitionIdRequestDetails details;
223                        if (mySearchEntity == null) {
224                                details = ReadPartitionIdRequestDetails.forSearchUuid(myUuid);
225                        } else if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
226                                details = ReadPartitionIdRequestDetails.forHistory(mySearchEntity.getResourceType(), null);
227                        } else {
228                                SearchParameterMap params =
229                                                mySearchEntity.getSearchParameterMap().orElse(null);
230                                details = ReadPartitionIdRequestDetails.forSearchType(mySearchEntity.getResourceType(), params, null);
231                        }
232                        myRequestPartitionId = myRequestPartitionHelperSvc.determineReadPartitionForRequest(myRequest, details);
233                }
234                return myRequestPartitionId;
235        }
236
237        public void setRequestPartitionId(RequestPartitionId theRequestPartitionId) {
238                myRequestPartitionId = theRequestPartitionId;
239        }
240
241        protected List<IBaseResource> doSearchOrEverything(final int theFromIndex, final int theToIndex) {
242                if (mySearchEntity.getTotalCount() != null && mySearchEntity.getNumFound() <= 0) {
243                        // No resources to fetch (e.g. we did a _summary=count search)
244                        return Collections.emptyList();
245                }
246                String resourceName = mySearchEntity.getResourceType();
247                Class<? extends IBaseResource> resourceType =
248                                myContext.getResourceDefinition(resourceName).getImplementingClass();
249                IFhirResourceDao<?> dao = myDaoRegistry.getResourceDao(resourceName);
250
251                final ISearchBuilder sb = mySearchBuilderFactory.newSearchBuilder(dao, resourceName, resourceType);
252
253                RequestPartitionId requestPartitionId = getRequestPartitionId();
254                final List<JpaPid> pidsSubList =
255                                mySearchCoordinatorSvc.getResources(myUuid, theFromIndex, theToIndex, myRequest, requestPartitionId);
256                return myTxService
257                                .withRequest(myRequest)
258                                .withRequestPartitionId(requestPartitionId)
259                                .execute(() -> {
260                                        return toResourceList(sb, pidsSubList);
261                                });
262        }
263
264        /**
265         * Returns false if the entity can't be found
266         */
267        public boolean ensureSearchEntityLoaded() {
268                if (mySearchEntity == null) {
269                        Optional<Search> searchOpt = myTxService
270                                        .withRequest(myRequest)
271                                        .withRequestPartitionId(myRequestPartitionId)
272                                        .execute(() -> mySearchCacheSvc.fetchByUuid(myUuid, myRequestPartitionId));
273                        if (!searchOpt.isPresent()) {
274                                return false;
275                        }
276
277                        setSearchEntity(searchOpt.get());
278
279                        ourLog.trace(
280                                        "Retrieved search with version {} and total {}",
281                                        mySearchEntity.getVersion(),
282                                        mySearchEntity.getTotalCount());
283
284                        return true;
285                }
286
287                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
288                        if (mySearchEntity.getTotalCount() == null) {
289                                calculateHistoryCount();
290                        }
291                }
292
293                return true;
294        }
295
296        /**
297         * Note that this method is called outside a DB transaction, and uses a loading cache
298         * (assuming the default {@literal COUNT_CACHED} mode) so this effectively throttles
299         * access to the database by preventing multiple concurrent DB calls for an expensive
300         * count operation.
301         */
302        private void calculateHistoryCount() {
303                MemoryCacheService.HistoryCountKey key;
304                if (mySearchEntity.getResourceId() != null) {
305                        key = MemoryCacheService.HistoryCountKey.forInstance(mySearchEntity.getResourceId());
306                } else if (mySearchEntity.getResourceType() != null) {
307                        key = MemoryCacheService.HistoryCountKey.forType(mySearchEntity.getResourceType());
308                } else {
309                        key = MemoryCacheService.HistoryCountKey.forSystem();
310                }
311
312                Function<MemoryCacheService.HistoryCountKey, Integer> supplier = k -> myTxService
313                                .withRequest(myRequest)
314                                .withRequestPartitionId(getRequestPartitionId())
315                                .execute(() -> {
316                                        HistoryBuilder historyBuilder = myHistoryBuilderFactory.newHistoryBuilder(
317                                                        mySearchEntity.getResourceType(),
318                                                        mySearchEntity.getResourceId(),
319                                                        mySearchEntity.getLastUpdatedLow(),
320                                                        mySearchEntity.getLastUpdatedHigh());
321                                        Long count = historyBuilder.fetchCount(getRequestPartitionId());
322                                        return count.intValue();
323                                });
324
325                boolean haveOffset = mySearchEntity.getLastUpdatedLow() != null || mySearchEntity.getLastUpdatedHigh() != null;
326
327                switch (myStorageSettings.getHistoryCountMode()) {
328                        case COUNT_ACCURATE: {
329                                int count = supplier.apply(key);
330                                mySearchEntity.setTotalCount(count);
331                                break;
332                        }
333                        case CACHED_ONLY_WITHOUT_OFFSET: {
334                                if (!haveOffset) {
335                                        int count = myMemoryCacheService.get(MemoryCacheService.CacheEnum.HISTORY_COUNT, key, supplier);
336                                        mySearchEntity.setTotalCount(count);
337                                }
338                                break;
339                        }
340                        case COUNT_DISABLED: {
341                                break;
342                        }
343                }
344        }
345
346        @Override
347        public InstantDt getPublished() {
348                ensureSearchEntityLoaded();
349                return new InstantDt(mySearchEntity.getCreated());
350        }
351
352        @Nonnull
353        @Override
354        public List<IBaseResource> getResources(final int theFromIndex, final int theToIndex) {
355                boolean entityLoaded = ensureSearchEntityLoaded();
356                assert entityLoaded;
357                assert mySearchEntity != null;
358                assert mySearchEntity.getSearchType() != null;
359
360                switch (mySearchEntity.getSearchType()) {
361                        case HISTORY:
362                                return myTxService
363                                                .withRequest(myRequest)
364                                                .withRequestPartitionId(getRequestPartitionId())
365                                                .execute(() -> doHistoryInTransaction(mySearchEntity.getOffset(), theFromIndex, theToIndex));
366                        case SEARCH:
367                        case EVERYTHING:
368                        default:
369                                List<IBaseResource> retVal = doSearchOrEverything(theFromIndex, theToIndex);
370                                /*
371                                 * If we got fewer resources back than we asked for, it's possible that the search
372                                 * completed. If that's the case, the cached version of the search entity is probably
373                                 * no longer valid so let's force a reload if it gets asked for again (most likely
374                                 * because someone is calling size() on us)
375                                 */
376                                if (retVal.size() < theToIndex - theFromIndex) {
377                                        mySearchEntity = null;
378                                }
379                                return retVal;
380                }
381        }
382
383        @Override
384        public String getUuid() {
385                return myUuid;
386        }
387
388        public SearchCacheStatusEnum getCacheStatus() {
389                return myCacheStatus;
390        }
391
392        void setCacheStatus(SearchCacheStatusEnum theSearchCacheStatusEnum) {
393                myCacheStatus = theSearchCacheStatusEnum;
394        }
395
396        @Override
397        public Integer preferredPageSize() {
398                ensureSearchEntityLoaded();
399                return mySearchEntity.getPreferredPageSize();
400        }
401
402        public void setContext(FhirContext theContext) {
403                myContext = theContext;
404        }
405
406        public void setEntityManager(EntityManager theEntityManager) {
407                myEntityManager = theEntityManager;
408        }
409
410        @VisibleForTesting
411        public void setSearchCoordinatorSvcForUnitTest(ISearchCoordinatorSvc theSearchCoordinatorSvc) {
412                mySearchCoordinatorSvc = theSearchCoordinatorSvc;
413        }
414
415        @VisibleForTesting
416        public void setTxServiceForUnitTest(HapiTransactionService theTxManager) {
417                myTxService = theTxManager;
418        }
419
420        @Override
421        public Integer size() {
422                ensureSearchEntityLoaded();
423                QueryParameterUtils.verifySearchHasntFailedOrThrowInternalErrorException(mySearchEntity);
424
425                Integer size = mySearchEntity.getTotalCount();
426                if (size != null) {
427                        return Math.max(0, size);
428                }
429
430                if (mySearchEntity.getSearchType() == SearchTypeEnum.HISTORY) {
431                        return null;
432                } else {
433                        return mySearchCoordinatorSvc
434                                        .getSearchTotal(myUuid, myRequest, myRequestPartitionId)
435                                        .orElse(null);
436                }
437        }
438
439        protected boolean hasIncludes() {
440                ensureSearchEntityLoaded();
441                return !mySearchEntity.getIncludes().isEmpty();
442        }
443
444        // Note: Leave as protected, HSPC depends on this
445        @SuppressWarnings("WeakerAccess")
446        protected List<IBaseResource> toResourceList(ISearchBuilder theSearchBuilder, List<JpaPid> thePids) {
447
448                List<JpaPid> includedPidList = new ArrayList<>();
449                if (mySearchEntity.getSearchType() == SearchTypeEnum.SEARCH) {
450                        Integer maxIncludes = myStorageSettings.getMaximumIncludesToLoadPerPage();
451
452                        // Decide whether to perform include or revincludes first based on which one has iterate.
453                        boolean performIncludesBeforeRevincludes = shouldPerformIncludesBeforeRevincudes();
454
455                        if (performIncludesBeforeRevincludes) {
456                                // Load _includes
457                                Set<JpaPid> includedPids = theSearchBuilder.loadIncludes(
458                                                myContext,
459                                                myEntityManager,
460                                                thePids,
461                                                mySearchEntity.toIncludesList(),
462                                                false,
463                                                mySearchEntity.getLastUpdated(),
464                                                myUuid,
465                                                myRequest,
466                                                maxIncludes);
467                                if (maxIncludes != null) {
468                                        maxIncludes -= includedPids.size();
469                                }
470                                thePids.addAll(includedPids);
471                                includedPidList.addAll(includedPids);
472
473                                // Load _revincludes
474                                Set<JpaPid> revIncludedPids = theSearchBuilder.loadIncludes(
475                                                myContext,
476                                                myEntityManager,
477                                                thePids,
478                                                mySearchEntity.toRevIncludesList(),
479                                                true,
480                                                mySearchEntity.getLastUpdated(),
481                                                myUuid,
482                                                myRequest,
483                                                maxIncludes);
484                                thePids.addAll(revIncludedPids);
485                                includedPidList.addAll(revIncludedPids);
486                        } else {
487                                // Load _revincludes
488                                Set<JpaPid> revIncludedPids = theSearchBuilder.loadIncludes(
489                                                myContext,
490                                                myEntityManager,
491                                                thePids,
492                                                mySearchEntity.toRevIncludesList(),
493                                                true,
494                                                mySearchEntity.getLastUpdated(),
495                                                myUuid,
496                                                myRequest,
497                                                maxIncludes);
498                                if (maxIncludes != null) {
499                                        maxIncludes -= revIncludedPids.size();
500                                }
501                                thePids.addAll(revIncludedPids);
502                                includedPidList.addAll(revIncludedPids);
503
504                                // Load _includes
505                                Set<JpaPid> includedPids = theSearchBuilder.loadIncludes(
506                                                myContext,
507                                                myEntityManager,
508                                                thePids,
509                                                mySearchEntity.toIncludesList(),
510                                                false,
511                                                mySearchEntity.getLastUpdated(),
512                                                myUuid,
513                                                myRequest,
514                                                maxIncludes);
515                                thePids.addAll(includedPids);
516                                includedPidList.addAll(includedPids);
517                        }
518                }
519
520                // Execute the query and make sure we return distinct results
521                List<IBaseResource> resources = new ArrayList<>();
522                theSearchBuilder.loadResourcesByPid(thePids, includedPidList, resources, false, myRequest);
523
524                resources = ServerInterceptorUtil.fireStoragePreshowResource(resources, myRequest, myInterceptorBroadcaster);
525
526                return resources;
527        }
528
529        private boolean shouldPerformIncludesBeforeRevincudes() {
530                // When revincludes contain a :iterate, we should perform them last so they can iterate through the includes
531                // found so far
532                boolean retval = false;
533
534                for (Include nextInclude : mySearchEntity.toRevIncludesList()) {
535                        if (nextInclude.isRecurse()) {
536                                retval = true;
537                                break;
538                        }
539                }
540                return retval;
541        }
542
543        public void setInterceptorBroadcaster(IInterceptorBroadcaster theInterceptorBroadcaster) {
544                myInterceptorBroadcaster = theInterceptorBroadcaster;
545        }
546
547        @VisibleForTesting
548        public void setSearchCacheSvcForUnitTest(ISearchCacheSvc theSearchCacheSvc) {
549                mySearchCacheSvc = theSearchCacheSvc;
550        }
551
552        @VisibleForTesting
553        public void setDaoRegistryForUnitTest(DaoRegistry theDaoRegistry) {
554                myDaoRegistry = theDaoRegistry;
555        }
556
557        @VisibleForTesting
558        public void setStorageSettingsForUnitTest(JpaStorageSettings theStorageSettings) {
559                myStorageSettings = theStorageSettings;
560        }
561
562        @VisibleForTesting
563        public void setSearchBuilderFactoryForUnitTest(SearchBuilderFactory theSearchBuilderFactory) {
564                mySearchBuilderFactory = theSearchBuilderFactory;
565        }
566}