Source code

001/*-
002 * #%L
003 * HAPI FHIR JPA Server
004 * %%
005 * Copyright (C) 2014 - 2023 Smile CDR, Inc.
006 * %%
007 * Licensed under the Apache License, Version 2.0 (the "License");
008 * you may not use this file except in compliance with the License.
009 * You may obtain a copy of the License at
010 *
011 *      http://www.apache.org/licenses/LICENSE-2.0
012 *
013 * Unless required by applicable law or agreed to in writing, software
014 * distributed under the License is distributed on an "AS IS" BASIS,
015 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
016 * See the License for the specific language governing permissions and
017 * limitations under the License.
018 * #L%
019 */
020package ca.uhn.fhir.jpa.dao.expunge;
021
022import ca.uhn.fhir.interceptor.api.HookParams;
023import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster;
024import ca.uhn.fhir.interceptor.api.Pointcut;
025import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails;
026import ca.uhn.fhir.interceptor.model.RequestPartitionId;
027import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService;
028import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity;
029import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity;
030import ca.uhn.fhir.jpa.entity.BulkImportJobEntity;
031import ca.uhn.fhir.jpa.entity.BulkImportJobFileEntity;
032import ca.uhn.fhir.jpa.entity.MdmLink;
033import ca.uhn.fhir.jpa.entity.PartitionEntity;
034import ca.uhn.fhir.jpa.entity.Search;
035import ca.uhn.fhir.jpa.entity.SearchInclude;
036import ca.uhn.fhir.jpa.entity.SearchResult;
037import ca.uhn.fhir.jpa.entity.SubscriptionTable;
038import ca.uhn.fhir.jpa.entity.TermCodeSystem;
039import ca.uhn.fhir.jpa.entity.TermCodeSystemVersion;
040import ca.uhn.fhir.jpa.entity.TermConcept;
041import ca.uhn.fhir.jpa.entity.TermConceptDesignation;
042import ca.uhn.fhir.jpa.entity.TermConceptMap;
043import ca.uhn.fhir.jpa.entity.TermConceptMapGroup;
044import ca.uhn.fhir.jpa.entity.TermConceptMapGroupElement;
045import ca.uhn.fhir.jpa.entity.TermConceptMapGroupElementTarget;
046import ca.uhn.fhir.jpa.entity.TermConceptParentChildLink;
047import ca.uhn.fhir.jpa.entity.TermConceptProperty;
048import ca.uhn.fhir.jpa.entity.TermValueSet;
049import ca.uhn.fhir.jpa.entity.TermValueSetConcept;
050import ca.uhn.fhir.jpa.entity.TermValueSetConceptDesignation;
051import ca.uhn.fhir.jpa.model.entity.ForcedId;
052import ca.uhn.fhir.jpa.model.entity.NpmPackageEntity;
053import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionEntity;
054import ca.uhn.fhir.jpa.model.entity.NpmPackageVersionResourceEntity;
055import ca.uhn.fhir.jpa.model.entity.ResourceHistoryProvenanceEntity;
056import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTable;
057import ca.uhn.fhir.jpa.model.entity.ResourceHistoryTag;
058import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboStringUnique;
059import ca.uhn.fhir.jpa.model.entity.ResourceIndexedComboTokenNonUnique;
060import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamCoords;
061import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamDate;
062import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamNumber;
063import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantity;
064import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamQuantityNormalized;
065import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamString;
066import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken;
067import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamUri;
068import ca.uhn.fhir.jpa.model.entity.ResourceLink;
069import ca.uhn.fhir.jpa.model.entity.ResourceSearchUrlEntity;
070import ca.uhn.fhir.jpa.model.entity.ResourceTable;
071import ca.uhn.fhir.jpa.model.entity.ResourceTag;
072import ca.uhn.fhir.jpa.model.entity.SearchParamPresentEntity;
073import ca.uhn.fhir.jpa.model.entity.TagDefinition;
074import ca.uhn.fhir.jpa.partition.IRequestPartitionHelperSvc;
075import ca.uhn.fhir.jpa.util.MemoryCacheService;
076import ca.uhn.fhir.rest.api.server.RequestDetails;
077import ca.uhn.fhir.rest.server.provider.ProviderConstants;
078import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
079import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster;
080import ca.uhn.fhir.util.StopWatch;
081import org.slf4j.Logger;
082import org.slf4j.LoggerFactory;
083import org.springframework.beans.factory.annotation.Autowired;
084import org.springframework.stereotype.Service;
085import org.springframework.transaction.annotation.Propagation;
086
087import java.util.List;
088import java.util.concurrent.atomic.AtomicInteger;
089import javax.annotation.Nullable;
090import javax.persistence.EntityManager;
091import javax.persistence.PersistenceContext;
092import javax.persistence.PersistenceContextType;
093import javax.persistence.TypedQuery;
094import javax.persistence.criteria.CriteriaBuilder;
095import javax.persistence.criteria.CriteriaQuery;
096
097@Service
098public class ExpungeEverythingService implements IExpungeEverythingService {
099        private static final Logger ourLog = LoggerFactory.getLogger(ExpungeEverythingService.class);
100
101        @PersistenceContext(type = PersistenceContextType.TRANSACTION)
102        protected EntityManager myEntityManager;
103
104        @Autowired
105        protected IInterceptorBroadcaster myInterceptorBroadcaster;
106
107        @Autowired
108        private HapiTransactionService myTxService;
109
110        @Autowired
111        private MemoryCacheService myMemoryCacheService;
112
113        @Autowired
114        private IRequestPartitionHelperSvc myRequestPartitionHelperSvc;
115
116        private int deletedResourceEntityCount;
117
118        @Override
119        public void expungeEverything(@Nullable RequestDetails theRequest) {
120
121                final AtomicInteger counter = new AtomicInteger();
122
123                // Notify Interceptors about pre-action call
124                HookParams hooks = new HookParams()
125                                .add(AtomicInteger.class, counter)
126                                .add(RequestDetails.class, theRequest)
127                                .addIfMatchesType(ServletRequestDetails.class, theRequest);
128                CompositeInterceptorBroadcaster.doCallHooks(
129                                myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_PRESTORAGE_EXPUNGE_EVERYTHING, hooks);
130
131                ourLog.info("BEGINNING GLOBAL $expunge");
132                Propagation propagation = Propagation.REQUIRES_NEW;
133                ReadPartitionIdRequestDetails details =
134                                ReadPartitionIdRequestDetails.forOperation(null, null, ProviderConstants.OPERATION_EXPUNGE);
135                RequestPartitionId requestPartitionId =
136                                myRequestPartitionHelperSvc.determineReadPartitionForRequest(theRequest, details);
137
138                myTxService
139                                .withRequest(theRequest)
140                                .withPropagation(propagation)
141                                .withRequestPartitionId(requestPartitionId)
142                                .execute(() -> {
143                                        counter.addAndGet(doExpungeEverythingQuery(
144                                                        "UPDATE " + TermCodeSystem.class.getSimpleName() + " d SET d.myCurrentVersion = null"));
145                                });
146                counter.addAndGet(
147                                expungeEverythingByTypeWithoutPurging(theRequest, Batch2WorkChunkEntity.class, requestPartitionId));
148                counter.addAndGet(
149                                expungeEverythingByTypeWithoutPurging(theRequest, Batch2JobInstanceEntity.class, requestPartitionId));
150                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
151                                theRequest, NpmPackageVersionResourceEntity.class, requestPartitionId));
152                counter.addAndGet(
153                                expungeEverythingByTypeWithoutPurging(theRequest, NpmPackageVersionEntity.class, requestPartitionId));
154                counter.addAndGet(
155                                expungeEverythingByTypeWithoutPurging(theRequest, NpmPackageEntity.class, requestPartitionId));
156                counter.addAndGet(
157                                expungeEverythingByTypeWithoutPurging(theRequest, SearchParamPresentEntity.class, requestPartitionId));
158                counter.addAndGet(
159                                expungeEverythingByTypeWithoutPurging(theRequest, BulkImportJobFileEntity.class, requestPartitionId));
160                counter.addAndGet(
161                                expungeEverythingByTypeWithoutPurging(theRequest, BulkImportJobEntity.class, requestPartitionId));
162                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ForcedId.class, requestPartitionId));
163                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
164                                theRequest, ResourceIndexedSearchParamDate.class, requestPartitionId));
165                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
166                                theRequest, ResourceIndexedSearchParamNumber.class, requestPartitionId));
167                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
168                                theRequest, ResourceIndexedSearchParamQuantity.class, requestPartitionId));
169                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
170                                theRequest, ResourceIndexedSearchParamQuantityNormalized.class, requestPartitionId));
171                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
172                                theRequest, ResourceIndexedSearchParamString.class, requestPartitionId));
173                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
174                                theRequest, ResourceIndexedSearchParamToken.class, requestPartitionId));
175                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
176                                theRequest, ResourceIndexedSearchParamUri.class, requestPartitionId));
177                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
178                                theRequest, ResourceIndexedSearchParamCoords.class, requestPartitionId));
179                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
180                                theRequest, ResourceIndexedComboStringUnique.class, requestPartitionId));
181                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
182                                theRequest, ResourceIndexedComboTokenNonUnique.class, requestPartitionId));
183                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceLink.class, requestPartitionId));
184                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, SearchResult.class, requestPartitionId));
185                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, SearchInclude.class, requestPartitionId));
186                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
187                                theRequest, TermValueSetConceptDesignation.class, requestPartitionId));
188                counter.addAndGet(
189                                expungeEverythingByTypeWithoutPurging(theRequest, TermValueSetConcept.class, requestPartitionId));
190                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermValueSet.class, requestPartitionId));
191                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
192                                theRequest, TermConceptParentChildLink.class, requestPartitionId));
193                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
194                                theRequest, TermConceptMapGroupElementTarget.class, requestPartitionId));
195                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
196                                theRequest, TermConceptMapGroupElement.class, requestPartitionId));
197                counter.addAndGet(
198                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptMapGroup.class, requestPartitionId));
199                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermConceptMap.class, requestPartitionId));
200                counter.addAndGet(
201                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptProperty.class, requestPartitionId));
202                counter.addAndGet(
203                                expungeEverythingByTypeWithoutPurging(theRequest, TermConceptDesignation.class, requestPartitionId));
204                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermConcept.class, requestPartitionId));
205                myTxService
206                                .withRequest(theRequest)
207                                .withPropagation(propagation)
208                                .withRequestPartitionId(requestPartitionId)
209                                .execute(() -> {
210                                        for (TermCodeSystem next : myEntityManager
211                                                        .createQuery("SELECT c FROM " + TermCodeSystem.class.getName() + " c", TermCodeSystem.class)
212                                                        .getResultList()) {
213                                                next.setCurrentVersion(null);
214                                                myEntityManager.merge(next);
215                                        }
216                                });
217                counter.addAndGet(
218                                expungeEverythingByTypeWithoutPurging(theRequest, TermCodeSystemVersion.class, requestPartitionId));
219                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TermCodeSystem.class, requestPartitionId));
220                counter.addAndGet(
221                                expungeEverythingByTypeWithoutPurging(theRequest, SubscriptionTable.class, requestPartitionId));
222                counter.addAndGet(
223                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceHistoryTag.class, requestPartitionId));
224                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceTag.class, requestPartitionId));
225                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, TagDefinition.class, requestPartitionId));
226                counter.addAndGet(expungeEverythingByTypeWithoutPurging(
227                                theRequest, ResourceHistoryProvenanceEntity.class, requestPartitionId));
228                counter.addAndGet(
229                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceHistoryTable.class, requestPartitionId));
230                counter.addAndGet(
231                                expungeEverythingByTypeWithoutPurging(theRequest, ResourceSearchUrlEntity.class, requestPartitionId));
232                int counterBefore = counter.get();
233                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, ResourceTable.class, requestPartitionId));
234                counter.addAndGet(expungeEverythingByTypeWithoutPurging(theRequest, PartitionEntity.class, requestPartitionId));
235
236                deletedResourceEntityCount = counter.get() - counterBefore;
237
238                myTxService
239                                .withRequest(theRequest)
240                                .withPropagation(propagation)
241                                .withRequestPartitionId(requestPartitionId)
242                                .execute(() -> {
243                                        counter.addAndGet(doExpungeEverythingQuery("DELETE from " + Search.class.getSimpleName() + " d"));
244                                });
245
246                purgeAllCaches();
247
248                ourLog.info("COMPLETED GLOBAL $expunge - Deleted {} rows", counter.get());
249        }
250
251        @Override
252        public int getExpungeDeletedEntityCount() {
253                return deletedResourceEntityCount;
254        }
255
256        private void purgeAllCaches() {
257                myMemoryCacheService.invalidateAllCaches();
258        }
259
260        private int expungeEverythingByTypeWithoutPurging(
261                        RequestDetails theRequest, Class<?> theEntityType, RequestPartitionId theRequestPartitionId) {
262                int outcome = 0;
263                while (true) {
264                        StopWatch sw = new StopWatch();
265
266                        int count = myTxService
267                                        .withRequest(theRequest)
268                                        .withPropagation(Propagation.REQUIRES_NEW)
269                                        .withRequestPartitionId(theRequestPartitionId)
270                                        .execute(() -> {
271                                                CriteriaBuilder cb = myEntityManager.getCriteriaBuilder();
272                                                CriteriaQuery<?> cq = cb.createQuery(theEntityType);
273                                                cq.from(theEntityType);
274                                                TypedQuery<?> query = myEntityManager.createQuery(cq);
275                                                query.setMaxResults(1000);
276                                                List<?> results = query.getResultList();
277                                                for (Object result : results) {
278                                                        myEntityManager.remove(result);
279                                                }
280                                                return results.size();
281                                        });
282
283                        outcome += count;
284                        if (count == 0) {
285                                break;
286                        }
287
288                        ourLog.info("Have deleted {} entities of type {} in {}", outcome, theEntityType.getSimpleName(), sw);
289                }
290                return outcome;
291        }
292
293        @Override
294        public int expungeEverythingByType(Class<?> theEntityType) {
295                int result = expungeEverythingByTypeWithoutPurging(null, theEntityType, RequestPartitionId.allPartitions());
296                purgeAllCaches();
297                return result;
298        }
299
300        @Override
301        public int expungeEverythingMdmLinks() {
302                return expungeEverythingByType(MdmLink.class);
303        }
304
305        private int doExpungeEverythingQuery(String theQuery) {
306                StopWatch sw = new StopWatch();
307                int outcome = myEntityManager.createQuery(theQuery).executeUpdate();
308                ourLog.debug("SqlQuery affected {} rows in {}: {}", outcome, sw, theQuery);
309                return outcome;
310        }
311}