package com.atlassian.bitbucket.mesh.git.gc;

import com.atlassian.bitbucket.mesh.AbstractWiredExecutionContextTest;
import com.atlassian.bitbucket.mesh.config.ApplicationConfig;
import com.atlassian.bitbucket.mesh.config.GcConfig;
import com.atlassian.bitbucket.mesh.git.GitAgent;
import com.atlassian.bitbucket.mesh.git.GitRepositoryLayout;
import com.atlassian.bitbucket.mesh.git.builder.GitProcess;
import com.atlassian.bitbucket.mesh.git.builder.GitProcessBuilder;
import com.atlassian.bitbucket.mesh.git.builder.GitProcessBuilderFactory;
import com.atlassian.bitbucket.mesh.git.builder.fetch.GitFetchTagMode;
import com.atlassian.bitbucket.mesh.git.builder.foreachref.AllRefsForEachRefStdioHandler;
import com.atlassian.bitbucket.mesh.git.builder.packrefs.GitPackRefsBuilder;
import com.atlassian.bitbucket.mesh.git.builder.repack.GitRepackBuilder;
import com.atlassian.bitbucket.mesh.git.exception.CommandFailedException;
import com.atlassian.bitbucket.mesh.git.exception.CommandTimeoutException;
import com.atlassian.bitbucket.mesh.io.MoreFiles;
import com.atlassian.bitbucket.mesh.junit.LogListener;
import com.atlassian.bitbucket.mesh.layout.StorageLayout;
import com.atlassian.bitbucket.mesh.process.ProcessSummary;
import com.atlassian.bitbucket.mesh.process.handler.LoggingStdioHandler;
import com.atlassian.bitbucket.mesh.process.handler.NullStdioHandler;
import com.atlassian.bitbucket.mesh.repository.*;
import com.atlassian.bitbucket.mesh.test.annotation.DirtiesRepo;
import com.atlassian.bitbucket.mesh.util.Version;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.ByteStreams;
import io.grpc.Status;
import org.apache.commons.lang3.mutable.MutableInt;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mock;
import org.mockito.Spy;
import org.springframework.core.io.ClassPathResource;

import javax.annotation.Nonnull;
import javax.annotation.Resource;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.util.EnumSet;
import java.util.LinkedList;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;

import static com.atlassian.bitbucket.mesh.git.GitConstants.*;
import static com.atlassian.bitbucket.mesh.git.gc.DefaultGarbageCollectionManager.GC_LOG;
import static com.atlassian.bitbucket.mesh.git.gc.PackRefsOperation.PACKREFS_TIMESTAMP;
import static com.atlassian.bitbucket.mesh.git.gc.RepackOperation.REPACK_TIMESTAMP;
import static com.atlassian.bitbucket.mesh.git.gc.VacuumOperation.VACUUM_FULL_TIMESTAMP;
import static com.atlassian.bitbucket.mesh.git.gc.VacuumOperation.VACUUM_TIMESTAMP;
import static java.util.Collections.singleton;
import static org.apache.commons.lang3.StringUtils.trim;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.same;
import static org.mockito.Mockito.*;

public abstract class BaseGarbageCollectionManagerWiredTest extends AbstractWiredExecutionContextTest {

    static final String COMMIT_INITIAL = "d36984a542c2cddb3c29797a11aec1199c7d75d2";
    static final int MAX_ATTEMPTS = 3;
    static final Pattern MTIMES_REGEX = Pattern.compile("\\.mtimes$");
    static final Pattern PATTERN_LOOSE_OBJECT = Pattern.compile("[0-9a-f]{38}");
    static final Pattern PATTERN_OBJECT_DIR = Pattern.compile("[0-9a-f]{2}");

    @Rule
    public final LogListener logListener = new LogListener(DefaultGarbageCollectionManager.class);
    @Rule
    public final TemporaryFolder temporaryFolder = new TemporaryFolder();

    @Resource
    protected GitAgent agent;
    @Resource
    @Spy
    protected GitProcessBuilderFactory builderFactory;
    @Resource
    protected ApplicationConfig config;
    @Mock
    protected ScheduledExecutorService executorService;
    protected DefaultGarbageCollectionManager garbageCollectionManager;
    protected GcConfig gcConfig;
    protected Repository repository;
    @Resource
    protected GitRepositoryLayout repositoryLayout;
    @Resource
    protected RepositoryManager repositoryManager;
    @Resource
    protected StorageLayout storageLayout;

    @Before
    public void setup() {
        // Immediate execute of the task for easier testing. In some cases, a gc task is scheduled while holding the
        // gc lock for the repository. To avoid issues, we'll mock an executor service that serializes all scheduled
        // tasks and executes them as soon as possible.
        AtomicBoolean running = new AtomicBoolean();
        LinkedList<Runnable> taskQueue = new LinkedList<>();
        when(executorService.schedule(any(Runnable.class), anyLong(), any(TimeUnit.class))).thenAnswer(invocation -> {
            Runnable task = invocation.getArgument(0);
            if (running.compareAndSet(false, true)) {
                //This is the outermost gc task; execute it immediately
                task.run();

                //Run any child tasks that were scheduled by the task
                while (!taskQueue.isEmpty()) {
                    task = taskQueue.pop();
                    task.run();
                }
                running.set(false);
            } else {
                //If any child tasks are scheduled, defer them until the outermost gc task completes to avoid gc.lock
                //issues
                taskQueue.push(task);
            }

            return mock(ScheduledFuture.class);
        });

        gcConfig = spy(config.getGit().getGc());
        garbageCollectionManager = new DefaultGarbageCollectionManager(gcConfig, builderFactory,
                executorService, repositoryLayout);
        repository = getRepositoryUnderTest();
    }

    @After
    public void tearDown() {
        garbageCollectionManager.shutdown();
    }

    @DirtiesRepo
    @Test
    public void testGcNotRetriedIfRepositoryDeleted() {
        Repository repositorySpy = spy(repository);
        when(repositorySpy.exists()).thenReturn(false);

        GitProcess<Void> process = mockPackRefsProcess(repositorySpy);
        when(process.run()).thenThrow(new CommandFailedException("testing failure",
                new ProcessSummary.Builder("cmdline")
                        .exitCode(128)
                        .build()));

        // Create some loose objects to ensure that GC is attempted
        createLooseObjects(repository);

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repositorySpy)
                .added(1)
                .build());

        // Verify that the command is not attempted
        verify(process, times(0)).run();
    }

    @DirtiesRepo
    @Test
    public void testGcNotRetriedIfRepositoryDeletedAfterFirstAttempt() {
        Repository repositorySpy = spy(repository);
        //Return true on first run, then false on the retry attempt
        when(repositorySpy.exists()).thenReturn(true, false);

        GitProcess<Void> process = mockPackRefsProcess(repositorySpy);
        when(process.run()).thenThrow(new CommandFailedException("testing failure",
                new ProcessSummary.Builder("cmdline")
                        .exitCode(128)
                        .build()));

        // Create some loose objects to ensure that GC is attempted
        createLooseObjects(repository);

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repositorySpy)
                .added(1)
                .build());

        // Verify that the command is not attempted again
        verify(process, times(1)).run();
    }

    @DirtiesRepo
    @Test
    public void testGcNotRetriedOnCommandTimeout() {
        GitProcess<Void> process = mockPackRefsProcess(repository);
        when(process.run())
                .thenThrow(new CommandTimeoutException(new ProcessSummary.Builder("git pack-refs").exitCode(-1).build()));

        // Create some loose objects to ensure that GC is attempted
        createLooseObjects(repository);

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );

        // Verify that the command is not retried
        verify(process, times(1)).run();
        logListener.assertWarn("Garbage collection timed out");
    }

    @DirtiesRepo
    @Test
    public void testGcNotRetriedOnGrpcTimeout() {
        GitProcess<Void> process = mockPackRefsProcess(repository);
        when(process.run()).thenThrow(Status.DEADLINE_EXCEEDED
                .withDescription("[git pack-refs] timed out")
                .asRuntimeException());

        // Create some loose objects to ensure that GC is attempted
        createLooseObjects(repository);

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );

        // Verify that the command is not retried
        verify(process, times(1)).run();
        logListener.assertWarn("Garbage collection timed out");
    }

    @Test
    public void testGcNotRetriedWhenRetryIsNoOp() {
        GcOperation operation = new GcOperation() {

            private boolean shouldRun = true;

            @Override
            public void run(@Nonnull GcContext context) {
                shouldRun = false;
                throw new RuntimeException("testing failure");
            }

            @Override
            public boolean shouldRun() {
                return shouldRun;
            }
        };

        garbageCollectionManager.maybeRun(repository, operation);

        //Verify that the operation has been called
        assertFalse(operation.shouldRun());
        logListener.assertWarn("Garbage collection completed with an error and will not be retried.");
    }

    @DirtiesRepo
    @Test
    public void testGcRetriedOnPackRefsError() {
        // Mock that the pack-refs command fails
        GitProcess<Void> process = mockPackRefsProcess(repository);
        when(process.run()).thenThrow(CommandFailedException.class);

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );

        verify(process, times(MAX_ATTEMPTS)).run();
        logListener.assertWarn("Abandoning garbage collection after 3 failed attempts");
    }

    @DirtiesRepo
    @Test
    public void testGcSkippedWhenLockHeld() throws IOException {
        // trigger creation of the app-info directory
        repositoryLayout.getApplicationInfoDir(repository);
        repositoryLayout.editFile(repository, GC_LOG, (reader, writer) -> {
            // we hold the GC lock, so attempts to acquire the lock will fail

            // create loose objects to ensure GC will run
            createLooseObjects(repository);

            collectGarbage(repository);

            assertFalse(hasPackRefsRun(repository));
        });
    }

    @DirtiesRepo
    @Test
    public void testPackRefsSkippedWhenRecentlyRun() throws IOException {
        long timestamp = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(1L);
        mockPackRefsRun(repository, timestamp);
        mockRepackRun(repository, timestamp);

        collectGarbage(repository);

        assertFalse(hasPackRefsRunSince(repository, timestamp));
    }

    @DirtiesRepo
    @Test
    public void testPause() {
        GcPause overlappingPause = null;
        try {
            try (GcPause pause = garbageCollectionManager.pause(repository, Duration.ofSeconds(2L))) {
                assertNotNull("Garbage collection was not paused", pause);

                // create loose objects to ensure GC will run
                createLooseObjects(repository);
                // try to run garbage collection
                collectGarbage(repository);
                // verify it didn't run
                assertFalse("Garbage collection ran while paused?", hasPackRefsRun(repository));

                // pause GC again, which should not block. GcPause should be additive and only
                // resumed after the _last_ pause has been released
                overlappingPause = garbageCollectionManager.pause(repository, Duration.ofSeconds(2L));
            }

            // try to run garbage collection
            collectGarbage(repository);
            // verify it didn't run
            assertFalse("Garbage collection ran while paused?", hasPackRefsRun(repository));
        } finally {
            // close the last pause to resume GC operations
            if (overlappingPause != null) {
                overlappingPause.close();
            }
        }

        // after the last lock is released, try to run GC again
        collectGarbage(repository);
        assertTrue("Garbage collection didn't run after resume", hasPackRefsRun(repository));
    }

    @DirtiesRepo
    @Test
    public void testPruneStaleOnly() throws IOException {
        Path staleTmpPackFile = createStaleTmpPackFile(repository);

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .staleOnly(true)
                .build()
        );

        verify(executorService).schedule(any(Runnable.class), anyLong(), any());
        assertFalse(Files.exists(staleTmpPackFile));
    }

    @DirtiesRepo
    @Test
    public void testPruningSkippedWhenLockHeld() throws IOException {
        // trigger creation of the app-info directory
        repositoryLayout.getApplicationInfoDir(repository);
        repositoryLayout.editFile(repository, GC_LOG, (reader, writer) -> {
            Path staleTmpPackFile = createStaleTmpPackFile(repository);

            // we hold the GC lock, so attempts to acquire the lock will fail
            garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                    .staleOnly(true)
                    .build()
            );

            verify(executorService).schedule(any(Runnable.class), anyLong(), any());
            assertTrue(Files.exists(staleTmpPackFile));
        });
    }

    @DirtiesRepo
    @Test
    public void testRepackRunsAfterIntervalExceeded() throws Exception {
        Repository repackRepository = getRepositoryThatShouldBeRepacked();

        // setup - ensure there are enough loose objects to trigger gc
        createLooseObjects(repackRepository);

        long lastRun = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1);
        mockRepackRun(repackRepository, lastRun);

        // time to GC - this should trigger repack to run in repackRepository
        collectGarbage(repository);

        assertTrue(hasRepackRunSince(repackRepository, lastRun));
        assertTrue(hasPackRefsRunSince(repackRepository, lastRun)); // implicitly run as part of gc

        maybeVerifyCommitGraph(repackRepository, true);
    }

    @DirtiesRepo
    @Test
    public void testRepackRunsWhenEnoughLooseObjects() {
        Repository repackRepository = getRepositoryThatShouldBeRepacked();
        createLooseObjects(repackRepository);

        // time to GC - this should trigger repack to run in repackRepository
        collectGarbage(repository);

        assertTrue(hasRepackRun(repackRepository));
    }

    @DirtiesRepo
    @Test
    public void testRepackRunsWhenEnoughPacks() throws IOException {
        Repository repackRepository = getRepositoryThatShouldBeRepacked();
        createFakePacks(repackRepository, false);

        // time to GC - this should trigger repack to run in repackRepository
        collectGarbage(repository);

        assertTrue(hasRepackRun(repackRepository));
    }

    @DirtiesRepo
    @Test
    public void testRepackRunsWhenLooseObjectsLargeEnough() {
        Repository repackRepository = getRepositoryThatShouldBeRepacked();
        createLooseObjects(repackRepository);

        //Set the loose object threshold high enough so we can test the loose object size threshold
        when(gcConfig.getRepackThresholdLoose()).thenReturn(256_000_000);
        //Set the loose object size threshold low enough to trigger "dirty" detection
        when(gcConfig.getRepackThresholdLooseSize()).thenReturn(256_000L);

        collectGarbage(repository);

        assertTrue(hasRepackRun(repackRepository));
    }

    @DirtiesRepo
    @Test
    public void testRepackSkippedWhenEnoughPacksButAllAreKeep() throws IOException {
        Repository repackRepository = getRepositoryThatShouldBeRepacked();
        createFakePacks(repackRepository, true);

        collectGarbage(repository);

        assertFalse(hasRepackRun(repackRepository));
    }

    @DirtiesRepo
    @Test
    public void testRepackSkippedWhenNotEnoughLooseObjects() {
        collectGarbage(repository);

        assertFalse(hasRepackRun(getRepositoryThatShouldBeRepacked()));
    }

    @DirtiesRepo
    @Test
    public void testRepackSkippedWhenRecentlyRun() throws Exception {
        Repository repackRepository = getRepositoryThatShouldBeRepacked();
        // setup - ensure there are enough loose objects to trigger gc
        createLooseObjects(repackRepository);

        long lastRun = System.currentTimeMillis() - 5000;
        mockRepackRun(repackRepository, lastRun);

        // run GC. This should not repack because it ran too recently
        collectGarbage(repository);

        assertFalse(hasRepackRunSince(repackRepository, lastRun));
    }

    @DirtiesRepo
    @Test
    public void testSchedulingExceptionHandled() {
        doThrow(new IllegalStateException()).when(executorService)
                .schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));

        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );

        verify(executorService, atLeastOnce()).schedule(
                any(Runnable.class), eq(config.getGit().getGc().getDelay().getSeconds()), eq(TimeUnit.SECONDS));
    }

    @DirtiesRepo
    @Test
    public void testSchedulingSkippedAfterShutdown() {
        garbageCollectionManager.shutdown();
        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );

        assertFalse(hasPackRefsRun(repository));
    }

    @Test
    public void testShutdown() {
        garbageCollectionManager.shutdown();

        verify(executorService).shutdown();
    }

    @DirtiesRepo
    @Test
    public void testStaleGcLockIsRemoved() throws IOException {
        createLooseObjects(repository);

        // create a lock file
        Path lockFile = repository.getPath().resolve(GC_LOG + ".lock");
        MoreFiles.touch(lockFile);

        // verify that the gc does not run with the lock-file in place
        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );
        assertFalse(hasPackRefsRun(repository));

        long gcTimeoutSec = TimeUnit.DAYS.toSeconds(2L);

        // now change the timestamp on the lockFile to make it stale
        Files.setLastModifiedTime(lockFile,
                FileTime.fromMillis(System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(10 * gcTimeoutSec)));

        // verify that gc now runs and that the lockFile is removed
        garbageCollectionManager.collectGarbage(new GarbageCollectionParameters.Builder(repository)
                .added(1)
                .build()
        );

        assertTrue(hasPackRefsRun(repository));
        assertFalse(Files.exists(lockFile));
    }

    protected static void createRef(Repository repository, String refId, String hash) throws IOException {
        Path ref = MoreFiles.resolve(repository.getPath(), normalizeRefId(refId));
        MoreFiles.mkdir(ref.getParent());
        Files.write(ref, singleton(hash));
    }

    protected static void createReflog(Repository repository, String refId, String content) throws IOException {
        Path reflog = MoreFiles.resolve(repository.getPath(), "logs", normalizeRefId(refId));
        MoreFiles.mkdir(reflog.getParent());
        Files.write(reflog, singleton(content));
    }

    protected static void createSharedReflog(ManagedRepository repository, String refId,
                                             String content) throws IOException {
        Path reflog = repository.getSharedObjects().getReflogDir(repository).resolve(normalizeRefId(refId));
        MoreFiles.mkdir(reflog.getParent());
        Files.write(reflog, singleton(content));
    }

    protected static String getReflog(Repository repository, String refId) throws IOException {
        Path reflog = MoreFiles.resolve(repository.getPath(), "logs", normalizeRefId(refId));
        try {
            return trim(MoreFiles.toString(reflog, StandardCharsets.UTF_8));
        } catch (NoSuchFileException | FileNotFoundException e) {
            return null;
        }
    }

    protected static String getSharedRefLog(ManagedRepository repository, String refId) throws IOException {
        Path reflog = repository.getSharedObjects().getReflogDir(repository).resolve(normalizeRefId(refId));
        try {
            return trim(MoreFiles.toString(reflog, StandardCharsets.UTF_8));
        } catch (NoSuchFileException | FileNotFoundException e) {
            return null;
        }
    }

    protected static String normalizeRefId(String refId) {
        return refId.replace('/', File.separatorChar);
    }

    /**
     * Sets the last modified timestamp for the specified {@code Path}, creating the target file if it doesn't
     * already exist. This is slightly different than {@link MoreFiles#touch touching} the file in that this is
     * setting a specific timestamp, rather than "now".
     *
     * @param path      the file to set the timestamp for
     * @param timestamp the timestamp to set, in milliseconds
     * @throws IOException if the timestamp cannot be set or, if the file doesn't exist, if it cannot be created
     */
    protected static void setLastModified(Path path, long timestamp) throws IOException {
        if (!Files.exists(path)) {
            Files.createFile(path);
        }
        Files.setLastModifiedTime(path, FileTime.fromMillis(timestamp));
    }

    protected void collectGarbage(Repository repository) {
        garbageCollectionManager.onRefsUpdated(new RefsUpdatedEvent(repository));
    }

    protected int countLooseObjects(Repository repository) {
        MutableInt count = new MutableInt(0);

        Path objectsDir = repositoryLayout.getObjectsDir(repository);
        try {
            Files.walkFileTree(objectsDir, EnumSet.noneOf(FileVisitOption.class), 2,
                    new SimpleFileVisitor<Path>() {

                        @Override
                        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
                            if (objectsDir.equals(dir) ||
                                    PATTERN_OBJECT_DIR.matcher(dir.getFileName().toString()).matches()) {
                                return FileVisitResult.CONTINUE;
                            }
                            return FileVisitResult.SKIP_SUBTREE;
                        }

                        @Override
                        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
                            if (PATTERN_LOOSE_OBJECT.matcher(file.getFileName().toString()).matches()) {
                                count.increment();
                            }

                            return FileVisitResult.CONTINUE;
                        }
                    });
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }

        return count.intValue();
    }

    protected int countPackFiles(Repository repository) throws IOException {
        Path packDir = repositoryLayout.getObjectsDir(repository).resolve("pack");
        try (Stream<Path> stream = Files.list(packDir)) {
            MutableInt packFiles = new MutableInt();
            MutableInt idxFiles = new MutableInt();
            stream.forEach(path -> {
                String filename = path.getFileName().toString();
                if (filename.endsWith(".pack")) {
                    packFiles.increment();
                } else if (filename.endsWith(".idx")) {
                    idxFiles.increment();
                }
            });
            assertEquals(".pack file count does not match .idx file count!", packFiles.intValue(), idxFiles.intValue());
            return packFiles.intValue();
        } catch (FileNotFoundException | NoSuchFileException ignored) {
            return 0;
        }
    }

    protected void createFakePacks(Repository repository, boolean createKeepfiles) throws IOException {
        createFakePacks(repository, 100, createKeepfiles);
    }

    protected void createFakePacks(Repository repository, int number, boolean createKeepFiles) throws IOException {
        String packName = "pack-" + UUID.randomUUID().toString().substring(0, 14);
        Path packDir = repositoryLayout.getObjectsDir(repository).resolve(PATH_PACK);
        byte[] packBytes = ByteStreams.toByteArray(new ClassPathResource("/gc/pack-mini.pack").getInputStream());
        byte[] idxBytes = ByteStreams.toByteArray(new ClassPathResource("/gc/pack-mini.idx").getInputStream());
        for (int i = 0; i < number; ++i) {
            Files.write(packDir.resolve(packName + i + ".pack"), packBytes);
            Files.write(packDir.resolve(packName + i + ".idx"), idxBytes);
            if (createKeepFiles) {
                MoreFiles.touch(packDir.resolve(packName + i + ".keep"));
            }
        }
    }

    protected void createLooseObjects(Repository targetRepository) {
        // This method loosens objects from the pack by resetting the master branch to the original commit and then
        // running git gc. This works, but is somewhat slow (200 - 250ms). We have a lot of tests that need plenty of
        // loose objects to trigger gc. To speed up the tests, we loosen objects once and cache them in a temp dir.
        // In subsequent invocations, we simply hardlink or cached copy the loose objects to the destination object dir.
        Path cachedLooseObjects = storageLayout.getTempDir().resolve("gcm-test-loose-objects");
        if (!Files.isDirectory(cachedLooseObjects)) {
            // Note that targetRepository does not have to be the repository under test (e.g. it can be the linked
            // shared-objects repository for managed repositories). In that case, the master branch may not exist.
            // To avoid that type of problem, we always create loose objects in the repository under test and then
            // hardlink them to the targetRepository.

            // reset master to the initial commit. This orphans nearly all the objects whose content-hash starts with 17
            agent.createRef(repository, "refs/heads/master", COMMIT_INITIAL);

            // trigger a gc. The repository is configured with gc.pruneexpire=never. Gc will eject the orphaned objects
            // from the pack and turn them into loose objects. The result? 40+ loose objects in objects/17
            GitProcessBuilder builder = builderFactory.builder(repository);
            // for Git 2.41+, disable cruft packs, otherwise loose objects aren't actually created
            if (config.getGit().getBinary().getVersion().compareTo(new Version(2, 41)) > -1) {
                builder.withConfiguration("gc.cruftPacks", false);
            }
            builder.gc().build().run();

            MoreFiles.mkdir(cachedLooseObjects);
            hardLinkLooseObjects(repository.getPath().resolve(PATH_OBJECTS), cachedLooseObjects);

            // restore master to its original value
            agent.createRef(repository, "refs/heads/master", "d0a1815035030ff854404f0ad99721206846d10b");
        }
        hardLinkLooseObjects(cachedLooseObjects, targetRepository.getPath().resolve(PATH_OBJECTS));
        Path marker = getRepackMarker(targetRepository);
        try {
            Files.deleteIfExists(marker);
        } catch (IOException e) {
            throw new IllegalStateException("Could not delete the gc marker; test asserts are unreliable!", e);
        }
    }

    protected ManagedRepository createManagedRepository(String repositoryId, Repository fromRepository) {
        ManagedRepository repo = createManagedRepository(repositoryId, null);
        //Initialize the repository with the contents of repository
        builderFactory.builder(repo).fetch()
                .repository(fromRepository.getPath())
                .refspecs("+refs/heads/*:refs/heads/*")
                .refspecs("+refs/tags/*:refs/tags/*")
                .tags(GitFetchTagMode.NO_TAGS)
                .build(new NullStdioHandler<>())
                .run();
        return repo;
    }

    protected ManagedRepository createManagedRepository(String repositoryId, ManagedRepository origin) {
        CreateRepositoryParameters.Builder builder = new CreateRepositoryParameters.Builder(repositoryId);
        if (origin != null) {
            builder.origin(origin);
        }

        ManagedRepository repository = (ManagedRepository) repositoryManager.create(builder.build());
        if (origin != null) {
            garbageCollectionManager.onRepositoryCreated(new ForkCreatedEvent(repository, origin));
        } else {
            garbageCollectionManager.onRepositoryCreated(new RepositoryCreatedEvent(repository));
        }

        return repository;
    }

    protected Path createStaleTmpPackFile(Repository repository) throws IOException {
        Path packDir = repositoryLayout.getObjectsDir(repository).resolve(PATH_PACK);
        Path tmpPackFile = packDir.resolve("tmp_pack_sNCtNU");
        MoreFiles.touch(tmpPackFile);
        setLastModified(tmpPackFile, System.currentTimeMillis() - TimeUnit.DAYS.toMillis(15L));
        return tmpPackFile;
    }

    protected void disablePrune(Repository repository) {
        builderFactory.builder(repository).config()
                .set(CONFIG_GC_PRUNE_EXPIRE, CONFIG_GC_PRUNE_EXPIRE_NEVER)
                .build()
                .run();
    }

    protected Path getFullVacuumMarker(Repository repository) {
        return repositoryLayout.getApplicationInfoDir(repository).resolve(VACUUM_FULL_TIMESTAMP);
    }

    protected Path getPackedRefsMarker(Repository repository) {
        return repositoryLayout.getApplicationInfoDir(repository).resolve(PACKREFS_TIMESTAMP);
    }

    protected Set<String> getRefs(Repository repository) {
        ImmutableSet.Builder<String> builder = ImmutableSet.builder();
        builderFactory.builder(repository).forEachRef()
                .format(AllRefsForEachRefStdioHandler.FORMAT)
                .build(new AllRefsForEachRefStdioHandler(ref -> {
                    builder.add(ref.getId());
                    return false;
                }, "refs/heads/master"))
                .run();
        return builder.build();
    }

    protected Path getRepackMarker(Repository repository) {
        return repositoryLayout.getApplicationInfoDir(repository).resolve(REPACK_TIMESTAMP);
    }

    /**
     * @return the repository that is expected to repack in response to a garbage collection request for
     *         {@link #getRepositoryUnderTest() the repository under test}; for {@link BitbucketServerRepository}
     *         it's the repository itself; for {@link ManagedRepository} it is the corresponding
     *         {@link SharedObjectsRepository}
     */
    protected abstract Repository getRepositoryThatShouldBeRepacked();

    protected abstract Repository getRepositoryUnderTest();

    protected Path getVacuumMarker(Repository repository) {
        return repositoryLayout.getApplicationInfoDir(repository).resolve(VACUUM_TIMESTAMP);
    }

    protected void hardLinkCruftPacks(Path sourceObjectsDir, Path targetObjectsDir) {
        try {
            Files.walkFileTree(sourceObjectsDir, new SimpleFileVisitor<Path>() {

                @Override
                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                    String filename = file.getFileName().toString();
                    Matcher m = MTIMES_REGEX.matcher(filename);
                    if (m.find()) {
                        String packId = filename.substring(0, m.start());
                        for (String suffix : new String[]{".mtimes", ".idx", ".pack"}) {
                            MoreFiles.createHardLinkOrCopy(
                                    targetObjectsDir.resolve(sourceObjectsDir.relativize(
                                            file.resolveSibling(packId + suffix))),
                                    file.resolveSibling(packId + suffix), false);
                        }
                    }
                    return FileVisitResult.CONTINUE;
                }
            });
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    protected void hardLinkLooseObjects(Path sourceObjectsDir, Path targetObjectsDir) {
        try {
            Files.walkFileTree(sourceObjectsDir, new SimpleFileVisitor<Path>() {

                @Override
                public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
                    Path relPath = sourceObjectsDir.relativize(dir);
                    if (relPath.getNameCount() > 0 && relPath.getFileName().toString().length() <= 2) {
                        MoreFiles.mkdir(targetObjectsDir.resolve(relPath));
                        return FileVisitResult.CONTINUE;
                    }
                    return FileVisitResult.SKIP_SUBTREE;
                }

                @Override
                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                    if (attrs.isRegularFile()) {
                        MoreFiles.createHardLinkOrCopy(
                                targetObjectsDir.resolve(sourceObjectsDir.relativize(file)), file, false);
                    }
                    return FileVisitResult.CONTINUE;
                }
            });
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }

    protected boolean hasCommitGraph(Repository repository, boolean full) {
        Path repositoryDir = repository.getPath();
        Path infoDir = MoreFiles.resolve(repositoryDir, PATH_OBJECTS, PATH_INFO);

        Path commitGraph;
        if (full) {
            commitGraph = infoDir.resolve(PATH_COMMIT_GRAPH);
        } else {
            commitGraph = MoreFiles.resolve(infoDir, PATH_COMMIT_GRAPHS, PATH_COMMIT_GRAPH_CHAIN);
        }

        return Files.isRegularFile(commitGraph);
    }

    protected boolean hasPackRefsRun(Repository repository) {
        return hasPackRefsRunSince(repository, 0L);
    }

    protected boolean hasPackRefsRunSince(Repository repository, long timestamp) {
        return MoreFiles.getLastModified(getPackedRefsMarker(repository)) > timestamp;
    }

    protected boolean hasRepackRun(Repository repository) {
        return hasRepackRunSince(repository, 0L);
    }

    protected boolean hasRepackRunSince(Repository repository, long timestamp) {
        return MoreFiles.getLastModified(getRepackMarker(repository)) > timestamp;
    }

    protected boolean hasVacuumingRun(ManagedRepository repository) {
        return hasVacuumingRunSince(repository, 0L);
    }

    protected boolean hasVacuumingRunSince(ManagedRepository repository, long timestamp) {
        return MoreFiles.getLastModified(getVacuumMarker(repository)) > timestamp;
    }

    protected boolean maybeCreateCruftPack(Repository targetRepository) {
        if (config.getGit().getBinary().getVersion().compareTo(new Version(2, 41)) < 0) {
            return false;
        }
        // This method creates cruft packs by resetting the master branch to the original commit and then
        // running git gc. This works, but is somewhat slow (200 - 250ms). We cache the created cruft pack in a directory.
        Path cachedCruftPack = storageLayout.getTempDir().resolve("gcm-test-cruft-pack");
        if (!Files.isDirectory(cachedCruftPack)) {
            // Note that targetRepository does not have to be the repository under test (e.g. it can be the linked
            // shared-objects repository for managed repositories). In that case, the master branch may not exist.
            // To avoid that type of problem, we always create loose objects in the repository under test and then
            // hardlink them to the targetRepository.

            // reset master to the initial commit. This orphans nearly all the objects whose content-hash starts with 17
            agent.createRef(repository, "refs/heads/master", COMMIT_INITIAL);

            // trigger a gc. The repository is configured with gc.pruneexpire=never. Gc will eject the orphaned objects
            // from the pack and create a separate cruft pack
            builderFactory.builder(repository).gc().build().run();

            MoreFiles.mkdir(cachedCruftPack);
            hardLinkCruftPacks(repository.getPath().resolve(PATH_OBJECTS).resolve(PATH_PACK), cachedCruftPack);

            // restore master to its original value
            agent.createRef(repository, "refs/heads/master", "d0a1815035030ff854404f0ad99721206846d10b");
        }
        hardLinkCruftPacks(cachedCruftPack, targetRepository.getPath().resolve(PATH_OBJECTS).resolve(PATH_PACK));
        Path marker = getRepackMarker(targetRepository);
        try {
            Files.deleteIfExists(marker);
        } catch (IOException e) {
            throw new IllegalStateException("Could not delete the gc marker; test asserts are unreliable!", e);
        }
        return true;
    }

    protected void maybeVerifyCommitGraph(Repository repository, boolean full) {
        // for Git 2.24+, verify the commit graph is written and valid
        if (config.getGit().getBinary().getVersion().compareTo(new Version(2, 24)) > -1) {
            assertTrue(hasCommitGraph(repository, full));

            builderFactory.builder(repository).command("commit-graph")
                    .argument("verify")
                    .argument("--no-progress")
                    .build(new LoggingStdioHandler())
                    .run();
        }
    }

    protected GitProcess<Void> mockPackRefsProcess(Repository repository) {
        @SuppressWarnings("unchecked")
        GitProcess<Void> packRefsProcess = mock(GitProcess.class);

        doAnswer(factoryInvocation -> {
            GitProcessBuilder builderSpy = (GitProcessBuilder) spy(factoryInvocation.callRealMethod());
            doAnswer(builderInvocation -> {
                GitPackRefsBuilder packRefsBuilder = (GitPackRefsBuilder) spy(builderInvocation.callRealMethod());
                doReturn(packRefsProcess).when(packRefsBuilder).build(any());

                return packRefsBuilder;
            }).when(builderSpy).packRefs();

            return builderSpy;
        }).when(builderFactory).builder(same(repository));

        return packRefsProcess;
    }

    protected void mockPackRefsRun(Repository repository, long timestamp) throws IOException {
        setLastModified(getPackedRefsMarker(repository), timestamp);
    }

    protected GitProcess<Void> mockRepackProcess(Repository repository) {
        @SuppressWarnings("unchecked")
        GitProcess<Void> repackProcess = mock(GitProcess.class);

        doAnswer(factoryInvocation -> {
            GitProcessBuilder builderSpy = (GitProcessBuilder) spy(factoryInvocation.callRealMethod());
            doAnswer(builderInvocation -> {
                GitRepackBuilder repackSpy = (GitRepackBuilder) spy(builderInvocation.callRealMethod());
                doReturn(repackProcess).when(repackSpy).build(any());

                return repackSpy;
            }).when(builderSpy).repack();

            return builderSpy;
        }).when(builderFactory).builder(same(repository));

        return repackProcess;
    }

    protected void mockRepackRun(Repository repository, long timestamp) throws IOException {
        setLastModified(getRepackMarker(repository), timestamp);
    }
}
