package com.atlassian.stash.internal.scm.git.command;

import com.atlassian.bitbucket.content.*;
import com.atlassian.bitbucket.repository.Repository;
import com.atlassian.bitbucket.scm.CommandSummary;
import com.atlassian.bitbucket.scm.DirectoryCommandParameters;
import com.atlassian.bitbucket.scm.git.GitScmConfig;
import com.atlassian.bitbucket.scm.git.GitUtils;
import com.atlassian.bitbucket.scm.git.command.GitCommandBuilderFactory;
import com.atlassian.bitbucket.scm.git.command.lstree.GitLsTreeBuilder;
import com.atlassian.bitbucket.util.PageRequest;
import com.atlassian.bitbucket.util.Timer;
import com.atlassian.bitbucket.util.TimerUtils;
import com.atlassian.stash.internal.scm.git.command.catfile.AbstractBatchCatFileHandler;
import com.atlassian.stash.internal.scm.git.command.catfile.BatchHeader;
import com.atlassian.stash.internal.scm.git.command.lstree.CallbackLsTreeOutputHandler;
import com.atlassian.stash.internal.scm.git.submodule.Submodules;
import com.atlassian.utils.process.ProcessException;
import com.atlassian.utils.process.Watchdog;
import com.google.common.base.Throwables;
import com.google.common.io.ByteStreams;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import java.io.*;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutorService;

/**
 * Uses {@code git ls-tree}, potentially paired with {@code git cat-file --batch}, to collapse directories which
 * contain a single subdirectory when streaming tree nodes.
 * <p>
 * For recursive listings, {@code git ls-tree} is used on its own. {@code git ls-tree --recursive} only outputs
 * files and submodules, so there are no directories to collapse.
 * <p>
 * For single-level listings, the {@code git ls-tree} output is cached to disk without parsing it, and then a
 * {@code git cat-file --batch} process is started. As the {@code ls-tree} output is read, directories are run
 * through {@code cat-file} to list their contents. Directories containing a single subdirectory are merged and
 * contents for the subdirectory are listed. This process continues until the listing finds a tree with multiple
 * entries, or where the single entry is not a directory. Single files or submodules are <i>not</i> collapsed into
 * directories.
 * <p>
 * Conceptually, {@code git cat-file --batch} could be used to do <i>all</i> of this work by forking a single
 * process. However, that has a couple issues:
 * <ul>
 *     <li>{@code cat-file --batch} dumps entire trees, meaning it's not possible to list subdirectories prior to
 *     reading the full root listing
 *     <ul>
 *         <li>This means either the entire listing for the requested tree would need to be held in memory, or
 *         cached to disk anyway</li>
 *     </ul>
 *     </li>
 *     <li>{@link DirectoryCommandParameters#isWithSizes()} can't be honored using a single {@code cat-file}</li>
 * </ul>
 * Rather than work around those limitations, {@code ls-tree} is used for the initial listing.
 * <p>
 * Since the {@code ls-tree} output is cached to disk without processing, the {@code cat-file} process is started
 * even though there may not be any directories in the listing. That's assumed to be a minority case, and the
 * overhead of starting a {@code git cat-file --batch} process (which just waits for input without doing much
 * setup) is limited enough that this approach has been chosen over parsing the {@code ls-tree} output up front.
 * Approaches that eliminate the unnecessary {@code git cat-file} process result in significant additional code
 * complexity for minimal benefit.
 *
 * @since 4.9
 */
public class DirectoryCommand extends SimpleGitCommand<Void> {

    private static final Logger log = LoggerFactory.getLogger(DirectoryCommand.class);

    private final GitCommandBuilderFactory builderFactory;
    private final ContentTreeCallback callback;
    private final GitScmConfig config;
    private final PageRequest pageRequest;
    private final DirectoryCommandParameters parameters;
    private final Repository repository;
    private final Submodules submodules;

    public DirectoryCommand(ExecutorService executorService, GitCommandBuilderFactory builderFactory,
                            GitScmConfig config, Repository repository, DirectoryCommandParameters parameters,
                            ContentTreeCallback callback, PageRequest pageRequest, Submodules submodules) {
        super(executorService);

        this.builderFactory = builderFactory;
        this.callback = callback;
        this.config = config;
        this.pageRequest = pageRequest;
        this.parameters = parameters;
        this.repository = repository;
        this.submodules = submodules;
    }

    @Override
    public Void call() {
        File list = parameters.isRecursive() ? null : maybeCreateTempFile();

        GitLsTreeBuilder builder = builderFactory.builder(repository).lsTree()
                .nullTerminated(true)
                .recursive(parameters.isRecursive())
                .sizes(parameters.isWithSizes())
                .tree(parameters.getCommitId(), StringUtils.defaultString(parameters.getPath()));
        if (list == null) {
            //If we couldn't create a temp file to cache the ls-tree output, or if a recursive listing has
            //been requested, we just stream the ls-tree output directly to the callback
            return configureAndCall(builder.build(
                    new CallbackLsTreeOutputHandler(callback, parameters.toContext(), pageRequest, submodules)));
        }

        //In order to collapse single-entry directories, we need to cache the listing to disk. That way we
        //can read it back while running git cat-file --batch to check trees
        try (Timer ignored = TimerUtils.start("git: stream directory")) {
            configureAndCall(builder.build(new FileOutputHandler(list)));

            DirectoryBatchCatFileHandler handler = new DirectoryBatchCatFileHandler(list);
            configureAndCall(builderFactory.builder(repository).catFile()
                    .batch(handler)
                    .build(handler));
        } finally {
            if (!list.delete()) {
                list.deleteOnExit();
            }
        }

        return null;
    }

    /**
     * Attempts to create a temporary file for storing {@code git ls-tree} output. If a file can't be created,
     * rather than propagating the exception, {@code null} is returned so that the listing will instead stream
     * directly to the caller.
     *
     * @return the created temporary file, or {@code null} if one could not be created
     */
    private File maybeCreateTempFile() {
        try {
            return File.createTempFile(repository.getSlug() + "-dir", ".list", config.getTempDir());
        } catch (IOException e) {
            log.warn("{}: Could not create a temp file; directories will not be collapsed", repository, e);
            return null;
        }
    }

    /**
     * A {@link git cat-file --batch} handler which manages both input and output streams to allow requesting
     * trees interactively for {@link Directory} nodes as they're streamed. This handler receives a file with
     * cached {@code git ls-tree} output on its constructor, and uses {@link CallbackLsTreeOutputHandler} for
     * parsing. The {@link ContentTreeCallback} provided by the caller is wrapped to intercept directories to
     * check whether they have subdirectories that should be collapsed.
     */
    private class DirectoryBatchCatFileHandler extends AbstractBatchCatFileHandler implements ContentTreeCallback {

        private final File list;
        private final CallbackLsTreeOutputHandler listHandler;

        DirectoryBatchCatFileHandler(File list) {
            this.list = list;

            listHandler = new CallbackLsTreeOutputHandler(this, parameters.toContext(), pageRequest, submodules);
        }

        @Override
        public void onComplete(@Nonnull CommandSummary summary) throws ProcessException {
            if (getOutput() == null) { //getOutput() is only null when super.onComplete has not been called
                //CallbackLsTreeOutputHandler should only be called once, but this onComplete method is called
                //twice (once for the CommandInputHandler side and once for the CommandOutputHandler side)
                listHandler.onComplete(summary);
            }

            super.onComplete(summary);
        }

        @Override
        public void onEnd(@Nonnull ContentTreeSummary summary) throws IOException {
            callback.onEnd(summary);
        }

        @Override
        public void onStart(@Nonnull ContentTreeContext context) throws IOException {
            callback.onStart(context);
        }

        /**
         * Forwards nodes to the delegate {@link ContentTreeCallback}.
         * <p>
         * If the provided {@link ContentTreeNode} is a {@link Directory}, a {@link DirectoryCollapsingConsumer}
         * is used to iteratively request trees from the coupled {@code git cat-file --batch} process. If the
         * returned tree only includes a single directory it's combined into the provided {@link Directory} to
         * eliminate the extra level. This collapsing processing loops until either:
         * <ul>
         *     <li>A tree is found which has multiple entries, or</li>
         *     <li>A tree is found whose single entry is not another directory</li>
         * </ul>
         * Directories that contain a single {@link com.atlassian.bitbucket.content.File File} or {@link Submodule}
         * are <i>not</i> collapsed. The type of {@link ContentTreeNode} received is always the same type that gets
         * forwarded to the {@link ContentTreeCallback}; the only change is to merge directory paths and overwrite
         * the {@link Directory#getContentId() content ID}.
         *
         * @param node the current tree node
         * @return {@code true} or {@code false} as returned by the delegate {@link ContentTreeCallback}
         * @throws IOException if the {@link ContentTreeNode node} is a {@link Directory} and its tree can't
         *                     be requested, or if the delegate {@link ContentTreeCallback} throws
         */
        @Override
        public boolean onTreeNode(@Nonnull ContentTreeNode node) throws IOException {
            if (node.getContentId() != null && node instanceof Directory) {
                DirectoryCollapsingConsumer consumer = new DirectoryCollapsingConsumer((Directory) node);
                while (consumer.collapsing) {
                    requestObject(consumer.getContentId(), consumer);
                }

                node = consumer.directory;
            }

            return callback.onTreeNode(node);
        }

        @Override
        public void setWatchdog(Watchdog watchdog) {
            //The callback may cancel processing before the entire list is streamed, and in order to do
            //that it needs to have the Watchdog set
            listHandler.setWatchdog(watchdog);

            super.setWatchdog(watchdog);
        }

        /**
         * Opens the {@code git ls-tree} output and streams it to the {@link CallbackLsTreeOutputHandler} for
         * parsing.
         *
         * @throws IOException if the {@code git ls-tree} output cannot be read
         */
        @Override
        protected void process() throws IOException {
            try (InputStream listStream = new BufferedInputStream(new FileInputStream(list))) {
                listHandler.process(listStream);
            } catch (ProcessException e) {
                //If the ProcessException was caused by an IOException, RuntimeException or Error, unwrap
                //it and propagate the cause. This is almost certain to work
                Throwables.propagateIfPossible(e.getCause(), IOException.class);

                //Otherwise, wrap the cause in an IOException (for even more wrapping later)
                throw new IOException(e.getCause() == null ? e : e.getCause());
            }
        }

        /**
         * A {@link BatchOutputConsumer} which requests trees and, if they only contain a single subdirectory,
         * collapses them into a working {@link Directory}.
         */
        private class DirectoryCollapsingConsumer implements BatchOutputConsumer {

            private boolean collapsing;
            private Directory directory;

            DirectoryCollapsingConsumer(Directory directory) {
                this.directory = directory;

                collapsing = true;
            }

            /**
             * Reads the first entry from the requested tree and determines whether it should be collapsed into
             * the working {@link Directory}.
             * <p>
             * With {@code git cat-file --batch} trees are not pretty-printed and have to be parsed manually.
             * Trees follow a format like: {@code &lt;mode&gt;&lt;sp&gt;&lt;path&gt;&lt;nul&gt;&lt;sha&gt;}.
             * The number of characters in the {@code &lt;mode&gt;} and {@code &lt;path&gt;} is not fixed, so
             * they must be read until the separator character (a space or a {@code nul} byte, respectively)
             * is found. The entry type is determined by parsing the mode.
             * <p>
             * If the first entry in the tree is not a directory (a file or submodule), no further processing
             * is performed and the entry is not collapsed into the working {@link Directory}.
             * <p>
             * If the first entry is a directory, its path and content ID are read, to exhaust the entry, and
             * then a further read is performed to check for another tree entry. If there is another entry,
             * the entry is not collapsed into the working {@link Directory}. Otherwise, if there's no further
             * entry, a new {@link SimpleDirectory} is created to replace the working {@link Directory}, merging
             * the working path with the subdirectory's path and replacing the content ID with the subdirectory's.
             *
             * @param header the {@code git cat-file --batch} header, which is ignored
             * @param stream a stream over the requested object's bytes
             * @throws IOException if the {@code stream} cannot be read
             */
            @Override
            public void accept(@Nonnull BatchHeader header, @Nonnull InputStream stream) throws IOException {
                if (header.isMissing()) {
                    //This should never happen, but if the tree we requested can't be found, rather than blowing
                    //up, stop collapsing and return whatever we've done so far
                    collapsing = false;

                    return;
                }

                String mode = readUntil(stream, ' ');
                if (GitUtils.isDirectory(StringUtils.leftPad(mode, 6, '0'))) {
                    String path = readUntil(stream, 0x00); //Null byte separates path from SHA
                    String contentId = readSha(stream);

                    //We've read the full entry. Now we need to check and see if there's another entry. If there
                    //is, we don't collapse
                    if (stream.read() == -1) {
                        //We've read the whole tree, and it only contains a single directory, so we merge it
                        //into the working directory
                        SimplePath merged = new SimplePath(new SimplePath(directory.getPath()), path);

                        directory = new SimpleDirectory.Builder(directory)
                                .path(merged)
                                .contentId(contentId) //Use the content ID of the tip directory, not the base
                                .build();
                        return;
                    }
                }

                //If we get here, it either means the tree entry we read wasn't a directory, or there were
                //multiple entries in the tree. Either way, we're not collapsing any further entries
                collapsing = false;
            }

            //Having this method prevents a ConstantConditions warning from IDEA in onTreeNode
            String getContentId() {
                return directory.getContentId();
            }

            /**
             * Reads 20 bytes from the stream and returns them as a hex-encoded string.
             *
             * @param stream the stream to read
             * @return the 40-character SHA-1
             * @throws IOException if the stream cannot be read
             */
            private String readSha(InputStream stream) throws IOException {
                byte[] buffer = new byte[20];
                ByteStreams.readFully(stream, buffer, 0, buffer.length);

                return Hex.encodeHexString(buffer);
            }

            /**
             * Reads from the provided {@code InputStream} until the specified separator character is found.
             * If the stream is exhausted before the separator is found an exception is thrown.
             *
             * @param stream    the stream to read
             * @param separator the separator to find
             * @return the characters before the separator
             * @throws IllegalStateException if the stream is exhausted before the separator is found
             * @throws IOException if the stream cannot be read
             */
            private String readUntil(InputStream stream, int separator) throws IOException {
                //It's not safe use a StringBuilder here and (char) cast the characters we read. That approach
                //will work for ASCII characters, but fails for Unicode. Instead, we read everything as bytes
                //and then build a String at the end using UTF-8 encoding explicitly
                ByteArrayOutputStream buffer = new ByteArrayOutputStream(255);
                for (int character = stream.read(); character != separator; character = stream.read()) {
                    if (character == -1) {
                        throw new IllegalStateException("Unexpected end-of-stream reading tree for [" +
                                getContentId() + "]");
                    }
                    buffer.write(character);
                }

                return new String(buffer.toByteArray(), StandardCharsets.UTF_8);
            }
        }
    }
}
