diff options
author | DongHun Kwak <dh0128.kwak@samsung.com> | 2021-04-12 15:20:05 +0900 |
---|---|---|
committer | DongHun Kwak <dh0128.kwak@samsung.com> | 2021-04-12 15:20:05 +0900 |
commit | 6adb4ac0619fd0e27f30581dbbbee71b7efb984d (patch) | |
tree | f1a3d6f0cf70a7f8e63c6f4a9329e0fba4a53d1b | |
download | python3-vcstool-upstream.tar.gz python3-vcstool-upstream.tar.bz2 python3-vcstool-upstream.zip |
Imported Upstream version python3-vcstool 0.2.14upstream/0.2.14upstream
48 files changed, 4447 insertions, 0 deletions
diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..79ffb05 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include vcstool-completion/vcs.bash vcstool-completion/vcs.tcsh vcstool-completion/vcs.zsh diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..8635ad6 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,18 @@ +Metadata-Version: 1.2 +Name: vcstool +Version: 0.2.14 +Summary: vcstool provides a command line tool to invoke vcs commands on multiple repositories. +Home-page: https://github.com/dirk-thomas/vcstool +Author: Dirk Thomas +Author-email: web@dirk-thomas.net +Maintainer: Dirk Thomas +Maintainer-email: web@dirk-thomas.net +License: Apache License, Version 2.0 +Download-URL: http://download.ros.org/downloads/vcstool/ +Description: vcstool enables batch commands on multiple different vcs repositories. Currently it supports git, hg, svn and bzr. +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Version Control +Classifier: Topic :: Utilities diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..2d28a08 --- /dev/null +++ b/README.rst @@ -0,0 +1,197 @@ +What is vcstool? +================ + +Vcstool is a version control system (VCS) tool, designed to make working with multiple repositories easier. + +Note: + This tool should not be confused with `vcstools <https://github.com/vcstools/vcstools/>`_ (with a trailing ``s``) which provides a Python API for interacting with different version control systems. + The biggest differences between the two are: + + * ``vcstool`` doesn't use any state beside the repository working copies available in the filesystem. + * The file format of ``vcstool export`` uses the relative paths of the repositories as keys in YAML which avoids collisions by design. + * ``vcstool`` has significantly fewer lines of code than ``vcstools`` including the command line tools built on top. + + +How does it work? +----------------- + +Vcstool operates on any folder from where it recursively searches for supported repositories. +On these repositories vcstool invokes the native VCS client with the requested command (i.e. *diff*). + + +Which VCS types are supported? +------------------------------ + +Vcstool supports `Git <http://git-scm.com>`_, `Mercurial <http://git-scm.comhttp://mercurial.selenic.com>`_, `Subversion <http://subversion.apache.org>`_, `Bazaar <http://bazaar.canonical.com/en/>`_. + + +How to use vcstool? +------------------- + +The script ``vcs`` can be used similarly to the VCS clients ``git``, ``hg`` etc. +The ``help`` command provides a list of available commands with an additional description:: + + vcs help + +By default vcstool searches for repositories under the current folder. +Optionally one path (or multiple paths) can be passed to search for repositories at different locations:: + + vcs status /path/to/several/repos /path/to/other/repos /path/to/single/repo + + +Exporting and importing sets of repositories +-------------------------------------------- + +Vcstool can export and import all the information required to reproduce the versions of a set of repositories. +Vcstool uses a simple `YAML <http://www.yaml.org/>`_ format to encode this information. +This format includes a root key ``repositories`` under which each local repository is described by a dictionary keyed by its relative path. +Each of these dictionaries contains keys ``type``, ``url``, and ``version``. +If the ``version`` key is omitted the default branch is being used. + +This results in something similar to the following for a set of two repositories (`vcstool <https://github.com/dirk-thomas/vcstool>`_ cloned via Git and `rosinstall <http://github.com/vcstools/rosinstall>`_ checked out via Subversion): + +.. code-block:: yaml + + repositories: + vcstool: + type: git + url: git@github.com:dirk-thomas/vcstool.git + version: master + old_tools/rosinstall: + type: svn + url: https://github.com/vcstools/rosinstall/trunk + version: 748 + + +Export set of repositories +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``vcs export`` command outputs the path, vcs type, URL and version information for all repositories in `YAML <http://www.yaml.org/>`_ format. +The output is usually piped to a file:: + + vcs export > my.repos + +If the repository is currently on the tip of a branch the branch is followed. +This implies that a later import might fetch a newer revision if the branch has evolved in the meantime. +Furthermore if the local branch has evolved from the remote repository an import might not result in the exact same state. + +To make sure to store the exact revision in the exported data use the command line argument ``--exact``. +Since a specific revision is not tied to neither a branch nor a remote (for Git and Mercurial) the tool will check if the current hash exists in any of the remotes. +If it exists in multiple the remotes ``origin`` and ``upstream`` are considered before any other in alphabetical order. + + +Import set of repositories +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``vcs import`` command clones all repositories which are passed in via ``stdin`` in YAML format. +Usually the data of a previously exported file is piped in:: + + vcs import < my.repos + +The ``import`` command also supports input in the `rosinstall file format <http://www.ros.org/doc/independent/api/rosinstall/html/rosinstall_file_format.html>`_. +Beside passing a file path the command also supports passing a URL. + +Only for this command vcstool supports the pseudo clients ``tar`` and ``zip`` which fetch a tarball / zipfile from a URL and unpack its content. +For those two types the ``version`` key is optional. +If specified only entries from the archive which are in the subfolder specified by the version value are being extracted. + + +Validate repositories file +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``vcs validate`` command takes a YAML file which is passed in via ``stdin`` and validates its contents and format. +The data of a previously-exported file or hand-generated file are piped in:: + + vcs validate < my.repos + +The ``validate`` command also supports input in the `rosinstall file format <http://www.ros.org/doc/independent/api/rosinstall/html/rosinstall_file_format.html>`_. + + +Advanced features +----------------- + +Show log since last tag +~~~~~~~~~~~~~~~~~~~~~~~ + +The ``vcs log`` command supports the argument ``--limit-untagged`` which will output the log for all commits since the last tag. + + +Parallelization and stdin +~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default ``vcs`` parallelizes the work across multiple repositories based on the number of CPU cores. +In the case that the invoked commands require input from ``stdin`` that parallelization is a problem. +In order to be able to provide input to each command separately these commands must run sequentially. +When needing to e.g. interactively provide credentials all commands should be executed sequentially by passing: + + --workers 1 + +In the case repositories are using SSH ``git@`` URLs but the host is not known yet ``vcs import`` automatically falls back to a single worker. + + +Run arbitrary commands +~~~~~~~~~~~~~~~~~~~~~~ + +The ``vcs custom`` command enables to pass arbitrary user-specified arguments to the vcs invocation. +The set of repositories to operate on can optionally be restricted by the type: + + vcs custom --git --args log --oneline -n 10 + +If the command should work on multiple repositories make sure to pass only generic arguments which work for all of these repository types. + + +How to install vcstool? +======================= + +On Debian-based platforms the recommended method is to install the package *python3-vcstool*. +On Ubuntu this is done using *apt-get*:: + + sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' + sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv-key 0xAB17C654 + sudo apt-get update + sudo apt-get install python3-vcstool + +On other systems, use the `PyPI <http://pypi.python.org>`_ package:: + + sudo pip install vcstool + + +Setup auto-completion +--------------------- + +For the shells *bash*, *tcsh* and *zsh* vcstool can provide auto-completion of the various VCS commands. +In order to enable that feature the shell specific completion file must be sourced. + +For *bash* append the following line to the ``~/.bashrc`` file:: + + source /usr/share/vcstool-completion/vcs.bash + +For *tcsh* append the following line to the ``~/.cshrc`` file:: + + source /usr/share/vcstool-completion/vcs.tcsh + +For *zsh* append the following line to the ``~/.zshrc`` file:: + + source /usr/share/vcstool-completion/vcs.zsh + + +How to contribute? +================== + +How to report problems? +----------------------- + +Before reporting a problem please make sure to use the latest version. +Issues can be filled on `GitHub <https://github.com/dirk-thomas/vcstool/issues>`_ after making sure that this problem has not yet been reported. + +Please make sure to include as much information, i.e. version numbers from vcstool, operating system, Python and a reproducible example of the commands which expose the problem. + + +How to try the latest changes? +------------------------------ + +Sourcing the ``setup.sh`` file prepends the ``src`` folder to the ``PYTHONPATH`` and the ``scripts`` folder to the ``PATH``. +Then vcstool can be used with the commands ``vcs-COMMAND`` (note the hyphen between ``vcs`` and ``command`` instead of a space). + +Alternatively the *develop* command from Python setuptools can be used: + sudo python setup.py develop diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8bfd5a1 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,4 @@ +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..bb7d2c0 --- /dev/null +++ b/setup.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +import sys + +from setuptools import find_packages +from setuptools import setup +from vcstool import __version__ + +install_requires = ['PyYAML', 'setuptools'] +if sys.version_info[0] == 2 and sys.version_info[1] < 7: + install_requires.append('argparse') + +setup( + name='vcstool', + version=__version__, + install_requires=install_requires, + packages=find_packages(), + author='Dirk Thomas', + author_email='web@dirk-thomas.net', + maintainer='Dirk Thomas', + maintainer_email='web@dirk-thomas.net', + url='https://github.com/dirk-thomas/vcstool', + download_url='http://download.ros.org/downloads/vcstool/', + classifiers=['Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Topic :: Software Development :: Version Control', + 'Topic :: Utilities'], + description='vcstool provides a command line tool to invoke vcs commands ' + 'on multiple repositories.', + long_description='\ +vcstool enables batch commands on multiple different vcs repositories. \ +Currently it supports git, hg, svn and bzr.', + license='Apache License, Version 2.0', + data_files=[ + ('share/vcstool-completion', [ + 'vcstool-completion/vcs.bash', + 'vcstool-completion/vcs.tcsh', + 'vcstool-completion/vcs.zsh' + ]) + ], + entry_points={ + 'console_scripts': [ + 'vcs = vcstool.commands.vcs:main', + 'vcs-branch = vcstool.commands.branch:main', + 'vcs-bzr = vcstool.commands.custom:bzr_main', + 'vcs-custom = vcstool.commands.custom:main', + 'vcs-diff = vcstool.commands.diff:main', + 'vcs-export = vcstool.commands.export:main', + 'vcs-git = vcstool.commands.custom:git_main', + 'vcs-help = vcstool.commands.help:main', + 'vcs-hg = vcstool.commands.custom:hg_main', + 'vcs-import = vcstool.commands.import_:main', + 'vcs-log = vcstool.commands.log:main', + 'vcs-pull = vcstool.commands.pull:main', + 'vcs-push = vcstool.commands.push:main', + 'vcs-remotes = vcstool.commands.remotes:main', + 'vcs-status = vcstool.commands.status:main', + 'vcs-svn = vcstool.commands.custom:svn_main', + 'vcs-validate = vcstool.commands.validate:main', + ] + } +) diff --git a/test/test_commands.py b/test/test_commands.py new file mode 100644 index 0000000..e07a7bc --- /dev/null +++ b/test/test_commands.py @@ -0,0 +1,395 @@ +import os +import subprocess +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from vcstool.util import rmtree # noqa: E402 + +REPOS_FILE = os.path.join(os.path.dirname(__file__), 'list.repos') +REPOS_FILE_URL = \ + 'https://raw.githubusercontent.com/dirk-thomas/vcstool/master/test/list.repos' # noqa: E501 +REPOS2_FILE = os.path.join(os.path.dirname(__file__), 'list2.repos') +TEST_WORKSPACE = os.path.join( + os.path.dirname(os.path.dirname(__file__)), 'test_workspace') + + +class TestCommands(unittest.TestCase): + + @classmethod + def setUpClass(cls): + assert not os.path.exists(TEST_WORKSPACE) + os.makedirs(TEST_WORKSPACE) + + try: + output = run_command( + 'import', ['--input', REPOS_FILE, '.']) + expected = get_expected_output('import') + # newer git versions don't append three dots after the commit hash + assert output == expected or \ + output == expected.replace(b'... ', b' ') + except Exception: + cls.tearDownClass() + raise + + @classmethod + def tearDownClass(cls): + rmtree(TEST_WORKSPACE) + + def test_branch(self): + output = run_command('branch') + expected = get_expected_output('branch') + self.assertEqual(output, expected) + + def test_custom(self): + output = run_command( + 'custom', + args=['--git', '--args', 'describe', '--abbrev=0', '--tags'], + subfolder='immutable') + expected = get_expected_output('custom_describe') + self.assertEqual(output, expected) + + def test_diff(self): + license_path = os.path.join( + TEST_WORKSPACE, 'immutable', 'hash', 'LICENSE') + file_length = None + try: + with open(license_path, 'ab') as h: + file_length = h.tell() + h.write(b'testing') + + output = run_command('diff', args=['--hide']) + expected = get_expected_output('diff_hide') + finally: + if file_length is not None: + with open(license_path, 'ab') as h: + h.truncate(file_length) + + self.assertEqual(output, expected) + + def test_export_exact_with_tags(self): + output = run_command( + 'export', + args=['--exact-with-tags'], + subfolder='immutable') + expected = get_expected_output('export_exact_with_tags') + self.assertEqual(output, expected) + + def test_export_exact(self): + output = run_command( + 'export', + args=['--exact'], + subfolder='immutable') + expected = get_expected_output('export_exact') + self.assertEqual(output, expected) + + def test_log(self): + output = run_command( + 'log', args=['--limit', '2'], subfolder='immutable') + expected = get_expected_output('log_limit') + self.assertEqual(output, expected) + + def test_pull(self): + output = run_command('pull', args=['--workers', '1']) + expected = get_expected_output('pull') + # replace message from older git versions + output = output.replace( + b'anch. Please specify which\nbranch you want to merge with. See', + b'anch.\nPlease specify which branch you want to merge with.\nSee') + # newer git versions warn on pull with default config + if _get_git_version() >= [2, 27, 0]: + pull_warning = b""" +warning: Pulling without specifying how to reconcile divergent branches is +discouraged. You can squelch this message by running one of the following +commands sometime before your next pull: + + git config pull.rebase false # merge (the default strategy) + git config pull.rebase true # rebase + git config pull.ff only # fast-forward only + +You can replace "git config" with "git config --global" to set a default +preference for all repositories. You can also pass --rebase, --no-rebase, +or --ff-only on the command line to override the configured default per +invocation. +""" + output = output.replace(pull_warning, b'') + self.assertEqual(output, expected) + + def test_pull_api(self): + try: + from cStringIO import StringIO + except ImportError: + from io import StringIO + from vcstool.commands.pull import main + stdout_stderr = StringIO() + + # change and restore cwd + cwd_bck = os.getcwd() + os.chdir(TEST_WORKSPACE) + try: + # change and restore USE_COLOR flag + from vcstool import executor + use_color_bck = executor.USE_COLOR + executor.USE_COLOR = False + try: + # change and restore os.environ + env_bck = os.environ + os.environ = dict(os.environ) + os.environ.update( + LANG='en_US.UTF-8', + PYTHONPATH=( + os.path.dirname(os.path.dirname(__file__)) + + os.pathsep + os.environ.get('PYTHONPATH', ''))) + try: + rc = main( + args=['--workers', '1'], + stdout=stdout_stderr, stderr=stdout_stderr) + finally: + os.environ = env_bck + finally: + executor.USE_COLOR = use_color_bck + finally: + os.chdir(cwd_bck) + + assert rc == 0 + # replace message from older git versions + output = stdout_stderr.getvalue().replace( + 'anch. Please specify which\nbranch you want to merge with. See', + 'anch.\nPlease specify which branch you want to merge with.\nSee') + # newer git versions warn on pull with default config + if _get_git_version() >= [2, 27, 0]: + pull_warning = """ +warning: Pulling without specifying how to reconcile divergent branches is +discouraged. You can squelch this message by running one of the following +commands sometime before your next pull: + + git config pull.rebase false # merge (the default strategy) + git config pull.rebase true # rebase + git config pull.ff only # fast-forward only + +You can replace "git config" with "git config --global" to set a default +preference for all repositories. You can also pass --rebase, --no-rebase, +or --ff-only on the command line to override the configured default per +invocation. +""" + output = output.replace(pull_warning, '') + expected = get_expected_output('pull').decode() + assert output == expected + + def test_reimport(self): + cwd_vcstool = os.path.join(TEST_WORKSPACE, 'vcstool') + subprocess.check_output( + ['git', 'remote', 'add', 'foo', 'http://foo.com/bar.git'], + stderr=subprocess.STDOUT, cwd=cwd_vcstool) + cwd_without_version = os.path.join(TEST_WORKSPACE, 'without_version') + subprocess.check_output( + ['git', 'checkout', '-b', 'foo'], + stderr=subprocess.STDOUT, cwd=cwd_without_version) + output = run_command( + 'import', ['--skip-existing', '--input', REPOS_FILE, '.']) + expected = get_expected_output('reimport_skip') + # newer git versions don't append three dots after the commit hash + assert output == expected or output == expected.replace(b'... ', b' ') + + subprocess.check_output( + ['git', 'remote', 'set-url', 'origin', 'http://foo.com/bar.git'], + stderr=subprocess.STDOUT, cwd=cwd_without_version) + run_command( + 'import', ['--skip-existing', '--input', REPOS_FILE, '.']) + + output = run_command( + 'import', ['--force', '--input', REPOS_FILE, '.']) + expected = get_expected_output('reimport_force') + # newer git versions don't append three dots after the commit hash + assert output == expected or output == expected.replace(b'... ', b' ') + + subprocess.check_output( + ['git', 'remote', 'remove', 'foo'], + stderr=subprocess.STDOUT, cwd=cwd_vcstool) + + def test_reimport_failed(self): + cwd_tag = os.path.join(TEST_WORKSPACE, 'immutable', 'tag') + subprocess.check_output( + ['git', 'remote', 'add', 'foo', 'http://foo.com/bar.git'], + stderr=subprocess.STDOUT, cwd=cwd_tag) + subprocess.check_output( + ['git', 'remote', 'rm', 'origin'], + stderr=subprocess.STDOUT, cwd=cwd_tag) + try: + run_command( + 'import', ['--skip-existing', '--input', REPOS_FILE, '.']) + finally: + subprocess.check_output( + ['git', 'remote', 'rm', 'foo'], + stderr=subprocess.STDOUT, cwd=cwd_tag) + subprocess.check_output( + ['git', 'remote', 'add', 'origin', + 'https://github.com/dirk-thomas/vcstool.git'], + stderr=subprocess.STDOUT, cwd=cwd_tag) + + def test_import_force_non_empty(self): + workdir = os.path.join(TEST_WORKSPACE, 'force-non-empty') + os.makedirs(os.path.join(workdir, 'vcstool', 'not-a-git-repo')) + try: + output = run_command( + 'import', ['--force', '--input', REPOS_FILE, '.'], + subfolder='force-non-empty') + expected = get_expected_output('import') + # newer git versions don't append ... after the commit hash + assert ( + output == expected or + output == expected.replace(b'... ', b' ')) + finally: + rmtree(workdir) + + def test_import_shallow(self): + workdir = os.path.join(TEST_WORKSPACE, 'import-shallow') + os.makedirs(workdir) + try: + output = run_command( + 'import', ['--shallow', '--input', REPOS_FILE, '.'], + subfolder='import-shallow') + # the actual output contains absolute paths + output = output.replace( + b'repository in ' + workdir.encode() + b'/', + b'repository in ./') + expected = get_expected_output('import_shallow') + # newer git versions don't append ... after the commit hash + assert ( + output == expected or + output == expected.replace(b'... ', b' ')) + + # check that repository history has only one commit + output = subprocess.check_output( + ['git', 'log', '--format=oneline'], + stderr=subprocess.STDOUT, cwd=os.path.join(workdir, 'vcstool')) + assert len(output.splitlines()) == 1 + finally: + rmtree(workdir) + + def test_import_url(self): + workdir = os.path.join(TEST_WORKSPACE, 'import-url') + os.makedirs(workdir) + try: + output = run_command( + 'import', ['--input', REPOS_FILE_URL, '.'], + subfolder='import-url') + # the actual output contains absolute paths + output = output.replace( + b'repository in ' + workdir.encode() + b'/', + b'repository in ./') + expected = get_expected_output('import') + # newer git versions don't append ... after the commit hash + assert ( + output == expected or + output == expected.replace(b'... ', b' ')) + finally: + rmtree(workdir) + + def test_validate(self): + output = run_command( + 'validate', ['--input', REPOS_FILE]) + expected = get_expected_output('validate') + self.assertEqual(output, expected) + + output = run_command( + 'validate', ['--input', REPOS2_FILE]) + expected = get_expected_output('validate2') + self.assertEqual(output, expected) + + output = run_command( + 'validate', ['--hide-empty', '--input', REPOS_FILE]) + expected = get_expected_output('validate_hide') + self.assertEqual(output, expected) + + def test_remote(self): + output = run_command('remotes', args=['--repos']) + expected = get_expected_output('remotes_repos') + self.assertEqual(output, expected) + + def test_status(self): + output = run_command('status') + # replace message from older git versions + # https://github.com/git/git/blob/3ec7d702a89c647ddf42a59bc3539361367de9d5/Documentation/RelNotes/2.10.0.txt#L373-L374 + output = output.replace( + b'working directory clean', b'working tree clean') + # the following seems to have changed between git 2.10.0 and 2.14.1 + output = output.replace( + b'.\nnothing to commit', b'.\n\nnothing to commit') + expected = get_expected_output('status') + self.assertEqual(output, expected) + + +def run_command(command, args=None, subfolder=None): + repo_root = os.path.dirname(os.path.dirname(__file__)) + script = os.path.join(repo_root, 'scripts', 'vcs-' + command) + env = dict(os.environ) + env.update( + LANG='en_US.UTF-8', + PYTHONPATH=repo_root + os.pathsep + env.get('PYTHONPATH', '')) + cwd = TEST_WORKSPACE + if subfolder: + cwd = os.path.join(cwd, subfolder) + output = subprocess.check_output( + [sys.executable, script] + (args or []), + stderr=subprocess.STDOUT, cwd=cwd, env=env) + # replace message from older git versions + output = output.replace( + b'git checkout -b new_branch_name', + b'git checkout -b <new-branch-name>') + output = output.replace( + b'(detached from ', b'(HEAD detached at ') + output = output.replace( + b"ady on 'master'\n=", + b"ady on 'master'\nYour branch is up-to-date with 'origin/master'.\n=") + output = output.replace( + b'# HEAD detached at ', + b'HEAD detached at ') + output = output.replace( + b'# On branch master', + b"On branch master\nYour branch is up-to-date with 'origin/master'.\n") + # the following seems to have changed between git 2.17.1 and 2.25.1 + output = output.replace( + b"Note: checking out '", b"Note: switching to '") + output = output.replace( + b'by performing another checkout.', + b'by switching back to a branch.') + output = output.replace( + b'using -b with the checkout command again.', + b'using -c with the switch command.') + output = output.replace( + b'git checkout -b <new-branch-name>', + b'git switch -c <new-branch-name>\n\n' + b'Or undo this operation with:\n\n' + b' git switch -\n\n' + b'Turn off this advice by setting config variable ' + b'advice.detachedHead to false') + # replace GitHub SSH clone URL + output = output.replace( + b'git@github.com:', b'https://github.com/') + return output + + +def get_expected_output(name): + path = os.path.join(os.path.dirname(__file__), name + '.txt') + with open(path, 'rb') as h: + content = h.read() + # change in git version 2.15.0 + # https://github.com/git/git/commit/7560f547e6 + if _get_git_version() < [2, 15, 0]: + # use hyphenation for older git versions + content = content.replace(b'up to date', b'up-to-date') + return content + + +def _get_git_version(): + output = subprocess.check_output(['git', '--version']) + prefix = b'git version ' + assert output.startswith(prefix) + output = output[len(prefix):].rstrip() + return [int(x) for x in output.split(b'.') if x != b'windows'] + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_flake8.py b/test/test_flake8.py new file mode 100644 index 0000000..9c40e18 --- /dev/null +++ b/test/test_flake8.py @@ -0,0 +1,66 @@ +import logging +import os + +from flake8 import configure_logging +from flake8.api.legacy import StyleGuide +from flake8.main.application import Application +from pydocstyle.config import log + +log.level = logging.INFO + + +def test_flake8(): + configure_logging(1) + argv = [ + '--extend-ignore=' + ','.join([ + 'A003', 'D100', 'D101', 'D102', 'D103', 'D104', 'D105', 'D107']), + '--exclude', 'vcstool/compat/shutil.py', + '--import-order-style=google'] + style_guide = get_style_guide(argv) + base_path = os.path.join(os.path.dirname(__file__), '..') + paths = [ + os.path.join(base_path, 'setup.py'), + os.path.join(base_path, 'test'), + os.path.join(base_path, 'vcstool'), + ] + scripts_path = os.path.join(base_path, 'scripts') + for script in os.listdir(scripts_path): + if script.startswith('.'): + continue + paths.append(os.path.join(scripts_path, script)) + report = style_guide.check_files(paths) + assert report.total_errors == 0, \ + 'Found %d code style warnings' % report.total_errors + + +def get_style_guide(argv=None): + # this is a fork of flake8.api.legacy.get_style_guide + # to allow passing command line argument + application = Application() + if hasattr(application, 'parse_preliminary_options'): + prelim_opts, remaining_args = application.parse_preliminary_options( + argv) + from flake8 import configure_logging + configure_logging(prelim_opts.verbose, prelim_opts.output_file) + from flake8.options import config + config_finder = config.ConfigFileFinder( + application.program, prelim_opts.append_config, + config_file=prelim_opts.config, + ignore_config_files=prelim_opts.isolated) + application.find_plugins(config_finder) + application.register_plugin_options() + application.parse_configuration_and_cli(config_finder, remaining_args) + else: + application.parse_preliminary_options_and_args([]) + application.make_config_finder() + application.find_plugins() + application.register_plugin_options() + application.parse_configuration_and_cli(argv) + application.make_formatter() + application.make_guide() + application.make_file_checker_manager() + return StyleGuide(application) + + +if __name__ == '__main__': + test_flake8() diff --git a/test/test_options.py b/test/test_options.py new file mode 100644 index 0000000..18a5313 --- /dev/null +++ b/test/test_options.py @@ -0,0 +1,39 @@ +import os +import subprocess +import sys +import unittest + + +class TestOptions(unittest.TestCase): + + def test_clients(self): + output = run_command(['--clients']) + expected = get_expected_output('clients') + self.assertEqual(output, expected) + + def test_commands(self): + output = run_command(['--commands']) + expected = get_expected_output('commands') + self.assertEqual(output, expected) + + +def run_command(args): + repo_root = os.path.dirname(os.path.dirname(__file__)) + script = os.path.join(repo_root, 'scripts', 'vcs') + env = dict(os.environ) + env.update( + LANG='en_US.UTF-8', + PYTHONPATH=repo_root + os.pathsep + env.get('PYTHONPATH', '')) + return subprocess.check_output( + [sys.executable, script] + (args or []), + stderr=subprocess.STDOUT, env=env) + + +def get_expected_output(name): + path = os.path.join(os.path.dirname(__file__), name + '.txt') + with open(path, 'rb') as h: + return h.read() + + +if __name__ == '__main__': + unittest.main() diff --git a/vcstool-completion/vcs.bash b/vcstool-completion/vcs.bash new file mode 100644 index 0000000..7830461 --- /dev/null +++ b/vcstool-completion/vcs.bash @@ -0,0 +1,12 @@ +function _vcs() +{ + local cur + COMPREPLY=() + cur=${COMP_WORDS[COMP_CWORD]} + + if [ $COMP_CWORD -eq 1 ]; then + COMPREPLY=( $( compgen -W "`vcs --commands`" -- $cur )) + fi +} + +complete -o dirnames -F _vcs vcs diff --git a/vcstool-completion/vcs.tcsh b/vcstool-completion/vcs.tcsh new file mode 100644 index 0000000..4ae780c --- /dev/null +++ b/vcstool-completion/vcs.tcsh @@ -0,0 +1 @@ +complete vcs 'p/1/`vcs --commands`/' 'C/*/d/' diff --git a/vcstool-completion/vcs.zsh b/vcstool-completion/vcs.zsh new file mode 100644 index 0000000..c10e247 --- /dev/null +++ b/vcstool-completion/vcs.zsh @@ -0,0 +1,12 @@ +function _vcs() +{ + local opts + reply=() + + if [[ ${CURRENT} == 2 ]]; then + opts=`vcs --commands` + reply=(${=opts}) + fi +} + +compctl -K "_vcs" "vcs" diff --git a/vcstool.egg-info/PKG-INFO b/vcstool.egg-info/PKG-INFO new file mode 100644 index 0000000..8635ad6 --- /dev/null +++ b/vcstool.egg-info/PKG-INFO @@ -0,0 +1,18 @@ +Metadata-Version: 1.2 +Name: vcstool +Version: 0.2.14 +Summary: vcstool provides a command line tool to invoke vcs commands on multiple repositories. +Home-page: https://github.com/dirk-thomas/vcstool +Author: Dirk Thomas +Author-email: web@dirk-thomas.net +Maintainer: Dirk Thomas +Maintainer-email: web@dirk-thomas.net +License: Apache License, Version 2.0 +Download-URL: http://download.ros.org/downloads/vcstool/ +Description: vcstool enables batch commands on multiple different vcs repositories. Currently it supports git, hg, svn and bzr. +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python +Classifier: Topic :: Software Development :: Version Control +Classifier: Topic :: Utilities diff --git a/vcstool.egg-info/SOURCES.txt b/vcstool.egg-info/SOURCES.txt new file mode 100644 index 0000000..0227c3e --- /dev/null +++ b/vcstool.egg-info/SOURCES.txt @@ -0,0 +1,46 @@ +MANIFEST.in +README.rst +setup.py +test/test_commands.py +test/test_flake8.py +test/test_options.py +vcstool/__init__.py +vcstool/crawler.py +vcstool/executor.py +vcstool/streams.py +vcstool/util.py +vcstool-completion/vcs.bash +vcstool-completion/vcs.tcsh +vcstool-completion/vcs.zsh +vcstool.egg-info/PKG-INFO +vcstool.egg-info/SOURCES.txt +vcstool.egg-info/dependency_links.txt +vcstool.egg-info/entry_points.txt +vcstool.egg-info/requires.txt +vcstool.egg-info/top_level.txt +vcstool/clients/__init__.py +vcstool/clients/bzr.py +vcstool/clients/git.py +vcstool/clients/hg.py +vcstool/clients/none.py +vcstool/clients/svn.py +vcstool/clients/tar.py +vcstool/clients/vcs_base.py +vcstool/clients/zip.py +vcstool/commands/__init__.py +vcstool/commands/branch.py +vcstool/commands/command.py +vcstool/commands/custom.py +vcstool/commands/diff.py +vcstool/commands/export.py +vcstool/commands/help.py +vcstool/commands/import_.py +vcstool/commands/log.py +vcstool/commands/pull.py +vcstool/commands/push.py +vcstool/commands/remotes.py +vcstool/commands/status.py +vcstool/commands/validate.py +vcstool/commands/vcs.py +vcstool/compat/__init__.py +vcstool/compat/shutil.py
\ No newline at end of file diff --git a/vcstool.egg-info/dependency_links.txt b/vcstool.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/vcstool.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/vcstool.egg-info/entry_points.txt b/vcstool.egg-info/entry_points.txt new file mode 100644 index 0000000..ddaad00 --- /dev/null +++ b/vcstool.egg-info/entry_points.txt @@ -0,0 +1,19 @@ +[console_scripts] +vcs = vcstool.commands.vcs:main +vcs-branch = vcstool.commands.branch:main +vcs-bzr = vcstool.commands.custom:bzr_main +vcs-custom = vcstool.commands.custom:main +vcs-diff = vcstool.commands.diff:main +vcs-export = vcstool.commands.export:main +vcs-git = vcstool.commands.custom:git_main +vcs-help = vcstool.commands.help:main +vcs-hg = vcstool.commands.custom:hg_main +vcs-import = vcstool.commands.import_:main +vcs-log = vcstool.commands.log:main +vcs-pull = vcstool.commands.pull:main +vcs-push = vcstool.commands.push:main +vcs-remotes = vcstool.commands.remotes:main +vcs-status = vcstool.commands.status:main +vcs-svn = vcstool.commands.custom:svn_main +vcs-validate = vcstool.commands.validate:main + diff --git a/vcstool.egg-info/requires.txt b/vcstool.egg-info/requires.txt new file mode 100644 index 0000000..2760cac --- /dev/null +++ b/vcstool.egg-info/requires.txt @@ -0,0 +1,2 @@ +PyYAML +setuptools diff --git a/vcstool.egg-info/top_level.txt b/vcstool.egg-info/top_level.txt new file mode 100644 index 0000000..b59e4d0 --- /dev/null +++ b/vcstool.egg-info/top_level.txt @@ -0,0 +1 @@ +vcstool diff --git a/vcstool/__init__.py b/vcstool/__init__.py new file mode 100644 index 0000000..cb18086 --- /dev/null +++ b/vcstool/__init__.py @@ -0,0 +1,3 @@ +from .clients import vcstool_clients # noqa + +__version__ = '0.2.14' diff --git a/vcstool/clients/__init__.py b/vcstool/clients/__init__.py new file mode 100644 index 0000000..a71f4fd --- /dev/null +++ b/vcstool/clients/__init__.py @@ -0,0 +1,43 @@ +vcstool_clients = [] + +try: + from .bzr import BzrClient + vcstool_clients.append(BzrClient) +except ImportError: + pass + +try: + from .git import GitClient + vcstool_clients.append(GitClient) +except ImportError: + pass + +try: + from .hg import HgClient + vcstool_clients.append(HgClient) +except ImportError: + pass + +try: + from .svn import SvnClient + vcstool_clients.append(SvnClient) +except ImportError: + pass + +try: + from .tar import TarClient + vcstool_clients.append(TarClient) +except ImportError: + pass + +try: + from .zip import ZipClient + vcstool_clients.append(ZipClient) +except ImportError: + pass + +_client_types = [c.type for c in vcstool_clients] +if len(_client_types) != len(set(_client_types)): + raise RuntimeError( + 'Multiple vcs clients share the same type: ' + + ', '.join(sorted(_client_types))) diff --git a/vcstool/clients/bzr.py b/vcstool/clients/bzr.py new file mode 100644 index 0000000..7804edf --- /dev/null +++ b/vcstool/clients/bzr.py @@ -0,0 +1,203 @@ +import copy +import os + +from .vcs_base import VcsClientBase +from .vcs_base import which +from ..util import rmtree + + +class BzrClient(VcsClientBase): + + type = 'bzr' + _executable = None + + @staticmethod + def is_repository(path): + return os.path.isdir(os.path.join(path, '.bzr')) + + def __init__(self, path): + super(BzrClient, self).__init__(path) + + def branch(self, command): + if command.all: + return self._not_applicable( + command, + message='at least with the option to list all branches') + + self._check_executable() + return self._get_parent_branch() + + def custom(self, command): + self._check_executable() + cmd = [BzrClient._executable] + command.args + return self._run_command(cmd) + + def diff(self, _command): + self._check_executable() + cmd = [BzrClient._executable, 'diff'] + return self._run_command(cmd) + + def import_(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + self._check_executable() + if BzrClient.is_repository(self.path): + # verify that existing repository is the same + result_parent_branch = self._get_parent_branch() + if result_parent_branch['returncode']: + return result_parent_branch + parent_branch = result_parent_branch['output'] + if parent_branch != command.url: + if not command.force: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + 'Path already exists and contains a different ' + 'repository', + 'returncode': 1 + } + try: + rmtree(self.path) + except OSError: + os.remove(self.path) + + not_exist = self._create_path() + if not_exist: + return not_exist + + if BzrClient.is_repository(self.path): + # pull updates for existing repo + cmd_pull = [BzrClient._executable, 'pull'] + return self._run_command(cmd_pull, retry=command.retry) + + else: + cmd_branch = [BzrClient._executable, 'branch'] + if command.version: + cmd_branch += ['-r', command.version] + cmd_branch += [command.url, '.'] + result_branch = self._run_command(cmd_branch, retry=command.retry) + if result_branch['returncode']: + result_branch['output'] = \ + "Could not branch repository '%s': %s" % \ + (command.url, result_branch['output']) + return result_branch + return result_branch + + def log(self, command): + self._check_executable() + if command.limit_tag or command.limit_untagged: + tag = None + if command.limit_tag: + tag = command.limit_tag + else: + # determine nearest tag + cmd_tag = [BzrClient._executable, 'tags', '--sort=time'] + result_tag = self._run_command(cmd_tag) + if result_tag['returncode']: + return result_tag + for line in result_tag['output'].splitlines(): + parts = line.split(' ', 2) + if parts[1] != '?': + tag = parts[0] + if not tag: + result_tag['output'] = 'Could not determine latest tag', + result_tag['returncode'] = 1 + return result_tag + # determine revision number of tag + cmd_tag_rev = [ + BzrClient._executable, 'revno', '--rev', 'tag:' + tag] + result_tag_rev = self._run_command(cmd_tag_rev) + if result_tag_rev['returncode']: + if command.limit_tag: + result_tag_rev['output'] = \ + "Repository lacks the tag '%s'" % tag + return result_tag_rev + try: + tag_rev = int(result_tag_rev['output']) + tag_next_rev = tag_rev + 1 + except ValueError: + tag_rev = result_tag_rev['output'] + tag_next_rev = tag_rev + # determine revision number of HEAD + cmd_head_rev = [BzrClient._executable, 'revno'] + result_head_rev = self._run_command(cmd_head_rev) + if result_head_rev['returncode']: + return result_head_rev + try: + head_rev = int(result_head_rev['output']) + except ValueError: + head_rev = result_head_rev['output'] + # output log since nearest tag + cmd_log = [ + BzrClient._executable, 'log', + '--rev', 'revno:%s..' % str(tag_next_rev)] + if tag_rev == head_rev: + return { + 'cmd': ' '.join(cmd_log), + 'cwd': self.path, + 'output': '', + 'returncode': 0 + } + if command.limit != 0: + cmd_log += ['--limit', '%d' % command.limit] + result_log = self._run_command(cmd_log) + return result_log + cmd = [BzrClient._executable, 'log'] + if command.limit != 0: + cmd += ['--limit', '%d' % command.limit] + return self._run_command(cmd) + + def pull(self, _command): + self._check_executable() + cmd = [BzrClient._executable, 'pull'] + return self._run_command(cmd) + + def push(self, _command): + self._check_executable() + cmd = [BzrClient._executable, 'push'] + return self._run_command(cmd) + + def remotes(self, _command): + self._check_executable() + return self._get_parent_branch() + + def status(self, _command): + self._check_executable() + cmd = [BzrClient._executable, 'status'] + return self._run_command(cmd) + + def _get_parent_branch(self): + cmd = [BzrClient._executable, 'info'] + # parsing the text output requires enforcing language + env = copy.copy(os.environ) + env['LANG'] = 'en_US.UTF-8' + result = self._run_command(cmd, env) + if result['returncode']: + return result + branch = None + prefix = ' parent branch: ' + for line in result['output'].splitlines(): + if line.startswith(prefix): + branch = line[len(prefix):] + break + if not branch: + result['output'] = 'Could not determine parent branch', + result['returncode'] = 1 + return result + result['output'] = branch + return result + + def _check_executable(self): + assert BzrClient._executable is not None, \ + "Could not find 'bzr' executable" + + +if not BzrClient._executable: + BzrClient._executable = which('bzr') diff --git a/vcstool/clients/git.py b/vcstool/clients/git.py new file mode 100644 index 0000000..1be9afc --- /dev/null +++ b/vcstool/clients/git.py @@ -0,0 +1,711 @@ +import os + +from vcstool.executor import USE_COLOR + +from .vcs_base import VcsClientBase +from .vcs_base import which +from ..util import rmtree + + +class GitClient(VcsClientBase): + + type = 'git' + _executable = None + _config_color_is_auto = None + + @staticmethod + def is_repository(path): + return os.path.isdir(os.path.join(path, '.git')) + + def __init__(self, path): + super(GitClient, self).__init__(path) + + def branch(self, command): + self._check_executable() + cmd = [GitClient._executable, 'branch'] + result = self._run_command(cmd) + + if not command.all and not result['returncode']: + # only show current branch + lines = result['output'].splitlines() + lines = [line[2:] for line in lines if line.startswith('* ')] + result['output'] = '\n'.join(lines) + + return result + + def custom(self, command): + self._check_executable() + cmd = [GitClient._executable] + command.args + return self._run_command(cmd) + + def diff(self, command): + self._check_executable() + cmd = [GitClient._executable, 'diff'] + self._check_color(cmd) + if command.context: + cmd += ['--unified=%d' % command.context] + return self._run_command(cmd) + + def export(self, command): + self._check_executable() + exact = command.exact + if not exact: + # determine if a specific branch is checked out or ec is detached + cmd_branch = [ + GitClient._executable, 'rev-parse', '--abbrev-ref', 'HEAD'] + result_branch = self._run_command(cmd_branch) + if result_branch['returncode']: + result_branch['output'] = 'Could not determine ref: ' + \ + result_branch['output'] + return result_branch + branch_name = result_branch['output'] + exact = branch_name == 'HEAD' # is detached + + if not exact: + # determine the remote of the current branch + cmd_remote = [ + GitClient._executable, 'rev-parse', '--abbrev-ref', + '@{upstream}'] + result_remote = self._run_command(cmd_remote) + if result_remote['returncode']: + result_remote['output'] = 'Could not determine ref: ' + \ + result_remote['output'] + return result_remote + branch_with_remote = result_remote['output'] + + # determine remote + suffix = '/' + branch_name + assert branch_with_remote.endswith(branch_name), \ + "'%s' does not end with '%s'" % \ + (branch_with_remote, branch_name) + remote = branch_with_remote[:-len(suffix)] + + # if a local ref exists with the same name as the remote branch + # the result will be prefixed to make it unambiguous + prefix = 'remotes/' + if remote.startswith(prefix): + remote = remote[len(prefix):] + + # determine url of remote + result_url = self._get_remote_url(remote) + if result_url['returncode']: + return result_url + url = result_url['output'] + + # the result is the remote url and the branch name + return { + 'cmd': ' && '.join([ + result_branch['cmd'], result_remote['cmd'], + result_url['cmd']]), + 'cwd': self.path, + 'output': '\n'.join([url, branch_name]), + 'returncode': 0, + 'export_data': {'url': url, 'version': branch_name} + } + + else: + # determine the hash + cmd_ref = [GitClient._executable, 'rev-parse', 'HEAD'] + result_ref = self._run_command(cmd_ref) + if result_ref['returncode']: + result_ref['output'] = 'Could not determine ref: ' + \ + result_ref['output'] + return result_ref + ref = result_ref['output'] + + # get all remote names + cmd_remotes = [GitClient._executable, 'remote'] + result_remotes = self._run_command(cmd_remotes) + if result_remotes['returncode']: + result_remotes['output'] = 'Could not determine remotes: ' + \ + result_remotes['output'] + return result_remotes + remotes = result_remotes['output'].splitlines() + + # prefer origin and upstream remotes + if 'upstream' in remotes: + remotes.remove('upstream') + remotes.insert(0, 'upstream') + if 'origin' in remotes: + remotes.remove('origin') + remotes.insert(0, 'origin') + + # for each remote name check if the hash is part of the remote + for remote in remotes: + # get all remote names + cmd_refs = [ + GitClient._executable, 'rev-list', '--remotes=' + remote, + '--tags'] + result_refs = self._run_command(cmd_refs) + if result_refs['returncode']: + result_refs['output'] = \ + "Could not determine refs of remote '%s': " % \ + remote + result_refs['output'] + return result_refs + refs = result_refs['output'].splitlines() + if ref not in refs: + continue + + cmds = [result_ref['cmd']] + if command.with_tags: + # check if there is exactly one tag pointing to that ref + cmd_tags = [ + GitClient._executable, 'tag', '--points-at', ref] + result_tags = self._run_command(cmd_tags) + if result_tags['returncode']: + result_tags['output'] = \ + "Could not determine tags for ref '%s': " % \ + ref + result_tags['output'] + return result_tags + cmds.append(result_tags['cmd']) + tags = result_tags['output'].splitlines() + if len(tags) == 1: + tag = tags[0] + # double check that the tag is part of the remote + # and references the same hash + cmd_ls_remote = [ + GitClient._executable, 'ls-remote', remote, + 'refs/tags/' + tag] + result_ls_remote = self._run_command(cmd_ls_remote) + if result_ls_remote['returncode']: + result_ls_remote['output'] = \ + "Could not check remote tags for '%s': " % \ + remote + result_ls_remote['output'] + return result_ls_remote + matches = self._get_hash_ref_tuples( + result_ls_remote['output']) + if len(matches) == 1 and matches[0][0] == ref: + ref = tag + + # determine url of remote + result_url = self._get_remote_url(remote) + if result_url['returncode']: + return result_url + url = result_url['output'] + cmds.append(result_url['cmd']) + + # the result is the remote url and the hash/tag + return { + 'cmd': ' && '.join(cmds), + 'cwd': self.path, + 'output': '\n'.join([url, ref]), + 'returncode': 0, + 'export_data': {'url': url, 'version': ref} + } + + return { + 'cmd': ' && '.join([result_ref['cmd'], result_remotes['cmd']]), + 'cwd': self.path, + 'output': "Could not determine remote containing '%s'" % ref, + 'returncode': 1, + } + + def _get_remote_url(self, remote): + cmd_url = [ + GitClient._executable, 'config', '--get', 'remote.%s.url' % remote] + result_url = self._run_command(cmd_url) + if result_url['returncode']: + result_url['output'] = 'Could not determine remote url: ' + \ + result_url['output'] + return result_url + + def import_(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + self._check_executable() + if GitClient.is_repository(self.path): + # verify that existing repository is the same + result_urls = self._get_remote_urls() + if result_urls['returncode']: + return result_urls + for url, remote in result_urls['output']: + if url == command.url: + break + else: + if command.skip_existing: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + 'Skipped existing repository with different URL', + 'returncode': 0 + } + if not command.force: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + 'Path already exists and contains a different ' + 'repository', + 'returncode': 1 + } + try: + rmtree(self.path) + except OSError: + os.remove(self.path) + + elif command.skip_existing and os.path.exists(self.path): + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'Skipped existing directory', + 'returncode': 0 + } + + elif command.force and os.path.exists(self.path): + # Not empty, not a git repository + try: + rmtree(self.path) + except OSError: + os.remove(self.path) + + not_exist = self._create_path() + if not_exist: + return not_exist + + if GitClient.is_repository(self.path): + if command.skip_existing: + checkout_version = None + elif command.version: + checkout_version = command.version + else: + # determine remote HEAD branch + cmd_remote = [GitClient._executable, 'remote', 'show', remote] + # override locale in order to parse output + env = os.environ.copy() + env['LC_ALL'] = 'C' + result_remote = self._run_command(cmd_remote, env=env) + if result_remote['returncode']: + result_remote['output'] = \ + 'Could not get remote information of repository ' \ + "'%s': %s" % (url, result_remote['output']) + return result_remote + prefix = ' HEAD branch: ' + for line in result_remote['output'].splitlines(): + if line.startswith(prefix): + checkout_version = line[len(prefix):] + break + else: + result_remote['returncode'] = 1 + result_remote['output'] = \ + 'Could not determine remote HEAD branch of ' \ + "repository '%s': %s" % (url, result_remote['output']) + return result_remote + + # fetch updates for existing repo + cmd_fetch = [GitClient._executable, 'fetch', remote] + if command.shallow: + result_version_type = self._check_version_type( + command.url, checkout_version) + if result_version_type['returncode']: + return result_version_type + version_type = result_version_type['version_type'] + if version_type == 'branch': + cmd_fetch.append( + 'refs/heads/%s:refs/remotes/%s/%s' % + (checkout_version, remote, checkout_version)) + elif version_type == 'hash': + cmd_fetch.append(checkout_version) + elif version_type == 'tag': + cmd_fetch.append( + '+refs/tags/%s:refs/tags/%s' % + (checkout_version, checkout_version)) + else: + assert False + cmd_fetch += ['--depth', '1'] + result_fetch = self._run_command(cmd_fetch, retry=command.retry) + if result_fetch['returncode']: + return result_fetch + cmd = result_fetch['cmd'] + output = result_fetch['output'] + + # ensure that a tracking branch exists which can be checked out + if command.shallow and version_type == 'branch': + cmd_show_ref = [ + GitClient._executable, 'show-ref', + 'refs/heads/%s' % checkout_version] + result_show_ref = self._run_command(cmd_show_ref) + if result_show_ref['returncode']: + # creating tracking branch + cmd_branch = [ + GitClient._executable, 'branch', checkout_version, + '%s/%s' % (remote, checkout_version)] + result_branch = self._run_command(cmd_branch) + if result_branch['returncode']: + result_branch['output'] = \ + "Could not create branch '%s': %s" % \ + (checkout_version, result_branch['output']) + return result_branch + cmd += ' && ' + ' '.join(cmd_branch) + output = '\n'.join([output, result_branch['output']]) + + else: + version_type = None + if command.version: + result_version_type = self._check_version_type( + command.url, command.version) + if result_version_type['returncode']: + return result_version_type + version_type = result_version_type['version_type'] + + if not command.shallow or version_type in (None, 'branch'): + cmd_clone = [GitClient._executable, 'clone', command.url, '.'] + if version_type == 'branch': + cmd_clone += ['-b', command.version] + checkout_version = None + else: + checkout_version = command.version + if command.shallow: + cmd_clone += ['--depth', '1'] + result_clone = self._run_command( + cmd_clone, retry=command.retry) + if result_clone['returncode']: + result_clone['output'] = \ + "Could not clone repository '%s': %s" % \ + (command.url, result_clone['output']) + return result_clone + cmd = result_clone['cmd'] + output = result_clone['output'] + else: + # getting a hash or tag with a depth of 1 can't use 'clone' + cmd_init = [GitClient._executable, 'init'] + result_init = self._run_command(cmd_init) + if result_init['returncode']: + return result_init + cmd = result_init['cmd'] + output = result_init['output'] + + cmd_remote_add = [ + GitClient._executable, 'remote', 'add', 'origin', + command.url] + result_remote_add = self._run_command(cmd_remote_add) + if result_remote_add['returncode']: + return result_remote_add + cmd += ' && ' + ' '.join(cmd_remote_add) + output = '\n'.join([output, result_remote_add['output']]) + + cmd_fetch = [GitClient._executable, 'fetch', 'origin'] + if version_type == 'hash': + cmd_fetch.append(command.version) + elif version_type == 'tag': + cmd_fetch.append( + 'refs/tags/%s:refs/tags/%s' % + (command.version, command.version)) + else: + assert False + cmd_fetch += ['--depth', '1'] + result_fetch = self._run_command( + cmd_fetch, retry=command.retry) + if result_fetch['returncode']: + return result_fetch + cmd += ' && ' + ' '.join(cmd_fetch) + output = '\n'.join([output, result_fetch['output']]) + + checkout_version = command.version + + if checkout_version: + cmd_checkout = [ + GitClient._executable, 'checkout', checkout_version, '--'] + result_checkout = self._run_command(cmd_checkout) + if result_checkout['returncode']: + result_checkout['output'] = \ + "Could not checkout ref '%s': %s" % \ + (checkout_version, result_checkout['output']) + return result_checkout + cmd += ' && ' + ' '.join(cmd_checkout) + output = '\n'.join([output, result_checkout['output']]) + + if command.recursive: + cmd_submodule = [ + GitClient._executable, 'submodule', 'update', '--init'] + result_submodule = self._run_command(cmd_submodule) + if result_submodule['returncode']: + result_submodule['output'] = \ + 'Could not init/update submodules: %s' % \ + result_submodule['output'] + return result_submodule + cmd += ' && ' + ' '.join(cmd_submodule) + output = '\n'.join([output, result_submodule['output']]) + + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': output, + 'returncode': 0 + } + + def _get_remote_urls(self): + cmd_remote = [GitClient._executable, 'remote', 'show'] + result_remote = self._run_command(cmd_remote) + if result_remote['returncode']: + result_remote['output'] = 'Could not determine remotes: ' + \ + result_remote['output'] + return result_remote + remote_urls = [] + cmd = result_remote['cmd'] + for remote in result_remote['output'].splitlines(): + result_url = self._get_remote_url(remote) + cmd += ' && ' + result_url['cmd'] + if not result_url['returncode']: + remote_urls.append((result_url['output'], remote)) + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': (remote_urls if remote_urls else + 'Could not determine any of the remote urls'), + 'returncode': 0 if remote_urls else 1 + } + + def _check_version_type(self, url, version): + cmd = [GitClient._executable, 'ls-remote', url, version] + result = self._run_command(cmd) + if result['returncode']: + result['output'] = 'Could not determine ref type of version: ' + \ + result['output'] + return result + if not result['output']: + result['version_type'] = 'hash' + return result + + refs = {} + for hash_, ref in self._get_hash_ref_tuples(result['output']): + refs[ref] = hash_ + + tag_ref = 'refs/tags/' + version + branch_ref = 'refs/heads/' + version + if tag_ref in refs and branch_ref in refs: + if refs[tag_ref] != refs[branch_ref]: + result['returncode'] = 1 + result['output'] = 'The version ref is a branch as well as ' \ + 'tag but with different hashes' + return result + if tag_ref in refs: + result['version_type'] = 'tag' + elif branch_ref in refs: + result['version_type'] = 'branch' + else: + result['version_type'] = 'hash' + return result + + def log(self, command): + self._check_executable() + if command.limit_tag: + # check if specific tag exists + cmd_tag = [GitClient._executable, 'tag', '-l', command.limit_tag] + result_tag = self._run_command(cmd_tag) + if result_tag['returncode']: + return result_tag + if not result_tag['output']: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Repository lacks the tag '%s'" % command.limit_tag, + 'returncode': 1 + } + # output log since specific tag + cmd = [GitClient._executable, 'log', '%s..' % command.limit_tag] + elif command.limit_untagged: + # determine nearest tag + cmd_tag = [ + GitClient._executable, 'describe', '--abbrev=0', '--tags'] + result_tag = self._run_command(cmd_tag) + if result_tag['returncode']: + return result_tag + # output log since nearest tag + cmd = [GitClient._executable, 'log', '%s..' % result_tag['output']] + else: + cmd = [GitClient._executable, 'log'] + cmd += ['--decorate'] + if command.limit != 0: + cmd += ['-%d' % command.limit] + if not command.verbose: + cmd += ['--pretty=short'] + self._check_color(cmd) + return self._run_command(cmd) + + def pull(self, _command): + self._check_executable() + cmd = [GitClient._executable, 'pull'] + self._check_color(cmd) + result = self._run_command(cmd) + + if result['returncode']: + # check for detached HEAD + cmd_rev_parse = [ + GitClient._executable, 'rev-parse', '--abbrev-ref', 'HEAD'] + result_rev_parse = self._run_command(cmd_rev_parse) + if result_rev_parse['returncode']: + result_rev_parse['output'] = 'Could not determine ref: ' + \ + result_rev_parse['output'] + return result_rev_parse + detached = result_rev_parse['output'] == 'HEAD' + + if detached: + # warn but not fail about the inability to pull a detached head + return { + 'cmd': '', + 'cwd': self.path, + 'output': result['output'], + 'returncode': 0, + } + + return result + + def push(self, _command): + self._check_executable() + cmd = [GitClient._executable, 'push'] + return self._run_command(cmd) + + def remotes(self, _command): + self._check_executable() + cmd = [GitClient._executable, 'remote', '-v'] + return self._run_command(cmd) + + def status(self, command): + self._check_executable() + while command.hide_empty: + # check if ahead + cmd = [GitClient._executable, 'log', '@{push}..'] + result = self._run_command(cmd) + if not result['returncode'] and result['output']: + # ahead, do not hide + break + # check if behind + cmd = [GitClient._executable, 'log', '..@{upstream}'] + result = self._run_command(cmd) + if not result['returncode'] and result['output']: + # behind, do not hide + break + cmd = [GitClient._executable, 'status', '-s'] + if command.quiet: + cmd += ['--untracked-files=no'] + result = self._run_command(cmd) + if result['returncode'] or not result['output']: + return result + break + cmd = [GitClient._executable, 'status'] + self._check_color(cmd) + if command.quiet: + cmd += ['--untracked-files=no'] + return self._run_command(cmd) + + def validate(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + self._check_executable() + + cmd_ls_remote = [GitClient._executable, 'ls-remote'] + cmd_ls_remote += ['-q', '--exit-code'] + cmd_ls_remote += [command.url] + env = os.environ.copy() + env['GIT_TERMINAL_PROMPT'] = '0' + result_ls_remote = self._run_command( + cmd_ls_remote, + retry=command.retry, + env=env) + if result_ls_remote['returncode']: + result_ls_remote['output'] = \ + "Failed to contact remote repository '%s': %s" % \ + (command.url, result_ls_remote['output']) + return result_ls_remote + + if command.version: + hashes = [] + refs = [] + + for hash_and_ref in self._get_hash_ref_tuples( + result_ls_remote['output'] + ): + hashes.append(hash_and_ref[0]) + + # ignore pull request refs + if not hash_and_ref[1].startswith('refs/pull/'): + if hash_and_ref[1].startswith('refs/tags/'): + refs.append(hash_and_ref[1][10:]) + elif hash_and_ref[1].startswith('refs/heads/'): + refs.append(hash_and_ref[1][11:]) + else: + refs.append(hash_and_ref[1]) + + # test for refs first + ref_found = command.version in refs + + if not ref_found: + for _hash in hashes: + if _hash.startswith(command.version): + ref_found = True + break + + if not ref_found: + cmd = result_ls_remote['cmd'] + output = "Found git repository '%s' but " % command.url + \ + 'unable to verify non-branch / non-tag ref ' + \ + "'%s' without cloning the repo" % command.version + + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': output, + 'returncode': 0 + } + else: + cmd = result_ls_remote['cmd'] + output = "Found git repository '%s' with ref '%s'" % \ + (command.url, command.version) + else: + cmd = result_ls_remote['cmd'] + output = "Found git repository '%s' with default branch" % \ + command.url + + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': output, + 'returncode': None + } + + def _check_color(self, cmd): + if not USE_COLOR: + return + # check if user uses colorization + if GitClient._config_color_is_auto is None: + _cmd = [GitClient._executable, 'config', '--get', 'color.ui'] + result = self._run_command(_cmd) + GitClient._config_color_is_auto = result['output'] in ['', 'auto'] + + # inject arguments to force colorization + if GitClient._config_color_is_auto: + cmd[1:1] = '-c', 'color.ui=always' + + def _check_executable(self): + assert GitClient._executable is not None, \ + "Could not find 'git' executable" + + def _get_hash_ref_tuples(self, ls_remote_output): + tuples = [] + for line in ls_remote_output.splitlines(): + if line.startswith('#'): + continue + try: + hash_, ref = line.split(None, 1) + except ValueError: + continue + tuples.append((hash_, ref)) + return tuples + + +if not GitClient._executable: + GitClient._executable = which('git') diff --git a/vcstool/clients/hg.py b/vcstool/clients/hg.py new file mode 100644 index 0000000..fda3699 --- /dev/null +++ b/vcstool/clients/hg.py @@ -0,0 +1,333 @@ +import os +from threading import Lock + +from vcstool.executor import USE_COLOR + +from .vcs_base import VcsClientBase +from .vcs_base import which +from ..util import rmtree + + +class HgClient(VcsClientBase): + + type = 'hg' + _executable = None + _config_color = None + _config_color_lock = Lock() + + @staticmethod + def is_repository(path): + return os.path.isdir(os.path.join(path, '.hg')) + + def __init__(self, path): + super(HgClient, self).__init__(path) + + def branch(self, command): + self._check_executable() + cmd = [HgClient._executable, 'branches' if command.all else 'branch'] + self._check_color(cmd) + return self._run_command(cmd) + + def custom(self, command): + self._check_executable() + cmd = [HgClient._executable] + command.args + return self._run_command(cmd) + + def diff(self, command): + self._check_executable() + cmd = [HgClient._executable, 'diff'] + self._check_color(cmd) + if command.context: + cmd += ['--unified %d' % command.context] + return self._run_command(cmd) + + def export(self, command): + self._check_executable() + result_url = self._get_url() + if result_url['returncode']: + return result_url + url = result_url['output'] + + cmd_id = [HgClient._executable, 'identify', '--id'] + result_id = self._run_command(cmd_id) + if result_id['returncode']: + result_id['output'] = \ + 'Could not determine id: ' + result_id['output'] + return result_id + id_ = result_id['output'] + + if not command.exact: + cmd_branch = [HgClient._executable, 'identify', '--branch'] + result_branch = self._run_command(cmd_branch) + if result_branch['returncode']: + result_branch['output'] = \ + 'Could not determine branch: ' + result_branch['output'] + return result_branch + branch = result_branch['output'] + + cmd_branch_id = [ + HgClient._executable, 'identify', '-r', branch, '--id'] + result_branch_id = self._run_command(cmd_branch_id) + if result_branch_id['returncode']: + result_branch_id['output'] = \ + 'Could not determine branch id: ' + \ + result_branch_id['output'] + return result_branch_id + if result_branch_id['output'] == id_: + id_ = branch + cmd_branch = cmd_branch_id + + return { + 'cmd': '%s && %s' % (result_url['cmd'], ' '.join(cmd_id)), + 'cwd': self.path, + 'output': '\n'.join([url, id_]), + 'returncode': 0, + 'export_data': {'url': url, 'version': id_} + } + + def _get_url(self): + cmd_url = [HgClient._executable, 'paths', 'default'] + result_url = self._run_command(cmd_url) + if result_url['returncode']: + result_url['output'] = \ + 'Could not determine url: ' + result_url['output'] + return result_url + return result_url + + def import_(self, command): + if not command.url or not command.version: + if not command.url and not command.version: + value_missing = "'url' and 'version'" + elif not command.url: + value_missing = "'url'" + else: + value_missing = "'version'" + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'Repository data lacks the %s value' % value_missing, + 'returncode': 1 + } + + self._check_executable() + if HgClient.is_repository(self.path): + # verify that existing repository is the same + result_url = self._get_url() + if result_url['returncode']: + return result_url + url = result_url['output'] + if url != command.url: + if not command.force: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + 'Path already exists and contains a different ' + 'repository', + 'returncode': 1 + } + try: + rmtree(self.path) + except OSError: + os.remove(self.path) + + not_exist = self._create_path() + if not_exist: + return not_exist + + if HgClient.is_repository(self.path): + # pull updates for existing repo + cmd_pull = [ + HgClient._executable, '--noninteractive', 'pull', '--update'] + result_pull = self._run_command(cmd_pull, retry=command.retry) + if result_pull['returncode']: + return result_pull + cmd = result_pull['cmd'] + output = result_pull['output'] + + else: + cmd_clone = [ + HgClient._executable, '--noninteractive', 'clone', command.url, + '.'] + result_clone = self._run_command(cmd_clone, retry=command.retry) + if result_clone['returncode']: + result_clone['output'] = \ + "Could not clone repository '%s': %s" % \ + (command.url, result_clone['output']) + return result_clone + cmd = result_clone['cmd'] + output = result_clone['output'] + + if command.version: + cmd_checkout = [ + HgClient._executable, '--noninteractive', 'checkout', + command.version] + result_checkout = self._run_command(cmd_checkout) + if result_checkout['returncode']: + result_checkout['output'] = \ + "Could not checkout '%s': %s" % \ + (command.version, result_checkout['output']) + return result_checkout + cmd += ' && ' + ' '.join(cmd_checkout) + output = '\n'.join([output, result_checkout['output']]) + + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': output, + 'returncode': 0 + } + + def log(self, command): + self._check_executable() + if command.limit_tag: + # check if specific tag exists + cmd_log = [ + HgClient._executable, 'log', + '--rev', 'tag(%s)' % command.limit_tag] + result_log = self._run_command(cmd_log) + if result_log['returncode']: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Repository lacks the tag '%s'" % command.limit_tag, + 'returncode': 1 + } + # output log since specific tag + cmd = [ + HgClient._executable, 'log', '--rev', + 'sort(tag(%s)::, -rev) and not tag (%s)' % + (command.limit_tag, command.limit_tag)] + elif command.limit_untagged: + # determine distance to nearest tag + cmd_log = [ + HgClient._executable, 'log', + '--rev', '.', '--template', '{latesttagdistance}'] + result_log = self._run_command(cmd_log) + if result_log['returncode']: + return result_log + # output log since nearest tag + cmd = [ + HgClient._executable, 'log', + '--limit', result_log['output'], '-b', '.'] + else: + cmd = [HgClient._executable, 'log'] + if command.limit != 0: + cmd += ['--limit', '%d' % command.limit] + if command.verbose: + cmd += ['--verbose'] + self._check_color(cmd) + return self._run_command(cmd) + + def pull(self, _command): + self._check_executable() + cmd = [HgClient._executable, '--noninteractive', 'pull', '--update'] + self._check_color(cmd) + return self._run_command(cmd) + + def push(self, _command): + self._check_executable() + cmd = [HgClient._executable, '--noninteractive', 'push'] + return self._run_command(cmd) + + def remotes(self, _command): + self._check_executable() + cmd = [HgClient._executable, 'paths'] + return self._run_command(cmd) + + def status(self, command): + self._check_executable() + cmd = [HgClient._executable, 'status'] + self._check_color(cmd) + if command.quiet: + cmd += ['--untracked-files=no'] + return self._run_command(cmd) + + def validate(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + self._check_executable() + + cmd_id_repo = [ + HgClient._executable, '--noninteractive', 'identify', + command.url] + result_id_repo = self._run_command( + cmd_id_repo, + retry=command.retry) + if result_id_repo['returncode']: + result_id_repo['output'] = \ + "Failed to contact remote repository '%s': %s" % \ + (command.url, result_id_repo['output']) + return result_id_repo + + if command.version: + cmd_id_ver = [ + HgClient._executable, '--noninteractive', 'identify', + '-r', command.version, command.url] + result_id_ver = self._run_command( + cmd_id_ver, + retry=command.retry) + if result_id_ver['returncode']: + result_id_ver['output'] = \ + 'Specified version not found on remote repository ' + \ + "'%s':'%s' : %s" % \ + (command.url, command.version, result_id_ver['output']) + return result_id_ver + + cmd = result_id_ver['cmd'] + output = "Found hg repository '%s' with changeset '%s'" % \ + (command.url, command.version) + else: + cmd = result_id_repo['cmd'] + output = "Found hg repository '%s' with default branch" % \ + command.url + + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': output, + 'returncode': None + } + + def _check_color(self, cmd): + if not USE_COLOR: + return + with HgClient._config_color_lock: + # check if user uses colorization + if HgClient._config_color is None: + HgClient._config_color = False + # check if config extension is available + _cmd = [HgClient._executable, 'config', '--help'] + result = self._run_command(_cmd) + if result['returncode']: + return + # check if color extension is available and not disabled + _cmd = [HgClient._executable, 'config', 'extensions.color'] + result = self._run_command(_cmd) + if result['returncode'] or result['output'].startswith('!'): + return + # check if color mode is not off or not set + _cmd = [HgClient._executable, 'config', 'color.mode'] + result = self._run_command(_cmd) + if not result['returncode'] and result['output'] == 'off': + return + HgClient._config_color = True + + # inject arguments to force colorization + if HgClient._config_color: + cmd[1:1] = '--color', 'always' + + def _check_executable(self): + assert HgClient._executable is not None, \ + "Could not find 'hg' executable" + + +if not HgClient._executable: + HgClient._executable = which('hg') diff --git a/vcstool/clients/none.py b/vcstool/clients/none.py new file mode 100644 index 0000000..a64d853 --- /dev/null +++ b/vcstool/clients/none.py @@ -0,0 +1,9 @@ +from .vcs_base import VcsClientBase + + +class NoneClient(VcsClientBase): + + type = 'none' + + def __init__(self, path): + super(NoneClient, self).__init__(path) diff --git a/vcstool/clients/svn.py b/vcstool/clients/svn.py new file mode 100644 index 0000000..9a3dcb8 --- /dev/null +++ b/vcstool/clients/svn.py @@ -0,0 +1,269 @@ +import os +from xml.etree.ElementTree import fromstring + +from .vcs_base import VcsClientBase, which + + +class SvnClient(VcsClientBase): + + type = 'svn' + _executable = None + + @staticmethod + def is_repository(path): + return os.path.isdir(os.path.join(path, '.svn')) + + def __init__(self, path): + super(SvnClient, self).__init__(path) + + def branch(self, command): + if command.all: + return self._not_applicable( + command, + message='at least with the option to list all branches') + + self._check_executable() + cmd_info = [SvnClient._executable, 'info', '--xml'] + result_info = self._run_command(cmd_info) + if result_info['returncode']: + result_info['output'] = \ + 'Could not determine url: ' + result_info['output'] + return result_info + info = result_info['output'] + + try: + root = fromstring(info) + entry = root.find('entry') + url = entry.findtext('url') + repository = entry.find('repository') + root_url = repository.findtext('root') + except Exception as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'Could not determine url from xml: %s' % e, + 'returncode': 1 + } + + if not url.startswith(root_url): + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Could not determine url suffix. The root url '%s' is not " + "a prefix of the url '%s'" % (root_url, url), + 'returncode': 1 + } + + return { + 'cmd': ' '.join(cmd_info), + 'cwd': self.path, + 'output': url[len(root_url):], + 'returncode': 0, + } + + def custom(self, command): + self._check_executable() + cmd = [SvnClient._executable] + command.args + return self._run_command(cmd) + + def diff(self, command): + self._check_executable() + cmd = [SvnClient._executable, 'diff'] + if command.context: + cmd += ['--unified=%d' % command.context] + return self._run_command(cmd) + + def export(self, command): + self._check_executable() + cmd_info = [SvnClient._executable, 'info', '--xml'] + result_info = self._run_command(cmd_info) + if result_info['returncode']: + result_info['output'] = \ + 'Could not determine url: ' + result_info['output'] + return result_info + info = result_info['output'] + + try: + root = fromstring(info) + entry = root.find('entry') + url = entry.findtext('url') + revision = entry.get('revision') + except Exception as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'Could not determine url from xml: %s' % e, + 'returncode': 1 + } + + export_data = {'url': url} + if command.exact: + export_data['version'] = revision + return { + 'cmd': ' '.join(cmd_info), + 'cwd': self.path, + 'output': url, + 'returncode': 0, + 'export_data': export_data + } + + def import_(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + not_exist = self._create_path() + if not_exist: + return not_exist + + self._check_executable() + + url = command.url + if command.version: + url += '@%s' % command.version + + cmd_checkout = [ + SvnClient._executable, '--non-interactive', 'checkout', url, '.'] + result_checkout = self._run_command(cmd_checkout, retry=command.retry) + if result_checkout['returncode']: + result_checkout['output'] = \ + "Could not checkout repository '%s': %s" % \ + (command.url, result_checkout['output']) + return result_checkout + + return { + 'cmd': ' '.join(cmd_checkout), + 'cwd': self.path, + 'output': result_checkout['output'], + 'returncode': 0 + } + + def log(self, command): + if command.limit_tag: + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'SvnClient can not determine log since tag', + 'returncode': NotImplemented + } + if command.limit_untagged: + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'SvnClient can not determine latest tag', + 'returncode': NotImplemented + } + self._check_executable() + cmd = [SvnClient._executable, 'log'] + if command.limit != 0: + cmd += ['--limit', '%d' % command.limit] + return self._run_command(cmd) + + def pull(self, _command): + self._check_executable() + cmd = [SvnClient._executable, '--non-interactive', 'update'] + return self._run_command(cmd) + + def push(self, command): + self._check_executable() + return self._not_applicable(command) + + def remotes(self, _command): + self._check_executable() + cmd_info = [SvnClient._executable, 'info', '--xml'] + result_info = self._run_command(cmd_info) + if result_info['returncode']: + result_info['output'] = \ + 'Could not determine url: ' + result_info['output'] + return result_info + info = result_info['output'] + + try: + root = fromstring(info) + entry = root.find('entry') + url = entry.findtext('url') + except Exception as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': 'Could not determine url from xml: %s' % e, + 'returncode': 1 + } + + return { + 'cmd': ' '.join(cmd_info), + 'cwd': self.path, + 'output': url, + 'returncode': 0, + } + + def status(self, command): + self._check_executable() + cmd = [SvnClient._executable, 'status'] + if command.quiet: + cmd += ['--quiet'] + return self._run_command(cmd) + + def validate(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + self._check_executable() + + cmd_info_repo = [SvnClient._executable, 'info', command.url] + result_info_repo = self._run_command( + cmd_info_repo, + retry=command.retry) + if result_info_repo['returncode']: + result_info_repo['output'] = \ + "Failed to contact remote repository '%s': %s" % \ + (command.url, result_info_repo['output']) + return result_info_repo + + if command.version: + cmd_info_ver = [ + SvnClient._executable, 'info', + command.url + '@' + command.version] + result_info_ver = self._run_command( + cmd_info_ver, + retry=command.retry) + + if result_info_ver['returncode']: + result_info_ver['output'] = \ + 'Specified version not found on remote repository' + \ + "'%s@%s' : %s" % \ + (command.url, command.version, result_info_ver['output']) + return result_info_ver + + cmd = result_info_ver['cmd'] + output = "Found svn repository '%s' with revision '%s'" % \ + (command.url, command.version) + else: + cmd = result_info_repo['cmd'] + output = "Found svn repository '%s' with default branch" % \ + command.url + + return { + 'cmd': cmd, + 'cwd': self.path, + 'output': output, + 'returncode': None + } + + def _check_executable(self): + assert SvnClient._executable is not None, \ + "Could not find 'svn' executable" + + +if not SvnClient._executable: + SvnClient._executable = which('svn') diff --git a/vcstool/clients/tar.py b/vcstool/clients/tar.py new file mode 100644 index 0000000..9894230 --- /dev/null +++ b/vcstool/clients/tar.py @@ -0,0 +1,124 @@ +import os +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO +import tarfile +try: + from urllib.error import URLError +except ImportError: + from urllib2 import URLError + +from .vcs_base import load_url +from .vcs_base import test_url +from .vcs_base import VcsClientBase +from ..util import rmtree + + +class TarClient(VcsClientBase): + + type = 'tar' + + @staticmethod + def is_repository(path): + return False + + def __init__(self, path): + super(TarClient, self).__init__(path) + + def import_(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + # clear destination + if os.path.exists(self.path): + for filename in os.listdir(self.path): + path = os.path.join(self.path, filename) + try: + rmtree(path) + except OSError: + os.remove(path) + else: + not_exist = self._create_path() + if not_exist: + return not_exist + + # download tarball + try: + data = load_url(command.url, retry=command.retry) + except URLError as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Could not fetch tarball from '%s': %s" % (command.url, e), + 'returncode': 1 + } + + # unpack tarball into destination + try: + # raise all fatal errors + tar = tarfile.open(mode='r', fileobj=BytesIO(data), errorlevel=1) + except (tarfile.ReadError, IOError, OSError) as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Failed to read tarball fetched from '%s': %s" % + (command.url, e), + 'returncode': 1 + } + + if not command.version: + members = None + else: + # remap all members from version subfolder into destination + def get_members(tar, prefix): + for tar_info in tar.getmembers(): + if tar_info.name.startswith(prefix): + tar_info.name = tar_info.name[len(prefix):] + yield tar_info + prefix = str(command.version) + '/' + members = get_members(tar, prefix) + tar.extractall(self.path, members) + + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Downloaded tarball from '%s' and unpacked it" % command.url, + 'returncode': 0 + } + + def validate(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + # test url + try: + test_url(command.url, retry=command.retry) + except URLError as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Failed to contact tarball url '%s': %s" % + (command.url, e), + 'returncode': 1 + } + return { + 'cmd': 'http HEAD url', + 'cwd': self.path, + 'output': "Tarball url '%s' exists" % command.url, + 'returncode': None + } diff --git a/vcstool/clients/vcs_base.py b/vcstool/clients/vcs_base.py new file mode 100644 index 0000000..19b6e96 --- /dev/null +++ b/vcstool/clients/vcs_base.py @@ -0,0 +1,134 @@ +import os +import socket +import subprocess +import time +try: + from urllib.request import Request + from urllib.request import urlopen + from urllib.error import HTTPError + from urllib.error import URLError +except ImportError: + from urllib2 import HTTPError + from urllib2 import Request + from urllib2 import URLError + from urllib2 import urlopen + +try: + from shutil import which # noqa +except ImportError: + from vcstool.compat.shutil import which # noqa + + +class VcsClientBase(object): + + type = None + + def __init__(self, path): + self.path = path + + def __getattribute__(self, name): + if name == 'import': + try: + return self.import_ + except AttributeError: + pass + return super(VcsClientBase, self).__getattribute__(name) + + def _not_applicable(self, command, message=None): + return { + 'cmd': '%s.%s(%s)' % ( + self.__class__.type, 'push', command.__class__.command), + 'output': "Command '%s' not applicable for client '%s'%s" % ( + command.__class__.command, self.__class__.type, + ': ' + message if message else ''), + 'returncode': NotImplemented + } + + def _run_command(self, cmd, env=None, retry=0): + for i in range(retry + 1): + result = run_command(cmd, os.path.abspath(self.path), env=env) + if not result['returncode']: + # return successful result + break + if i >= retry: + # return the failure after retries + break + # increasing sleep before each retry + time.sleep(i + 1) + return result + + def _create_path(self): + if not os.path.exists(self.path): + try: + os.makedirs(self.path) + except os.error as e: + return { + 'cmd': 'os.makedirs(%s)' % self.path, + 'cwd': self.path, + 'output': + "Could not create directory '%s': %s" % (self.path, e), + 'returncode': 1 + } + return None + + +def run_command(cmd, cwd, env=None): + if not os.path.exists(cwd): + cwd = None + result = {'cmd': ' '.join(cmd), 'cwd': cwd} + try: + proc = subprocess.Popen( + cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=env) + output, _ = proc.communicate() + result['output'] = output.rstrip().decode('utf8') + result['returncode'] = proc.returncode + except subprocess.CalledProcessError as e: + result['output'] = e.output.decode('utf8') + result['returncode'] = e.returncode + return result + + +def load_url(url, retry=2, retry_period=1, timeout=10): + try: + fh = urlopen(url, timeout=timeout) + except HTTPError as e: + if e.code == 503 and retry: + time.sleep(retry_period) + return load_url( + url, retry=retry - 1, retry_period=retry_period, + timeout=timeout) + e.msg += ' (%s)' % url + raise + except URLError as e: + if isinstance(e.reason, socket.timeout) and retry: + time.sleep(retry_period) + return load_url( + url, retry=retry - 1, retry_period=retry_period, + timeout=timeout) + raise URLError(str(e) + ' (%s)' % url) + return fh.read() + + +def test_url(url, retry=2, retry_period=1, timeout=10): + request = Request(url) + request.get_method = lambda: 'HEAD' + + try: + response = urlopen(request) + except HTTPError as e: + if e.code == 503 and retry: + time.sleep(retry_period) + return test_url( + url, retry=retry - 1, retry_period=retry_period, + timeout=timeout) + e.msg += ' (%s)' % url + raise + except URLError as e: + if isinstance(e.reason, socket.timeout) and retry: + time.sleep(retry_period) + return test_url( + url, retry=retry - 1, retry_period=retry_period, + timeout=timeout) + raise URLError(str(e) + ' (%s)' % url) + return response diff --git a/vcstool/clients/zip.py b/vcstool/clients/zip.py new file mode 100644 index 0000000..7de4270 --- /dev/null +++ b/vcstool/clients/zip.py @@ -0,0 +1,144 @@ +import os +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO +try: + from urllib.error import URLError +except ImportError: + from urllib2 import URLError +import zipfile + +from .vcs_base import load_url +from .vcs_base import test_url +from .vcs_base import VcsClientBase +from ..util import rmtree + + +class ZipClient(VcsClientBase): + + type = 'zip' + + @staticmethod + def is_repository(path): + return False + + def __init__(self, path): + super(ZipClient, self).__init__(path) + + def import_(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + # clear destination + if os.path.exists(self.path): + for filename in os.listdir(self.path): + path = os.path.join(self.path, filename) + try: + rmtree(path) + except OSError: + os.remove(path) + else: + not_exist = self._create_path() + if not_exist: + return not_exist + + # download zipfile + try: + data = load_url(command.url, retry=command.retry) + except URLError as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Could not fetch zipfile from '%s': %s" % (command.url, e), + 'returncode': 1 + } + + def create_path(path): + if not os.path.exists(path): + try: + os.makedirs(path) + except os.error as e: + return { + 'cmd': 'os.makedirs(%s)' % path, + 'cwd': path, + 'output': + "Could not create directory '%s': %s" % (path, e), + 'returncode': 1 + } + return None + + # unpack zipfile into destination + try: + zip_file = zipfile.ZipFile(BytesIO(data), mode='r') + except zipfile.BadZipfile as e: + return { + 'cmd': 'ZipFile(%s)' % command.url, + 'cwd': self.path, + 'output': + "Could not read zipfile from '%s': %s" % (command.url, e), + 'returncode': 1 + } + try: + if not command.version: + zip_file.extractall(self.path) + else: + prefix = str(command.version) + '/' + for name in zip_file.namelist(): + if name.startswith(prefix): + if not name[len(prefix):]: + continue + # remap members from version subfolder into destination + dst = os.path.join(self.path, name[len(prefix):]) + if dst.endswith('/'): + # create directories + not_exist = create_path(dst) + if not_exist: + return not_exist + else: + with zip_file.open(name, mode='r') as src_handle: + with open(dst, 'wb') as dst_handle: + dst_handle.write(src_handle.read()) + finally: + zip_file.close() + + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Downloaded zipfile from '%s' and unpacked it" % command.url, + 'returncode': 0 + } + + def validate(self, command): + if not command.url: + return { + 'cmd': '', + 'cwd': self.path, + 'output': "Repository data lacks the 'url' value", + 'returncode': 1 + } + + # test url + try: + test_url(command.url, retry=command.retry) + except URLError as e: + return { + 'cmd': '', + 'cwd': self.path, + 'output': + "Failed to contact zip url '%s': %s" % (command.url, e), + 'returncode': 1 + } + return { + 'cmd': 'http HEAD', + 'cwd': self.path, + 'output': "Zip url '%s' exists" % command.url, + 'returncode': None + } diff --git a/vcstool/commands/__init__.py b/vcstool/commands/__init__.py new file mode 100644 index 0000000..f96d55b --- /dev/null +++ b/vcstool/commands/__init__.py @@ -0,0 +1,30 @@ +from .branch import BranchCommand +from .custom import CustomCommand +from .diff import DiffCommand +from .export import ExportCommand +from .import_ import ImportCommand +from .log import LogCommand +from .pull import PullCommand +from .push import PushCommand +from .remotes import RemotesCommand +from .status import StatusCommand +from .validate import ValidateCommand + +vcstool_commands = [] +vcstool_commands.append(BranchCommand) +vcstool_commands.append(CustomCommand) +vcstool_commands.append(DiffCommand) +vcstool_commands.append(ExportCommand) +vcstool_commands.append(ImportCommand) +vcstool_commands.append(LogCommand) +vcstool_commands.append(PullCommand) +vcstool_commands.append(PushCommand) +vcstool_commands.append(RemotesCommand) +vcstool_commands.append(StatusCommand) +vcstool_commands.append(ValidateCommand) + +_commands = [c.command for c in vcstool_commands] +if len(_commands) != len(set(_commands)): + raise RuntimeError( + 'Multiple commands share the same command name: ' + + ', '.join(sorted(_commands))) diff --git a/vcstool/commands/branch.py b/vcstool/commands/branch.py new file mode 100644 index 0000000..7ead41c --- /dev/null +++ b/vcstool/commands/branch.py @@ -0,0 +1,36 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class BranchCommand(Command): + + command = 'branch' + help = 'Show the branches' + + def __init__(self, args): + super(BranchCommand, self).__init__(args) + self.all = args.all + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Show the current branch', prog='vcs branch') + group = parser.add_argument_group('"branch" command parameters') + group.add_argument( + '--all', action='store_true', default=False, help='Show all branches') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, BranchCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/command.py b/vcstool/commands/command.py new file mode 100644 index 0000000..7765989 --- /dev/null +++ b/vcstool/commands/command.py @@ -0,0 +1,102 @@ +import argparse +from multiprocessing import cpu_count +import os + +from vcstool.crawler import find_repositories +from vcstool.executor import execute_jobs +from vcstool.executor import generate_jobs +from vcstool.executor import output_repositories +from vcstool.executor import output_results + + +class Command(object): + + command = None + + def __init__(self, args): + self.debug = args.debug if 'debug' in args else False + self.hide_empty = args.hide_empty if 'hide_empty' in args else False + self.nested = args.nested if 'nested' in args else False + self.output_repos = args.repos if 'repos' in args else False + if 'paths' in args: + self.paths = args.paths + else: + self.paths = [args.path] + + +def check_greater_zero(value): + try: + value = int(value) + except ValueError: + raise argparse.ArgumentTypeError("invalid int value: '%s'" % value) + if value <= 0: + raise argparse.ArgumentTypeError( + "invalid positive int value: '%d'" % value) + return value + + +def add_common_arguments( + parser, skip_hide_empty=False, skip_nested=False, path_nargs='*', + path_help=None +): + parser.formatter_class = argparse.ArgumentDefaultsHelpFormatter + group = parser.add_argument_group('Common parameters') + group.add_argument( + '--debug', action='store_true', default=False, + help='Show debug messages') + if not skip_hide_empty: + group.add_argument( + '-s', '--hide-empty', '--skip-empty', action='store_true', + default=False, help='Hide repositories with empty output') + if not skip_nested: + group.add_argument( + '-n', '--nested', action='store_true', + default=False, help='Search for nested repositories') + try: + default_workers = cpu_count() + except NotImplementedError: + default_workers = 4 + group.add_argument( + '-w', '--workers', type=check_greater_zero, metavar='N', + default=default_workers, help='Number of parallel worker threads') + group.add_argument( + '--repos', action='store_true', default=False, + help='List repositories which the command operates on') + if path_nargs == '?': + path_help = path_help or 'Base path to look for repositories' + group.add_argument( + 'path', nargs=path_nargs, type=existing_dir, default=os.curdir, + help=path_help) + elif path_nargs == '*': + path_help = path_help or 'Base paths to look for repositories' + group.add_argument( + 'paths', nargs=path_nargs, type=existing_dir, default=[os.curdir], + help=path_help) + + +def existing_dir(path): + if not os.path.exists(path): + raise argparse.ArgumentTypeError("Path '%s' does not exist." % path) + if not os.path.isdir(path): + raise argparse.ArgumentTypeError( + "Path '%s' is not a directory." % path) + return path + + +def simple_main(parser, command_class, args=None): + add_common_arguments(parser) + args = parser.parse_args(args) + + command = command_class(args) + clients = find_repositories(command.paths, nested=command.nested) + if command.output_repos: + output_repositories(clients) + jobs = generate_jobs(clients, command) + results = execute_jobs( + jobs, show_progress=True, number_of_workers=args.workers, + debug_jobs=args.debug) + + output_results(results, hide_empty=args.hide_empty) + + any_error = any(r['returncode'] for r in results) + return 1 if any_error else 0 diff --git a/vcstool/commands/custom.py b/vcstool/commands/custom.py new file mode 100644 index 0000000..fdaa9a6 --- /dev/null +++ b/vcstool/commands/custom.py @@ -0,0 +1,120 @@ +import argparse +import sys + +from vcstool.clients import vcstool_clients +from vcstool.crawler import find_repositories +from vcstool.executor import execute_jobs +from vcstool.executor import generate_jobs +from vcstool.executor import output_repositories +from vcstool.executor import output_results +from vcstool.streams import set_streams + +from .command import add_common_arguments +from .command import Command + + +class CustomCommand(Command): + + command = 'custom' + help = 'Run a custom command' + + def __init__(self, args): + super(CustomCommand, self).__init__(args) + self.args = args.args + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Run a custom command', prog='vcs custom') + group = parser.add_argument_group( + '"custom" command parameters restricting the repositories') + for client_type in [ + c.type for c in vcstool_clients if c.type not in ['tar'] + ]: + group.add_argument( + '--' + client_type, action='store_true', default=False, + help="Run command on '%s' repositories" % client_type) + group = parser.add_argument_group('"custom" command parameters') + group.add_argument( + '--args', required=True, nargs='*', help='Arbitrary arguments passed ' + 'to each vcs invocation. It must be passed after other arguments ' + 'since it collects all following options.') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + + parser = get_parser() + add_common_arguments(parser) + + # separate anything followed after --args to not confuse argparse + if args is None: + args = sys.argv[1:] + try: + index = args.index('--args') + 1 + except ValueError: + # should generate error due to missing --args + parser.parse_known_args(args) + + client_args = args[index:] + args = parser.parse_args(args[0:index]) + args.args = client_args + + # check if any client type is specified + any_client_type = False + for client in vcstool_clients: + if client.type in args and args.__dict__[client.type]: + any_client_type = True + break + # if no client type is specified enable all client types + if not any_client_type: + for client in vcstool_clients: + if client.type in args: + args.__dict__[client.type] = True + + command = CustomCommand(args) + + # filter repositories by specified client types + clients = find_repositories(command.paths, nested=command.nested) + clients = [c for c in clients if c.type in args and args.__dict__[c.type]] + + if command.output_repos: + output_repositories(clients) + jobs = generate_jobs(clients, command) + results = execute_jobs( + jobs, show_progress=True, number_of_workers=args.workers, + debug_jobs=args.debug) + + output_results(results, hide_empty=args.hide_empty) + + any_error = any(r['returncode'] for r in results) + return 1 if any_error else 0 + + +def bzr_main(args=None): + if args is None: + args = sys.argv[1:] + return main(['--bzr', '--args'] + args) + + +def git_main(args=None): + if args is None: + args = sys.argv[1:] + return main(['--git', '--args'] + args) + + +def hg_main(args=None): + if args is None: + args = sys.argv[1:] + return main(['--hg', '--args'] + args) + + +def svn_main(args=None): + if args is None: + args = sys.argv[1:] + return main(['--svn', '--args'] + args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/diff.py b/vcstool/commands/diff.py new file mode 100644 index 0000000..0cc2fc6 --- /dev/null +++ b/vcstool/commands/diff.py @@ -0,0 +1,37 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class DiffCommand(Command): + + command = 'diff' + help = 'Show changes in the working tree' + + def __init__(self, args): + super(DiffCommand, self).__init__(args) + self.context = args.context + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Show changes in the working tree', prog='vcs diff') + group = parser.add_argument_group('"diff" command parameters') + group.add_argument( + '--context', metavar='N', type=int, + help='Generate diffs with <n> lines of context') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, DiffCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/export.py b/vcstool/commands/export.py new file mode 100644 index 0000000..037c514 --- /dev/null +++ b/vcstool/commands/export.py @@ -0,0 +1,125 @@ +from __future__ import print_function + +import argparse +import os +import sys + +from vcstool.crawler import find_repositories +from vcstool.executor import ansi +from vcstool.executor import execute_jobs +from vcstool.executor import generate_jobs +from vcstool.executor import output_repositories +from vcstool.executor import output_results +from vcstool.streams import set_streams + +from .command import add_common_arguments +from .command import Command + + +class ExportCommand(Command): + + command = 'export' + help = 'Export the list of repositories' + + def __init__(self, args): + super(ExportCommand, self).__init__(args) + self.exact = args.exact or args.exact_with_tags + self.with_tags = args.exact_with_tags + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Export the list of repositories', prog='vcs export') + group = parser.add_argument_group('"export" command parameters') + group_exact = group.add_mutually_exclusive_group() + group_exact.add_argument( + '--exact', action='store_true', default=False, + help='Export commit hashes instead of branch names') + group_exact.add_argument( + '--exact-with-tags', action='store_true', default=False, + help='Export unique tag names or commit hashes instead of branch ' + 'names') + return parser + + +def output_export_data(result, hide_empty=False): + # errors are handled by a separate function + if result['returncode']: + return + + try: + lines = [] + lines.append(' %s:' % result['path']) + lines.append(' type: ' + result['client'].__class__.type) + export_data = result['export_data'] + lines.append(' url: ' + export_data['url']) + if 'version' in export_data and export_data['version']: + lines.append(' version: ' + export_data['version']) + print('\n'.join(lines)) + except KeyError as e: + print( + ansi('redf') + ( + "Command '%s' failed for path '%s': %s: %s" % ( + result['command'].__class__.command, + result['client'].path, e.__class__.__name__, e)) + + ansi('reset'), + file=sys.stderr) + + +def output_error_information(result, hide_empty=False): + # successful results are handled by a separate function + if not result['returncode']: + return + + if result['returncode'] == NotImplemented: + color = 'yellow' + else: + color = 'red' + + line = '%s: %s' % (result['path'], result['output']) + print(ansi('%sf' % color) + line + ansi('reset'), file=sys.stderr) + + +def get_relative_path_of_result(result): + client = result['client'] + return os.path.relpath(client.path, result['command'].paths[0]) + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + + parser = get_parser() + add_common_arguments(parser, skip_hide_empty=True, path_nargs='?') + args = parser.parse_args(args) + + command = ExportCommand(args) + clients = find_repositories(command.paths, nested=command.nested) + if command.output_repos: + output_repositories(clients) + jobs = generate_jobs(clients, command) + results = execute_jobs(jobs, number_of_workers=args.workers) + + # check if at least one repo was found in the client directory + basename = None + for result in results: + result['path'] = get_relative_path_of_result(result) + if result['path'] == '.': + basename = os.path.basename(os.path.abspath(result['client'].path)) + # in that case prefix all relative paths with the client directory basename + if basename is not None: + for result in results: + if result['path'] == '.': + result['path'] = basename + else: + result['path'] = os.path.join(basename, result['path']) + + print('repositories:') + output_results(results, output_handler=output_export_data) + output_results(results, output_handler=output_error_information) + + any_error = any(r['returncode'] for r in results) + return 1 if any_error else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/help.py b/vcstool/commands/help.py new file mode 100644 index 0000000..cfd6072 --- /dev/null +++ b/vcstool/commands/help.py @@ -0,0 +1,123 @@ +from __future__ import print_function + +import argparse +import sys + +from pkg_resources import load_entry_point +from vcstool.clients import vcstool_clients +from vcstool.commands import vcstool_commands +from vcstool.streams import set_streams + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + + # no help to extract command first (which might be followed by --help) + parser = get_parser(add_help=False) + ns, _ = parser.parse_known_args(args) + + # help for a specific command + if ns.command: + # relay help request foe specific command + entrypoint = get_entrypoint(ns.command) + if not entrypoint: + return 1 + return entrypoint(['--help']) + + # regular parsing validating options and arguments + parser = get_parser() + ns = parser.parse_args(args) + + if ns.clients: + print('The available VCS clients are:') + for client in vcstool_clients: + print(' ' + client.type) + return 0 + + if ns.commands: + print(' '.join([cmd.command for cmd in vcstool_commands])) + return 0 + + # output detailed command list + parser = get_parser_with_command_only() + parser.print_help() + return 0 + + +def get_parser(add_help=True): + parser = argparse.ArgumentParser( + prog='vcs', description=_get_description(), + epilog=_get_epilog(), add_help=add_help) + group = parser.add_mutually_exclusive_group() + group.add_argument( + 'command', metavar='<command>', nargs='?', + help='The available commands: ' + ', '.join( + [cmd.command for cmd in vcstool_commands])) + group.add_argument( + '--clients', action='store_true', default=False, + help='Show the available VCS clients') + group.add_argument( + '--commands', action='store_true', default=False, + help='Output the available commands for auto-completion') + from vcstool import __version__ + group.add_argument( + '--version', action='version', version='%(prog)s ' + __version__, + help='Show the vcstool version') + return parser + + +def get_entrypoint(command): + # accept command with same prefix if unique + commands = [cmd.command for cmd in vcstool_commands] + commands = [cmd for cmd in commands if cmd.startswith(command)] + if len(commands) != 1: + print( + "vcs: '%s' is not a vcs command. See 'vcs help'." % command, + file=sys.stderr) + if commands: + print( + '\nDid you mean one of these?\n' + '\n '.join(commands), + file=sys.stderr) + return None + + return load_entry_point( + 'vcstool', 'console_scripts', 'vcs-' + commands[0]) + + +def get_parser_with_command_only(): + parser = argparse.ArgumentParser( + prog='vcs', usage='%(prog)s <command>', + formatter_class=argparse.RawDescriptionHelpFormatter, + description='%s\n\n%s' % ( + _get_description(), + '\n'.join(_get_command_help(vcstool_commands))), + epilog=_get_epilog(), add_help=False) + parser.add_argument('command', help=argparse.SUPPRESS) + return parser + + +def _get_description(): + return 'Most commands take directory arguments, ' \ + 'recursively searching for repositories\n' \ + 'in these directories. ' \ + 'If no arguments are supplied to a command, it recurses\n' \ + 'on the current directory (inclusive) by default.' + + +def _get_epilog(): + return "See '%(prog)s <command> --help' for more information " \ + 'on a specific command.' + + +def _get_command_help(commands): + lines = ['The available commands are:'] + max_len = max(len(cmd.command) for cmd in commands) + for cmd in vcstool_commands: + lines.append( + ' %s%s %s' % + (cmd.command, ' ' * (max_len - len(cmd.command)), cmd.help)) + return lines + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/import_.py b/vcstool/commands/import_.py new file mode 100644 index 0000000..188b721 --- /dev/null +++ b/vcstool/commands/import_.py @@ -0,0 +1,267 @@ +from __future__ import print_function + +import argparse +import os +import sys + +from vcstool import __version__ as vcstool_version +from vcstool.clients import vcstool_clients +from vcstool.clients.vcs_base import run_command +from vcstool.clients.vcs_base import which +from vcstool.executor import ansi +from vcstool.executor import execute_jobs +from vcstool.executor import output_repositories +from vcstool.executor import output_results +from vcstool.streams import set_streams +import yaml + +try: + import urllib.request as request +except ImportError: + import urllib2 as request + +from .command import add_common_arguments +from .command import Command + + +class ImportCommand(Command): + + command = 'import' + help = 'Import the list of repositories' + + def __init__( + self, args, url, version=None, recursive=False, shallow=False + ): + super(ImportCommand, self).__init__(args) + self.url = url + self.version = version + self.force = args.force + self.retry = args.retry + self.skip_existing = args.skip_existing + self.recursive = recursive + self.shallow = shallow + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Import the list of repositories', prog='vcs import') + group = parser.add_argument_group('"import" command parameters') + group.add_argument( + '--input', type=file_or_url_type, default='-', + help='Where to read YAML from', metavar='FILE_OR_URL') + group.add_argument( + '--force', action='store_true', default=False, + help="Delete existing directories if they don't contain the " + 'repository being imported') + group.add_argument( + '--shallow', action='store_true', default=False, + help='Create a shallow clone without a history') + group.add_argument( + '--recursive', action='store_true', default=False, + help='Recurse into submodules') + group.add_argument( + '--retry', type=int, metavar='N', default=2, + help='Retry commands requiring network access N times on failure') + group.add_argument( + '--skip-existing', action='store_true', default=False, + help="Don't overwrite existing directories or change custom checkouts " + 'in repos using the same URL (but fetch repos with same URL)') + + return parser + + +def file_or_url_type(value): + if os.path.exists(value) or '://' not in value: + return argparse.FileType('r')(value) + # use another user agent to avoid getting a 403 (forbidden) error, + # since some websites blacklist or block unrecognized user agents + return request.Request( + value, headers={'User-Agent': 'vcstool/' + vcstool_version}) + + +def get_repositories(yaml_file): + try: + root = yaml.safe_load(yaml_file) + except yaml.YAMLError as e: + raise RuntimeError('Input data is not valid yaml format: %s' % e) + + try: + repositories = root['repositories'] + return get_repos_in_vcstool_format(repositories) + except KeyError as e: + raise RuntimeError('Input data is not valid format: %s' % e) + except TypeError as e: + # try rosinstall file format + try: + return get_repos_in_rosinstall_format(root) + except Exception: + raise RuntimeError('Input data is not valid format: %s' % e) + + +def get_repos_in_vcstool_format(repositories): + repos = {} + if repositories is None: + print( + ansi('yellowf') + 'List of repositories is empty' + ansi('reset'), + file=sys.stderr) + return repos + for path in repositories: + repo = {} + attributes = repositories[path] + try: + repo['type'] = attributes['type'] + repo['url'] = attributes['url'] + if 'version' in attributes: + repo['version'] = attributes['version'] + except KeyError as e: + print( + ansi('yellowf') + ( + "Repository '%s' does not provide the necessary " + 'information: %s' % (path, e)) + ansi('reset'), + file=sys.stderr) + continue + repos[path] = repo + return repos + + +def get_repos_in_rosinstall_format(root): + repos = {} + for i, item in enumerate(root): + if len(item.keys()) != 1: + raise RuntimeError('Input data is not valid format') + repo = {'type': list(item.keys())[0]} + attributes = list(item.values())[0] + try: + path = attributes['local-name'] + except KeyError as e: + print( + ansi('yellowf') + ( + 'Repository #%d does not provide the necessary ' + 'information: %s' % (i, e)) + ansi('reset'), + file=sys.stderr) + continue + try: + repo['url'] = attributes['uri'] + if 'version' in attributes: + repo['version'] = attributes['version'] + except KeyError as e: + print( + ansi('yellowf') + ( + "Repository '%s' does not provide the necessary " + 'information: %s' % (path, e)) + ansi('reset'), + file=sys.stderr) + continue + repos[path] = repo + return repos + + +def generate_jobs(repos, args): + jobs = [] + for path, repo in repos.items(): + path = os.path.join(args.path, path) + clients = [c for c in vcstool_clients if c.type == repo['type']] + if not clients: + from vcstool.clients.none import NoneClient + job = { + 'client': NoneClient(path), + 'command': None, + 'cwd': path, + 'output': + "Repository type '%s' is not supported" % repo['type'], + 'returncode': NotImplemented + } + jobs.append(job) + continue + + client = clients[0](path) + command = ImportCommand( + args, repo['url'], + str(repo['version']) if 'version' in repo else None, + recursive=args.recursive, shallow=args.shallow) + job = {'client': client, 'command': command} + jobs.append(job) + return jobs + + +def add_dependencies(jobs): + paths = [job['client'].path for job in jobs] + for job in jobs: + job['depends'] = set() + path = job['client'].path + while True: + parent_path = os.path.dirname(path) + if parent_path == path: + break + path = parent_path + if path in paths: + job['depends'].add(path) + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + + parser = get_parser() + add_common_arguments( + parser, skip_hide_empty=True, skip_nested=True, path_nargs='?', + path_help='Base path to clone repositories to') + args = parser.parse_args(args) + try: + input_ = args.input + if isinstance(input_, request.Request): + input_ = request.urlopen(input_) + repos = get_repositories(input_) + except (RuntimeError, request.URLError) as e: + print(ansi('redf') + str(e) + ansi('reset'), file=sys.stderr) + return 1 + jobs = generate_jobs(repos, args) + add_dependencies(jobs) + + if args.repos: + output_repositories([job['client'] for job in jobs]) + + workers = args.workers + # for ssh URLs check if the host is known to prevent ssh asking for + # confirmation when using more than one worker + if workers > 1: + ssh_keygen = None + checked_hosts = set() + for job in list(jobs): + if job['command'] is None: + continue + url = job['command'].url + # only check the host from a ssh URL + if not url.startswith('git@') or ':' not in url: + continue + host = url[4:].split(':', 1)[0] + + # only check each host name once + if host in checked_hosts: + continue + checked_hosts.add(host) + + # get ssh-keygen path once + if ssh_keygen is None: + ssh_keygen = which('ssh-keygen') or False + if not ssh_keygen: + continue + + result = run_command([ssh_keygen, '-F', host], '') + if result['returncode']: + print( + 'At least one hostname (%s) is unknown, switching to a ' + 'single worker to allow interactively answering the ssh ' + 'question to confirm the fingerprint' % host) + workers = 1 + break + + results = execute_jobs( + jobs, show_progress=True, number_of_workers=workers, + debug_jobs=args.debug) + output_results(results) + + any_error = any(r['returncode'] for r in results) + return 1 if any_error else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/log.py b/vcstool/commands/log.py new file mode 100644 index 0000000..cb7f826 --- /dev/null +++ b/vcstool/commands/log.py @@ -0,0 +1,50 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class LogCommand(Command): + + command = 'log' + help = 'Show commit logs' + + def __init__(self, args): + super(LogCommand, self).__init__(args) + self.limit = args.limit + self.limit_tag = args.limit_tag + self.limit_untagged = args.limit_untagged + self.verbose = args.verbose + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Show commit logs', prog='vcs log') + group = parser.add_argument_group('"log" command parameters') + group.add_argument( + '-l', '--limit', metavar='N', type=int, default=3, + help='Limit number of logs (0 for unlimited)') + ex_group = group.add_mutually_exclusive_group() + ex_group.add_argument( + '--limit-tag', metavar='TAG', + help='Limit number of log to the specified tag') + ex_group.add_argument( + '--limit-untagged', action='store_true', default=False, + help='Limit number of log to the last tagged commit') + group.add_argument( + '--verbose', action='store_true', default=False, + help='Show the full commit message') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, LogCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/pull.py b/vcstool/commands/pull.py new file mode 100644 index 0000000..df7e223 --- /dev/null +++ b/vcstool/commands/pull.py @@ -0,0 +1,34 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class PullCommand(Command): + + command = 'pull' + help = 'Bring changes from the repository into the working copy' + + def __init__(self, args): + super(PullCommand, self).__init__(args) + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Bring changes from the repository into the working copy', + prog='vcs pull') + parser.add_argument_group('"pull" command parameters') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, PullCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/push.py b/vcstool/commands/push.py new file mode 100644 index 0000000..6f801d6 --- /dev/null +++ b/vcstool/commands/push.py @@ -0,0 +1,34 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class PushCommand(Command): + + command = 'push' + help = 'Push changes from the working copy to the repository' + + def __init__(self, args): + super(PushCommand, self).__init__(args) + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Push changes from the working copy to the repository', + prog='vcs push') + parser.add_argument_group('"push" command parameters') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, PushCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/remotes.py b/vcstool/commands/remotes.py new file mode 100644 index 0000000..29a8518 --- /dev/null +++ b/vcstool/commands/remotes.py @@ -0,0 +1,33 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class RemotesCommand(Command): + + command = 'remotes' + help = 'Show the URL of the repository' + + def __init__(self, args): + super(RemotesCommand, self).__init__(args) + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Show the URL of the repository', prog='vcs remotes') + parser.add_argument_group('"remotes" command parameters') + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, RemotesCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/status.py b/vcstool/commands/status.py new file mode 100644 index 0000000..877ea82 --- /dev/null +++ b/vcstool/commands/status.py @@ -0,0 +1,37 @@ +import argparse +import sys + +from vcstool.streams import set_streams + +from .command import Command +from .command import simple_main + + +class StatusCommand(Command): + + command = 'status' + help = 'Show the working tree status' + + def __init__(self, args): + super(StatusCommand, self).__init__(args) + self.quiet = args.quiet + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Show the working tree status', prog='vcs status') + group = parser.add_argument_group('"status" command parameters') + group.add_argument( + '-q', '--quiet', action='store_true', default=False, + help="Don't show unversioned items") + return parser + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + parser = get_parser() + return simple_main(parser, StatusCommand, args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/validate.py b/vcstool/commands/validate.py new file mode 100644 index 0000000..8295257 --- /dev/null +++ b/vcstool/commands/validate.py @@ -0,0 +1,94 @@ +from __future__ import print_function + +import argparse +import sys + +from vcstool.clients import vcstool_clients +from vcstool.commands.import_ import get_repositories +from vcstool.executor import ansi +from vcstool.executor import execute_jobs +from vcstool.executor import output_results +from vcstool.streams import set_streams + +from .command import add_common_arguments +from .command import Command + + +class ValidateCommand(Command): + + command = 'validate' + help = 'Validate the repository list file' + + def __init__(self, args, url, version=None): + super(ValidateCommand, self).__init__(args) + self.url = url + self.version = version + self.retry = args.retry + + +def get_parser(): + parser = argparse.ArgumentParser( + description='Validate a repositories file', prog='vcs validate') + group = parser.add_argument_group('"validate" command parameters') + group.add_argument( + '--input', type=argparse.FileType('r'), default='-') + group.add_argument( + '--retry', type=int, metavar='N', default=2, + help='Retry commands requiring network access N times on failure') + return parser + + +def generate_jobs(repos, args): + jobs = [] + for path, repo in repos.items(): + clients = [c for c in vcstool_clients if c.type == repo['type']] + if not clients: + from vcstool.clients.none import NoneClient + job = { + 'client': NoneClient(path), + 'command': None, + 'cwd': path, + 'output': + "Repository type '%s' is not supported" % repo['type'], + 'returncode': NotImplemented + } + jobs.append(job) + continue + + client = clients[0](path) + args.path = None # expected to be present + command = ValidateCommand( + args, repo['url'], + str(repo['version']) if 'version' in repo else None) + job = {'client': client, 'command': command} + jobs.append(job) + return jobs + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + + parser = get_parser() + add_common_arguments( + parser, skip_nested=True, path_nargs=False) + args = parser.parse_args(args) + try: + repos = get_repositories(args.input) + except RuntimeError as e: + print(ansi('redf') + str(e) + ansi('reset'), file=sys.stderr) + return 1 + + jobs = generate_jobs(repos, args) + + results = execute_jobs( + jobs, show_progress=True, number_of_workers=args.workers, + debug_jobs=args.debug) + + output_results(results, hide_empty=args.hide_empty) + + any_error = any(r['returncode'] for r in results) + return 1 if any_error else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/commands/vcs.py b/vcstool/commands/vcs.py new file mode 100644 index 0000000..b006d0c --- /dev/null +++ b/vcstool/commands/vcs.py @@ -0,0 +1,36 @@ +from __future__ import print_function + +import sys + +from vcstool.commands.help import get_entrypoint +from vcstool.commands.help import get_parser +from vcstool.commands.help import main as help_main +from vcstool.streams import set_streams + + +def main(args=None, stdout=None, stderr=None): + set_streams(stdout=stdout, stderr=stderr) + + # no help to extract command first (which might be followed by --help) + parser = get_parser(add_help=False) + ns, _ = parser.parse_known_args(args) + args = args if args is not None else sys.argv[1:] + + # relay to specific command + if ns.command and ns.command != 'help': + entrypoint = get_entrypoint(ns.command) + if not entrypoint: + return 1 + + args.remove(ns.command) + return entrypoint(args) + + # remove help command if specified + if ns.command: + args.remove(ns.command) + + return help_main(args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/vcstool/compat/__init__.py b/vcstool/compat/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/vcstool/compat/__init__.py diff --git a/vcstool/compat/shutil.py b/vcstool/compat/shutil.py new file mode 100644 index 0000000..7b22d04 --- /dev/null +++ b/vcstool/compat/shutil.py @@ -0,0 +1,73 @@ +# This function has been copied from Python shutil for backward +# compatibility with Python < 3.2. + +# Copyright (c) +# 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, +# 2012, 2013, 2014 Python Software Foundation; All Rights Reserved +# https://www.python.org/download/releases/2.7/license/ + +import os +import sys + + +def which(cmd, mode=os.F_OK | os.X_OK, path=None): + """Given a command, mode, and a PATH string, return the path which + conforms to the given mode on the PATH, or None if there is no such + file. + + `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result + of os.environ.get("PATH"), or can be overridden with a custom search + path. + + """ + # Check that a given file can be accessed with the correct mode. + # Additionally check that `file` is not a directory, as on Windows + # directories pass the os.access check. + def _access_check(fn, mode): + return (os.path.exists(fn) and os.access(fn, mode) + and not os.path.isdir(fn)) + + # If we're given a path with a directory part, look it up directly rather + # than referring to PATH directories. This includes checking relative to the + # current directory, e.g. ./script + if os.path.dirname(cmd): + if _access_check(cmd, mode): + return cmd + return None + + if path is None: + path = os.environ.get("PATH", os.defpath) + if not path: + return None + path = path.split(os.pathsep) + + if sys.platform == "win32": + # The current directory takes precedence on Windows. + if not os.curdir in path: + path.insert(0, os.curdir) + + # PATHEXT is necessary to check on Windows. + pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + # See if the given file matches any of the expected path extensions. + # This will allow us to short circuit when given "python.exe". + # If it does match, only test that one, otherwise we have to try + # others. + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): + files = [cmd] + else: + files = [cmd + ext for ext in pathext] + else: + # On other platforms you don't have things like PATHEXT to tell you + # what file suffixes are executable, so just pass on cmd as-is. + files = [cmd] + + seen = set() + for dir in path: + normdir = os.path.normcase(dir) + if not normdir in seen: + seen.add(normdir) + for thefile in files: + name = os.path.join(dir, thefile) + if _access_check(name, mode): + return name + return None diff --git a/vcstool/crawler.py b/vcstool/crawler.py new file mode 100644 index 0000000..4c42492 --- /dev/null +++ b/vcstool/crawler.py @@ -0,0 +1,41 @@ +import os + +from . import vcstool_clients + + +def find_repositories(paths, nested=False): + repos = [] + visited = [] + for path in paths: + _find_repositories(path, repos, visited, nested=nested) + return repos + + +def _find_repositories(path, repos, visited, nested=False): + abs_path = os.path.abspath(path) + if abs_path in visited: + return + visited.append(abs_path) + + client = get_vcs_client(path) + if client: + repos.append(client) + if not nested: + return + + try: + listdir = os.listdir(path) + except OSError: + listdir = [] + for name in sorted(listdir): + subpath = os.path.join(path, name) + if not os.path.isdir(subpath): + continue + _find_repositories(subpath, repos, visited, nested=nested) + + +def get_vcs_client(path): + for client_class in vcstool_clients: + if client_class.is_repository(path): + return client_class(path) + return None diff --git a/vcstool/executor.py b/vcstool/executor.py new file mode 100644 index 0000000..a2c4737 --- /dev/null +++ b/vcstool/executor.py @@ -0,0 +1,272 @@ +from __future__ import print_function + +import logging +import os +try: + from queue import Empty, Queue +except ImportError: + from Queue import Empty, Queue +import sys +import threading +import traceback + +logger = logging.getLogger(__name__) +logging.basicConfig() + + +def output_repositories(clients): + from vcstool.streams import stdout + ordered_clients = {client.path: client for client in clients} + for k in sorted(ordered_clients.keys()): + client = ordered_clients[k] + print('%s (%s)' % (k, client.__class__.type), file=stdout) + + +def generate_jobs(clients, command): + jobs = [] + realpaths = {} + for client in clients: + # check if client is a duplicate of another path + realpath = os.path.realpath(client.path) + if realpath not in realpaths: + realpaths[realpath] = [client.path] + else: + # override command on client to ignore multiple invocations + # on same repository + duplicate_path = realpaths[realpath][0] + realpaths[realpath].append(client.path) + method_name = command.__class__.command + method = getattr(client, method_name, None) + if method is not None: + setattr(client, method_name, DuplicateCommandHandler( + client, duplicate_path)) + + job = {'client': client, 'command': command} + jobs.append(job) + return jobs + + +class DuplicateCommandHandler(object): + + def __init__(self, client, duplicate_path): + self.client = client + self.duplicate_path = duplicate_path + + def __call__(self, _command): + return { + 'cmd': '', + 'cwd': self.client.path, + 'output': "Same repository as '%s'" % self.duplicate_path, + 'returncode': None + } + + +def get_ready_job(jobs): + for job in jobs: + if not job.get('depends', set()): + jobs.remove(job) + return job + return None + + +def execute_jobs( + jobs, show_progress=False, number_of_workers=10, debug_jobs=False +): + from vcstool.streams import stdout + if debug_jobs: + logger.setLevel(logging.DEBUG) + + results = [] + + job_queue = Queue() + result_queue = Queue() + + # create worker threads + workers = [] + for _ in range(min(number_of_workers, len(jobs))): + worker = Worker(job_queue, result_queue) + workers.append(worker) + + # fill job_queue with jobs for each worker + pending_jobs = list(jobs) + running_job_paths = [] + while job_queue.qsize() < len(workers): + job = get_ready_job(pending_jobs) + if not job: + break + running_job_paths.append(job['client'].path) + logger.debug("started '%s'" % job['client'].path) + job_queue.put(job) + logger.debug('ongoing %s' % running_job_paths) + + # start all workers + [w.start() for w in workers] + + # collect results + while len(results) < len(jobs): + (job, result) = result_queue.get() + logger.debug("finished '%s'" % job['client'].path) + running_job_paths.remove(result['job']['client'].path) + if show_progress and len(jobs) > 1: + if result['returncode'] == NotImplemented: + stdout.write('s') + elif result['returncode']: + stdout.write('E') + else: + stdout.write('.') + if debug_jobs: + stdout.write('\n') + stdout.flush() + result.update(job) + results.append(result) + if pending_jobs: + for pending_job in pending_jobs: + pending_job.get('depends', set()).discard(job['client'].path) + while job_queue.qsize() < len(workers): + job = get_ready_job(pending_jobs) + if not job: + break + running_job_paths.append(job['client'].path) + logger.debug("started '%s'" % job['client'].path) + job_queue.put(job) + assert running_job_paths + if running_job_paths: + logger.debug('ongoing ' + str(running_job_paths)) + if show_progress and len(jobs) > 1 and not debug_jobs: + print('', file=stdout) # finish progress line + + # join all workers + for w in workers: + w.done = True + [w.join() for w in workers] + return results + + +class Worker(threading.Thread): + + def __init__(self, job_queue, result_queue): + super(Worker, self).__init__() + self.daemon = True + self.done = False + self.job_queue = job_queue + self.result_queue = result_queue + + def run(self): + # process all incoming jobs + while not self.done: + try: + # fetch next job + job = self.job_queue.get(timeout=0.1) + # process job + result = self.process_job(job) + # send result + self.result_queue.put((job, result)) + except Empty: + pass + + def process_job(self, job): + command = job['command'] + if not command: + return { + 'cmd': '', + 'job': job, + 'output': job['output'], + 'returncode': 1 + } + method_name = command.__class__.command + try: + method = getattr(job['client'], method_name, None) + if method is None: + return { + 'cmd': '%s.%s(%s)' % ( + job['client'].__class__.type, method_name, + job['command'].__class__.command), + 'job': job, + 'output': + "Command '%s' not implemented for client '%s'" % ( + job['command'].__class__.command, + job['client'].__class__.type), + 'returncode': NotImplemented + } + result = method(job['command']) + result['job'] = job + return result + except Exception as e: + exc_tb = sys.exc_info()[2] + filename, lineno, _, _ = traceback.extract_tb(exc_tb)[-1] + return { + 'cmd': '%s.%s(%s)' % ( + job['client'].__class__.type, method_name, + job['command'].__class__.command), + 'job': job, + 'output': + "Invocation of command '%s' on client '%s' failed: " + '%s: %s (%s:%s)' % ( + job['command'].__class__.command, + job['client'].__class__.type, + type(e).__name__, e, filename, lineno), + 'returncode': 1 + } + + +def output_result(result, hide_empty=False): + from vcstool.streams import stdout + output = result['output'] + if hide_empty and result['returncode'] is None: + output = '' + if result['returncode'] == NotImplemented: + if output: + output = ansi('yellowf') + output + ansi('reset') + elif result['returncode']: + if not output: + output = 'Failed with return code %d' % result['returncode'] + output = ansi('redf') + output + ansi('reset') + elif not result['cmd']: + if output: + output = ansi('yellowf') + output + ansi('reset') + if output or not hide_empty: + client = result['client'] + print( + ansi('bluef') + '=== ' + + ansi('boldon') + client.path + ansi('boldoff') + + ' (' + client.__class__.type + ') ===' + ansi('reset'), + file=stdout) + if output: + try: + print(output, file=stdout) + except UnicodeEncodeError: + print( + output.encode(sys.getdefaultencoding(), 'replace'), + file=stdout) + + +def output_results(results, output_handler=output_result, hide_empty=False): + # output results in alphabetic order + path_to_idx = { + result['client'].path: i for i, result in enumerate(results)} + idxs_in_order = [path_to_idx[path] for path in sorted(path_to_idx.keys())] + for i in idxs_in_order: + output_handler(results[i], hide_empty=hide_empty) + + +USE_COLOR = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() +# disable color on Windows except if ConEmuANSI is explicitly enabled +if os.name == 'nt' and os.environ.get('ConEmuANSI', None) != 'ON': + USE_COLOR = False + + +def ansi(keyword): + if not USE_COLOR: + return '' + codes = { + 'bluef': '\033[34m', + 'boldon': '\033[1m', + 'boldoff': '\033[22m', + 'cyanf': '\033[36m', + 'redf': '\033[31m', + 'reset': '\033[0m', + 'yellowf': '\033[33m', + } + if keyword in codes: + return codes[keyword] + return '' diff --git a/vcstool/streams.py b/vcstool/streams.py new file mode 100644 index 0000000..c8fdcc6 --- /dev/null +++ b/vcstool/streams.py @@ -0,0 +1,17 @@ +import sys + +stdout = sys.stdout +stderr = sys.stderr + + +def set_streams(stdout=None, stderr=None): + _set_streams(stdout_=stdout, stderr_=stderr) + + +def _set_streams(stdout_=None, stderr_=None): + global stdout + global stderr + if stdout_ is not None: + stdout = stdout_ + if stderr_ is not None: + stderr = stderr_ diff --git a/vcstool/util.py b/vcstool/util.py new file mode 100644 index 0000000..c8869ae --- /dev/null +++ b/vcstool/util.py @@ -0,0 +1,18 @@ +from errno import EACCES, EPERM +import os +from shutil import rmtree as shutil_rmtree +import stat +import sys + + +def rmtree(path): + kwargs = {} + if sys.platform == 'win32': + kwargs['onerror'] = _onerror_windows + return shutil_rmtree(path, **kwargs) + + +def _onerror_windows(function, path, excinfo): + if isinstance(excinfo[1], OSError) and excinfo[1].errno in (EACCES, EPERM): + os.chmod(path, stat.S_IWRITE) + function(path) |