Merge pull request #1292 from pre-commit/hook_impl

Move most of the actual hook script into `pre-commit hook-impl`
This commit is contained in:
Anthony Sottile
2020-01-22 09:01:06 -08:00
committed by GitHub
10 changed files with 471 additions and 201 deletions

View File

@@ -0,0 +1,180 @@
import argparse
import os.path
import subprocess
import sys
from typing import Optional
from typing import Sequence
from typing import Tuple
from pre_commit.commands.run import run
from pre_commit.envcontext import envcontext
from pre_commit.parse_shebang import normalize_cmd
from pre_commit.store import Store
Z40 = '0' * 40
def _run_legacy(
hook_type: str,
hook_dir: str,
args: Sequence[str],
) -> Tuple[int, bytes]:
if os.environ.get('PRE_COMMIT_RUNNING_LEGACY'):
raise SystemExit(
f"bug: pre-commit's script is installed in migration mode\n"
f'run `pre-commit install -f --hook-type {hook_type}` to fix '
f'this\n\n'
f'Please report this bug at '
f'https://github.com/pre-commit/pre-commit/issues',
)
if hook_type == 'pre-push':
stdin = sys.stdin.buffer.read()
else:
stdin = b''
# not running in legacy mode
legacy_hook = os.path.join(hook_dir, f'{hook_type}.legacy')
if not os.access(legacy_hook, os.X_OK):
return 0, stdin
with envcontext((('PRE_COMMIT_RUNNING_LEGACY', '1'),)):
cmd = normalize_cmd((legacy_hook, *args))
return subprocess.run(cmd, input=stdin).returncode, stdin
def _validate_config(
retv: int,
config: str,
skip_on_missing_config: bool,
) -> None:
if not os.path.isfile(config):
if skip_on_missing_config or os.getenv('PRE_COMMIT_ALLOW_NO_CONFIG'):
print(f'`{config}` config file not found. Skipping `pre-commit`.')
raise SystemExit(retv)
else:
print(
f'No {config} file was found\n'
f'- To temporarily silence this, run '
f'`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n'
f'- To permanently silence this, install pre-commit with the '
f'--allow-missing-config option\n'
f'- To uninstall pre-commit run `pre-commit uninstall`',
)
raise SystemExit(1)
def _ns(
hook_type: str,
color: bool,
*,
all_files: bool = False,
origin: Optional[str] = None,
source: Optional[str] = None,
remote_name: Optional[str] = None,
remote_url: Optional[str] = None,
commit_msg_filename: Optional[str] = None,
) -> argparse.Namespace:
return argparse.Namespace(
color=color,
hook_stage=hook_type.replace('pre-', ''),
origin=origin,
source=source,
remote_name=remote_name,
remote_url=remote_url,
commit_msg_filename=commit_msg_filename,
all_files=all_files,
files=(),
hook=None,
verbose=False,
show_diff_on_failure=False,
)
def _rev_exists(rev: str) -> bool:
return not subprocess.call(('git', 'rev-list', '--quiet', rev))
def _pre_push_ns(
color: bool,
args: Sequence[str],
stdin: bytes,
) -> Optional[argparse.Namespace]:
remote_name = args[0]
remote_url = args[1]
for line in stdin.decode().splitlines():
_, local_sha, _, remote_sha = line.split()
if local_sha == Z40:
continue
elif remote_sha != Z40 and _rev_exists(remote_sha):
return _ns(
'pre-push', color,
origin=local_sha, source=remote_sha,
remote_name=remote_name, remote_url=remote_url,
)
else:
# ancestors not found in remote
ancestors = subprocess.check_output((
'git', 'rev-list', local_sha, '--topo-order', '--reverse',
'--not', f'--remotes={remote_name}',
)).decode().strip()
if not ancestors:
continue
else:
first_ancestor = ancestors.splitlines()[0]
cmd = ('git', 'rev-list', '--max-parents=0', local_sha)
roots = set(subprocess.check_output(cmd).decode().splitlines())
if first_ancestor in roots:
# pushing the whole tree including root commit
return _ns(
'pre-push', color,
all_files=True,
remote_name=remote_name, remote_url=remote_url,
)
else:
rev_cmd = ('git', 'rev-parse', f'{first_ancestor}^')
source = subprocess.check_output(rev_cmd).decode().strip()
return _ns(
'pre-push', color,
origin=local_sha, source=source,
remote_name=remote_name, remote_url=remote_url,
)
# nothing to push
return None
def _run_ns(
hook_type: str,
color: bool,
args: Sequence[str],
stdin: bytes,
) -> Optional[argparse.Namespace]:
if hook_type == 'pre-push':
return _pre_push_ns(color, args, stdin)
elif hook_type in {'prepare-commit-msg', 'commit-msg'}:
return _ns(hook_type, color, commit_msg_filename=args[0])
elif hook_type in {'pre-merge-commit', 'pre-commit'}:
return _ns(hook_type, color)
else:
raise AssertionError(f'unexpected hook type: {hook_type}')
def hook_impl(
store: Store,
*,
config: str,
color: bool,
hook_type: str,
hook_dir: str,
skip_on_missing_config: bool,
args: Sequence[str],
) -> int:
retv, stdin = _run_legacy(hook_type, hook_dir, args)
_validate_config(retv, config, skip_on_missing_config)
ns = _run_ns(hook_type, color, args, stdin)
if ns is None:
return retv
else:
return retv | run(config, store, ns)

View File

@@ -60,7 +60,7 @@ def shebang() -> str:
f'python{sys.version_info[0]}',
]
for path, exe in itertools.product(path_choices, exe_choices):
if os.path.exists(os.path.join(path, exe)):
if os.access(os.path.join(path, exe), os.X_OK):
py = exe
break
else:
@@ -92,12 +92,10 @@ def _install_hook_script(
f'Use -f to use only pre-commit.',
)
params = {
'CONFIG': config_file,
'HOOK_TYPE': hook_type,
'INSTALL_PYTHON': sys.executable,
'SKIP_ON_MISSING_CONFIG': skip_on_missing_config,
}
args = ['hook-impl', f'--config={config_file}', f'--hook-type={hook_type}']
if skip_on_missing_config:
args.append('--skip-on-missing-config')
params = {'INSTALL_PYTHON': sys.executable, 'ARGS': args}
with open(hook_path, 'w') as hook_file:
contents = resource_text('hook-tmpl')

View File

@@ -57,7 +57,7 @@ def _find_by_sys_executable() -> Optional[str]:
def _norm(path: str) -> Optional[str]:
_, exe = os.path.split(path.lower())
exe, _, _ = exe.partition('.exe')
if find_executable(exe) and exe not in {'python', 'pythonw'}:
if exe not in {'python', 'pythonw'} and find_executable(exe):
return exe
return None

View File

@@ -13,6 +13,7 @@ from pre_commit import git
from pre_commit.commands.autoupdate import autoupdate
from pre_commit.commands.clean import clean
from pre_commit.commands.gc import gc
from pre_commit.commands.hook_impl import hook_impl
from pre_commit.commands.init_templatedir import init_templatedir
from pre_commit.commands.install_uninstall import install
from pre_commit.commands.install_uninstall import install_hooks
@@ -197,6 +198,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
_add_color_option(clean_parser)
_add_config_option(clean_parser)
hook_impl_parser = subparsers.add_parser('hook-impl')
_add_color_option(hook_impl_parser)
_add_config_option(hook_impl_parser)
hook_impl_parser.add_argument('--hook-type')
hook_impl_parser.add_argument('--hook-dir')
hook_impl_parser.add_argument(
'--skip-on-missing-config', action='store_true',
)
hook_impl_parser.add_argument(dest='rest', nargs=argparse.REMAINDER)
gc_parser = subparsers.add_parser('gc', help='Clean unused cached repos.')
_add_color_option(gc_parser)
_add_config_option(gc_parser)
@@ -329,6 +340,16 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
return clean(store)
elif args.command == 'gc':
return gc(store)
elif args.command == 'hook-impl':
return hook_impl(
store,
config=args.config,
color=args.color,
hook_type=args.hook_type,
hook_dir=args.hook_dir,
skip_on_missing_config=args.skip_on_missing_config,
args=args.rest[1:],
)
elif args.command == 'install':
return install(
args.config, store,

View File

@@ -29,10 +29,8 @@ def find_executable(
environ = _environ if _environ is not None else os.environ
if 'PATHEXT' in environ:
possible_exe_names = tuple(
exe + ext.lower() for ext in environ['PATHEXT'].split(os.pathsep)
) + (exe,)
exts = environ['PATHEXT'].split(os.pathsep)
possible_exe_names = tuple(f'{exe}{ext}' for ext in exts) + (exe,)
else:
possible_exe_names = (exe,)

View File

@@ -1,197 +1,44 @@
#!/usr/bin/env python3
"""File generated by pre-commit: https://pre-commit.com"""
import distutils.spawn
# File generated by pre-commit: https://pre-commit.com
# ID: 138fd403232d2ddd5efb44317e38bf03
import os
import subprocess
import sys
from typing import Callable
from typing import Dict
from typing import Tuple
# we try our best, but the shebang of this script is difficult to determine:
# - macos doesn't ship with python3
# - windows executables are almost always `python.exe`
# therefore we continue to support python2 for this small script
if sys.version_info < (3, 3):
from distutils.spawn import find_executable as which
else:
from shutil import which
# work around https://github.com/Homebrew/homebrew-core/issues/30445
os.environ.pop('__PYVENV_LAUNCHER__', None)
HERE = os.path.dirname(os.path.abspath(__file__))
Z40 = '0' * 40
ID_HASH = '138fd403232d2ddd5efb44317e38bf03'
# start templated
CONFIG = ''
HOOK_TYPE = ''
INSTALL_PYTHON = ''
SKIP_ON_MISSING_CONFIG = False
ARGS = ['hook-impl']
# end templated
ARGS.extend(('--hook-dir', os.path.realpath(os.path.dirname(__file__))))
ARGS.append('--')
ARGS.extend(sys.argv[1:])
class EarlyExit(RuntimeError):
pass
class FatalError(RuntimeError):
pass
def _norm_exe(exe: str) -> Tuple[str, ...]:
"""Necessary for shebang support on windows.
roughly lifted from `identify.identify.parse_shebang`
"""
with open(exe, 'rb') as f:
if f.read(2) != b'#!':
return ()
try:
first_line = f.readline().decode()
except UnicodeDecodeError:
return ()
cmd = first_line.split()
if cmd[0] == '/usr/bin/env':
del cmd[0]
return tuple(cmd)
def _run_legacy() -> Tuple[int, bytes]:
if __file__.endswith('.legacy'):
raise SystemExit(
f"bug: pre-commit's script is installed in migration mode\n"
f'run `pre-commit install -f --hook-type {HOOK_TYPE}` to fix '
f'this\n\n'
f'Please report this bug at '
f'https://github.com/pre-commit/pre-commit/issues',
)
if HOOK_TYPE == 'pre-push':
stdin = sys.stdin.buffer.read()
else:
stdin = b''
legacy_hook = os.path.join(HERE, f'{HOOK_TYPE}.legacy')
if os.access(legacy_hook, os.X_OK):
cmd = _norm_exe(legacy_hook) + (legacy_hook,) + tuple(sys.argv[1:])
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE if stdin else None)
proc.communicate(stdin)
return proc.returncode, stdin
else:
return 0, stdin
def _validate_config() -> None:
cmd = ('git', 'rev-parse', '--show-toplevel')
top_level = subprocess.check_output(cmd).decode().strip()
cfg = os.path.join(top_level, CONFIG)
if os.path.isfile(cfg):
pass
elif SKIP_ON_MISSING_CONFIG or os.getenv('PRE_COMMIT_ALLOW_NO_CONFIG'):
print(f'`{CONFIG}` config file not found. Skipping `pre-commit`.')
raise EarlyExit()
else:
raise FatalError(
f'No {CONFIG} file was found\n'
f'- To temporarily silence this, run '
f'`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n'
f'- To permanently silence this, install pre-commit with the '
f'--allow-missing-config option\n'
f'- To uninstall pre-commit run '
f'`pre-commit uninstall`',
)
def _exe() -> Tuple[str, ...]:
with open(os.devnull, 'wb') as devnull:
for exe in (INSTALL_PYTHON, sys.executable):
try:
if not subprocess.call(
(exe, '-c', 'import pre_commit.main'),
stdout=devnull, stderr=devnull,
):
return (exe, '-m', 'pre_commit.main', 'run')
except OSError:
pass
if distutils.spawn.find_executable('pre-commit'):
return ('pre-commit', 'run')
raise FatalError(
'`pre-commit` not found. Did you forget to activate your virtualenv?',
)
def _rev_exists(rev: str) -> bool:
return not subprocess.call(('git', 'rev-list', '--quiet', rev))
def _pre_push(stdin: bytes) -> Tuple[str, ...]:
remote_name = sys.argv[1]
remote_url = sys.argv[2]
opts: Tuple[str, ...] = ()
for line in stdin.decode().splitlines():
_, local_sha, _, remote_sha = line.split()
if local_sha == Z40:
continue
elif remote_sha != Z40 and _rev_exists(remote_sha):
opts = ('--origin', local_sha, '--source', remote_sha)
else:
# ancestors not found in remote
ancestors = subprocess.check_output((
'git', 'rev-list', local_sha, '--topo-order', '--reverse',
'--not', f'--remotes={remote_name}',
)).decode().strip()
if not ancestors:
continue
else:
first_ancestor = ancestors.splitlines()[0]
cmd = ('git', 'rev-list', '--max-parents=0', local_sha)
roots = set(subprocess.check_output(cmd).decode().splitlines())
if first_ancestor in roots:
# pushing the whole tree including root commit
opts = ('--all-files',)
else:
rev_cmd = ('git', 'rev-parse', f'{first_ancestor}^')
source = subprocess.check_output(rev_cmd).decode().strip()
opts = ('--origin', local_sha, '--source', source)
if opts:
return (
*opts, '--remote-name', remote_name, '--remote-url', remote_url,
)
else:
# An attempt to push an empty changeset
raise EarlyExit()
def _opts(stdin: bytes) -> Tuple[str, ...]:
fns: Dict[str, Callable[[bytes], Tuple[str, ...]]] = {
'prepare-commit-msg': lambda _: ('--commit-msg-filename', sys.argv[1]),
'commit-msg': lambda _: ('--commit-msg-filename', sys.argv[1]),
'pre-merge-commit': lambda _: (),
'pre-commit': lambda _: (),
'pre-push': _pre_push,
}
stage = HOOK_TYPE.replace('pre-', '')
return ('--config', CONFIG, '--hook-stage', stage) + fns[HOOK_TYPE](stdin)
if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
# this is the python 2.7 implementation
def _subprocess_call(cmd: Tuple[str, ...]) -> int:
return subprocess.Popen(cmd).wait()
DNE = '`pre-commit` not found. Did you forget to activate your virtualenv?'
if os.access(INSTALL_PYTHON, os.X_OK):
CMD = [INSTALL_PYTHON, '-mpre_commit']
elif which('pre-commit'):
CMD = ['pre-commit']
else:
_subprocess_call = subprocess.call
raise SystemExit(DNE)
CMD.extend(ARGS)
if sys.platform == 'win32': # https://bugs.python.org/issue19124
import subprocess
def main() -> int:
retv, stdin = _run_legacy()
try:
_validate_config()
return retv | _subprocess_call(_exe() + _opts(stdin))
except EarlyExit:
return retv
except FatalError as e:
print(e.args[0])
return 1
except KeyboardInterrupt:
return 1
if __name__ == '__main__':
exit(main())
if sys.version_info < (3, 7): # https://bugs.python.org/issue25942
raise SystemExit(subprocess.Popen(CMD).wait())
else:
raise SystemExit(subprocess.call(CMD))
else:
os.execvp(CMD[0], CMD)

View File

@@ -0,0 +1,225 @@
import subprocess
import sys
from unittest import mock
import pytest
import pre_commit.constants as C
from pre_commit import git
from pre_commit.commands import hook_impl
from pre_commit.envcontext import envcontext
from pre_commit.util import cmd_output
from pre_commit.util import make_executable
from testing.fixtures import git_dir
from testing.fixtures import sample_local_config
from testing.fixtures import write_config
from testing.util import cwd
from testing.util import git_commit
def test_validate_config_file_exists(tmpdir):
cfg = tmpdir.join(C.CONFIG_FILE).ensure()
hook_impl._validate_config(0, cfg, True)
def test_validate_config_missing(capsys):
with pytest.raises(SystemExit) as excinfo:
hook_impl._validate_config(123, 'DNE.yaml', False)
ret, = excinfo.value.args
assert ret == 1
assert capsys.readouterr().out == (
'No DNE.yaml file was found\n'
'- To temporarily silence this, run '
'`PRE_COMMIT_ALLOW_NO_CONFIG=1 git ...`\n'
'- To permanently silence this, install pre-commit with the '
'--allow-missing-config option\n'
'- To uninstall pre-commit run `pre-commit uninstall`\n'
)
def test_validate_config_skip_missing_config(capsys):
with pytest.raises(SystemExit) as excinfo:
hook_impl._validate_config(123, 'DNE.yaml', True)
ret, = excinfo.value.args
assert ret == 123
expected = '`DNE.yaml` config file not found. Skipping `pre-commit`.\n'
assert capsys.readouterr().out == expected
def test_validate_config_skip_via_env_variable(capsys):
with pytest.raises(SystemExit) as excinfo:
with envcontext((('PRE_COMMIT_ALLOW_NO_CONFIG', '1'),)):
hook_impl._validate_config(0, 'DNE.yaml', False)
ret, = excinfo.value.args
assert ret == 0
expected = '`DNE.yaml` config file not found. Skipping `pre-commit`.\n'
assert capsys.readouterr().out == expected
def test_run_legacy_does_not_exist(tmpdir):
retv, stdin = hook_impl._run_legacy('pre-commit', tmpdir, ())
assert (retv, stdin) == (0, b'')
def test_run_legacy_executes_legacy_script(tmpdir, capfd):
hook = tmpdir.join('pre-commit.legacy')
hook.write('#!/usr/bin/env bash\necho hi "$@"\nexit 1\n')
make_executable(hook)
retv, stdin = hook_impl._run_legacy('pre-commit', tmpdir, ('arg1', 'arg2'))
assert capfd.readouterr().out.strip() == 'hi arg1 arg2'
assert (retv, stdin) == (1, b'')
def test_run_legacy_pre_push_returns_stdin(tmpdir):
with mock.patch.object(sys.stdin.buffer, 'read', return_value=b'stdin'):
retv, stdin = hook_impl._run_legacy('pre-push', tmpdir, ())
assert (retv, stdin) == (0, b'stdin')
def test_run_legacy_recursive(tmpdir):
hook = tmpdir.join('pre-commit.legacy').ensure()
make_executable(hook)
# simulate a call being recursive
def call(*_, **__):
return hook_impl._run_legacy('pre-commit', tmpdir, ())
with mock.patch.object(subprocess, 'run', call):
with pytest.raises(SystemExit):
call()
def test_run_ns_pre_commit():
ns = hook_impl._run_ns('pre-commit', True, (), b'')
assert ns is not None
assert ns.hook_stage == 'commit'
assert ns.color is True
def test_run_ns_commit_msg():
ns = hook_impl._run_ns('commit-msg', False, ('.git/COMMIT_MSG',), b'')
assert ns is not None
assert ns.hook_stage == 'commit-msg'
assert ns.color is False
assert ns.commit_msg_filename == '.git/COMMIT_MSG'
@pytest.fixture
def push_example(tempdir_factory):
src = git_dir(tempdir_factory)
git_commit(cwd=src)
src_head = git.head_rev(src)
clone = tempdir_factory.get()
cmd_output('git', 'clone', src, clone)
git_commit(cwd=clone)
clone_head = git.head_rev(clone)
return (src, src_head, clone, clone_head)
def test_run_ns_pre_push_updating_branch(push_example):
src, src_head, clone, clone_head = push_example
with cwd(clone):
args = ('origin', src)
stdin = f'HEAD {clone_head} refs/heads/b {src_head}\n'.encode()
ns = hook_impl._run_ns('pre-push', False, args, stdin)
assert ns is not None
assert ns.hook_stage == 'push'
assert ns.color is False
assert ns.remote_name == 'origin'
assert ns.remote_url == src
assert ns.source == src_head
assert ns.origin == clone_head
assert ns.all_files is False
def test_run_ns_pre_push_new_branch(push_example):
src, src_head, clone, clone_head = push_example
with cwd(clone):
args = ('origin', src)
stdin = f'HEAD {clone_head} refs/heads/b {hook_impl.Z40}\n'.encode()
ns = hook_impl._run_ns('pre-push', False, args, stdin)
assert ns is not None
assert ns.source == src_head
assert ns.origin == clone_head
def test_run_ns_pre_push_new_branch_existing_rev(push_example):
src, src_head, clone, _ = push_example
with cwd(clone):
args = ('origin', src)
stdin = f'HEAD {src_head} refs/heads/b2 {hook_impl.Z40}\n'.encode()
ns = hook_impl._run_ns('pre-push', False, args, stdin)
assert ns is None
def test_pushing_orphan_branch(push_example):
src, src_head, clone, _ = push_example
cmd_output('git', 'checkout', '--orphan', 'b2', cwd=clone)
git_commit(cwd=clone, msg='something else to get unique hash')
clone_rev = git.head_rev(clone)
with cwd(clone):
args = ('origin', src)
stdin = f'HEAD {clone_rev} refs/heads/b2 {hook_impl.Z40}\n'.encode()
ns = hook_impl._run_ns('pre-push', False, args, stdin)
assert ns is not None
assert ns.all_files is True
def test_run_ns_pre_push_deleting_branch(push_example):
src, src_head, clone, _ = push_example
with cwd(clone):
args = ('origin', src)
stdin = f'(delete) {hook_impl.Z40} refs/heads/b {src_head}'.encode()
ns = hook_impl._run_ns('pre-push', False, args, stdin)
assert ns is None
def test_hook_impl_main_noop_pre_push(cap_out, store, push_example):
src, src_head, clone, _ = push_example
stdin = f'(delete) {hook_impl.Z40} refs/heads/b {src_head}'.encode()
with mock.patch.object(sys.stdin.buffer, 'read', return_value=stdin):
with cwd(clone):
write_config('.', sample_local_config())
ret = hook_impl.hook_impl(
store,
config=C.CONFIG_FILE,
color=False,
hook_type='pre-push',
hook_dir='.git/hooks',
skip_on_missing_config=False,
args=('origin', src),
)
assert ret == 0
assert cap_out.get() == ''
def test_hook_impl_main_runs_hooks(cap_out, tempdir_factory, store):
with cwd(git_dir(tempdir_factory)):
write_config('.', sample_local_config())
ret = hook_impl.hook_impl(
store,
config=C.CONFIG_FILE,
color=False,
hook_type='pre-commit',
hook_dir='.git/hooks',
skip_on_missing_config=False,
args=(),
)
assert ret == 0
expected = '''\
Block if "DO NOT COMMIT" is found....................(no files to check)Skipped
'''
assert cap_out.get() == expected

View File

@@ -51,7 +51,8 @@ def test_shebang_posix_not_on_path():
def test_shebang_posix_on_path(tmpdir):
tmpdir.join(f'python{sys.version_info[0]}').ensure()
exe = tmpdir.join(f'python{sys.version_info[0]}').ensure()
make_executable(exe)
with mock.patch.object(sys, 'platform', 'posix'):
with mock.patch.object(os, 'defpath', tmpdir.strpath):

View File

@@ -81,8 +81,8 @@ def test_adjust_args_try_repo_repo_relative(in_git_dir):
FNS = (
'autoupdate', 'clean', 'gc', 'install', 'install_hooks', 'migrate_config',
'run', 'sample_config', 'uninstall',
'autoupdate', 'clean', 'gc', 'hook_impl', 'install', 'install_hooks',
'migrate_config', 'run', 'sample_config', 'uninstall',
)
CMDS = tuple(fn.replace('_', '-') for fn in FNS)

View File

@@ -1,6 +1,6 @@
import contextlib
import distutils.spawn
import os
import os.path
import shutil
import sys
import pytest
@@ -12,7 +12,7 @@ from pre_commit.util import make_executable
def _echo_exe() -> str:
exe = distutils.spawn.find_executable('echo')
exe = shutil.which('echo')
assert exe is not None
return exe