Move all our CLI file formatters to the format dir (#13296)

* move all our file formatters to the format dir

* Apply suggestions from code review

Co-authored-by: Erovia <Erovia@users.noreply.github.com>

Co-authored-by: Erovia <Erovia@users.noreply.github.com>
This commit is contained in:
Zach White 2021-07-20 11:52:14 -07:00 committed by GitHub
parent c4db9f7fb2
commit 4ab8734d6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 240 additions and 148 deletions

View file

@ -40,7 +40,10 @@ subcommands = [
'qmk.cli.doctor', 'qmk.cli.doctor',
'qmk.cli.fileformat', 'qmk.cli.fileformat',
'qmk.cli.flash', 'qmk.cli.flash',
'qmk.cli.format.c',
'qmk.cli.format.json', 'qmk.cli.format.json',
'qmk.cli.format.python',
'qmk.cli.format.text',
'qmk.cli.generate.api', 'qmk.cli.generate.api',
'qmk.cli.generate.config_h', 'qmk.cli.generate.config_h',
'qmk.cli.generate.dfu_header', 'qmk.cli.generate.dfu_header',

139
lib/python/qmk/cli/cformat.py Normal file → Executable file
View file

@ -1,137 +1,28 @@
"""Format C code according to QMK's style. """Point people to the new command name.
""" """
from os import path import sys
from shutil import which from pathlib import Path
from subprocess import CalledProcessError, DEVNULL, Popen, PIPE
from argcomplete.completers import FilesCompleter
from milc import cli from milc import cli
from qmk.path import normpath
from qmk.c_parse import c_source_files
c_file_suffixes = ('c', 'h', 'cpp')
core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios')
def find_clang_format():
"""Returns the path to clang-format.
"""
for clang_version in range(20, 6, -1):
binary = f'clang-format-{clang_version}'
if which(binary):
return binary
return 'clang-format'
def find_diffs(files):
"""Run clang-format and diff it against a file.
"""
found_diffs = False
for file in files:
cli.log.debug('Checking for changes in %s', file)
clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True)
diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True)
if diff.returncode != 0:
print(diff.stdout)
found_diffs = True
return found_diffs
def cformat_run(files):
"""Spawn clang-format subprocess with proper arguments
"""
# Determine which version of clang-format to use
clang_format = [find_clang_format(), '-i']
try:
cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL)
cli.log.info('Successfully formatted the C code.')
return True
except CalledProcessError as e:
cli.log.error('Error formatting C code!')
cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode)
cli.log.debug('STDOUT:')
cli.log.debug(e.stdout)
cli.log.debug('STDERR:')
cli.log.debug(e.stderr)
return False
def filter_files(files, core_only=False):
"""Yield only files to be formatted and skip the rest
"""
if core_only:
# Filter non-core files
for index, file in enumerate(files):
# The following statement checks each file to see if the file path is
# - in the core directories
# - not in the ignored directories
if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored):
files[index] = None
cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file)
for file in files:
if file and file.name.split('.')[-1] in c_file_suffixes:
yield file
else:
cli.log.debug('Skipping file %s', file)
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') @cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') @cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') @cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.')
@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') @cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) @cli.subcommand('Pointer to the new command name: qmk format-c.', hidden=True)
def cformat(cli): def cformat(cli):
"""Format C code according to QMK's style. """Pointer to the new command name: qmk format-c.
""" """
# Find the list of files to format cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.')
if cli.args.files: argv = [sys.executable, *sys.argv]
files = list(filter_files(cli.args.files, cli.args.core_only)) argv[argv.index('cformat')] = 'format-c'
script_path = Path(argv[1])
script_path_exe = Path(f'{argv[1]}.exe')
if not files: if not script_path.exists() and script_path_exe.exists():
cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files))) # For reasons I don't understand ".exe" is stripped from the script name on windows.
exit(0) argv[1] = str(script_path_exe)
if cli.args.all_files: return cli.run(argv, capture_output=False).returncode
cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))
elif cli.args.all_files:
all_files = c_source_files(core_dirs)
files = list(filter_files(all_files, True))
else:
git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs]
git_diff = cli.run(git_diff_cmd, stdin=DEVNULL)
if git_diff.returncode != 0:
cli.log.error("Error running %s", git_diff_cmd)
print(git_diff.stderr)
return git_diff.returncode
files = []
for file in git_diff.stdout.strip().split('\n'):
if not any([file.startswith(ignore) for ignore in ignored]):
if path.exists(file) and file.split('.')[-1] in c_file_suffixes:
files.append(file)
# Sanity check
if not files:
cli.log.error('No changed files detected. Use "qmk cformat -a" to format all core files')
return False
# Run clang-format on the files we've found
if cli.args.dry_run:
return not find_diffs(files)
else:
return cformat_run(files)

24
lib/python/qmk/cli/fileformat.py Normal file → Executable file
View file

@ -1,13 +1,23 @@
"""Format files according to QMK's style. """Point people to the new command name.
""" """
import sys
from pathlib import Path
from milc import cli from milc import cli
import subprocess
@cli.subcommand('Pointer to the new command name: qmk format-text.', hidden=True)
@cli.subcommand("Format files according to QMK's style.", hidden=True)
def fileformat(cli): def fileformat(cli):
"""Run several general formatting commands. """Pointer to the new command name: qmk format-text.
""" """
dos2unix = subprocess.run(['bash', '-c', 'git ls-files -z | xargs -0 dos2unix'], stdout=subprocess.DEVNULL) cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.')
return dos2unix.returncode argv = [sys.executable, *sys.argv]
argv[argv.index('fileformat')] = 'format-text'
script_path = Path(argv[1])
script_path_exe = Path(f'{argv[1]}.exe')
if not script_path.exists() and script_path_exe.exists():
# For reasons I don't understand ".exe" is stripped from the script name on windows.
argv[1] = str(script_path_exe)
return cli.run(argv, capture_output=False).returncode

View file

@ -0,0 +1,137 @@
"""Format C code according to QMK's style.
"""
from os import path
from shutil import which
from subprocess import CalledProcessError, DEVNULL, Popen, PIPE
from argcomplete.completers import FilesCompleter
from milc import cli
from qmk.path import normpath
from qmk.c_parse import c_source_files
c_file_suffixes = ('c', 'h', 'cpp')
core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios')
def find_clang_format():
"""Returns the path to clang-format.
"""
for clang_version in range(20, 6, -1):
binary = f'clang-format-{clang_version}'
if which(binary):
return binary
return 'clang-format'
def find_diffs(files):
"""Run clang-format and diff it against a file.
"""
found_diffs = False
for file in files:
cli.log.debug('Checking for changes in %s', file)
clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True)
diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True)
if diff.returncode != 0:
print(diff.stdout)
found_diffs = True
return found_diffs
def cformat_run(files):
"""Spawn clang-format subprocess with proper arguments
"""
# Determine which version of clang-format to use
clang_format = [find_clang_format(), '-i']
try:
cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL)
cli.log.info('Successfully formatted the C code.')
return True
except CalledProcessError as e:
cli.log.error('Error formatting C code!')
cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode)
cli.log.debug('STDOUT:')
cli.log.debug(e.stdout)
cli.log.debug('STDERR:')
cli.log.debug(e.stderr)
return False
def filter_files(files, core_only=False):
"""Yield only files to be formatted and skip the rest
"""
if core_only:
# Filter non-core files
for index, file in enumerate(files):
# The following statement checks each file to see if the file path is
# - in the core directories
# - not in the ignored directories
if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored):
files[index] = None
cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file)
for file in files:
if file and file.name.split('.')[-1] in c_file_suffixes:
yield file
else:
cli.log.debug('Skipping file %s', file)
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.')
@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.')
@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True)
def format_c(cli):
"""Format C code according to QMK's style.
"""
# Find the list of files to format
if cli.args.files:
files = list(filter_files(cli.args.files, cli.args.core_only))
if not files:
cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files)))
exit(0)
if cli.args.all_files:
cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))
elif cli.args.all_files:
all_files = c_source_files(core_dirs)
files = list(filter_files(all_files, True))
else:
git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs]
git_diff = cli.run(git_diff_cmd, stdin=DEVNULL)
if git_diff.returncode != 0:
cli.log.error("Error running %s", git_diff_cmd)
print(git_diff.stderr)
return git_diff.returncode
files = []
for file in git_diff.stdout.strip().split('\n'):
if not any([file.startswith(ignore) for ignore in ignored]):
if path.exists(file) and file.split('.')[-1] in c_file_suffixes:
files.append(file)
# Sanity check
if not files:
cli.log.error('No changed files detected. Use "qmk cformat -a" to format all core files')
return False
# Run clang-format on the files we've found
if cli.args.dry_run:
return not find_diffs(files)
else:
return cformat_run(files)

View file

@ -0,0 +1,26 @@
"""Format python code according to QMK's style.
"""
from subprocess import CalledProcessError, DEVNULL
from milc import cli
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True)
def format_python(cli):
"""Format python code according to QMK's style.
"""
edit = '--diff' if cli.args.dry_run else '--in-place'
yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python']
try:
cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL)
cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.')
return True
except CalledProcessError:
if cli.args.dry_run:
cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!')
else:
cli.log.error('Error formatting python code!')
return False

View file

@ -0,0 +1,27 @@
"""Ensure text files have the proper line endings.
"""
from subprocess import CalledProcessError
from milc import cli
@cli.subcommand("Ensure text files have the proper line endings.", hidden=True)
def format_text(cli):
"""Ensure text files have the proper line endings.
"""
try:
file_list_cmd = cli.run(['git', 'ls-files', '-z'], check=True)
except CalledProcessError as e:
cli.log.error('Could not get file list: %s', e)
exit(1)
except Exception as e:
cli.log.error('Unhandled exception: %s: %s', e.__class__.__name__, e)
cli.log.exception(e)
exit(1)
dos2unix = cli.run(['xargs', '-0', 'dos2unix'], stdin=None, input=file_list_cmd.stdout)
if dos2unix.returncode != 0:
print(dos2unix.stderr)
return dos2unix.returncode

View file

@ -1,26 +1,24 @@
"""Format python code according to QMK's style. """Point people to the new command name.
""" """
from subprocess import CalledProcessError, DEVNULL import sys
from pathlib import Path
from milc import cli from milc import cli
@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) @cli.subcommand('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True)
def pyformat(cli): def pyformat(cli):
"""Format python code according to QMK's style. """Pointer to the new command name: qmk format-python.
""" """
edit = '--diff' if cli.args.dry_run else '--in-place' cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.')
yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] argv = [sys.executable, *sys.argv]
try: argv[argv.index('pyformat')] = 'format-python'
cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) script_path = Path(argv[1])
cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') script_path_exe = Path(f'{argv[1]}.exe')
return True
except CalledProcessError: if not script_path.exists() and script_path_exe.exists():
if cli.args.dry_run: # For reasons I don't understand ".exe" is stripped from the script name on windows.
cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') argv[1] = str(script_path_exe)
else:
cli.log.error('Error formatting python code!')
return False return cli.run(argv, capture_output=False).returncode