Enable git pre hook (#188)

* Port PyTorch code quanlity tools to IPEX.
[We use clang-tidy and flake8  to perform additional
formatting and semantic checking of code. We provide a pre-commit git hook for
performing these checks, before a commit is created -- ln -s ../../tools/git-pre-commit .git/hooks/pre-commit

* Refine the code to ensure the git pre hook to work normally

Author: EikanWang

.clang-format

@@ -0,0 +1,88 @@
+---
+AccessModifierOffset: -1
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   false
+AlignTrailingComments: false
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: false
+BinPackParameters: false
+BraceWrapping:
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: false
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ FOR_EACH_RANGE, FOR_EACH, ]
+IncludeCategories:
+  - Regex:           '^<.*\.h(pp)?>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IndentCaseLabels: true
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 2000000
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        8
+UseTab:          Never
+...

.clang-tidy

@@ -0,0 +1,41 @@
+---
+# NOTE there must be no spaces before the '-', so put the comma last.
+InheritParentConfig: true
+Checks: '
+bugprone-*,
+-bugprone-forward-declaration-namespace,
+-bugprone-macro-parentheses,
+-bugprone-lambda-function-name,
+-bugprone-reserved-identifier,
+cppcoreguidelines-*,
+-cppcoreguidelines-avoid-magic-numbers,
+-cppcoreguidelines-interfaces-global-init,
+-cppcoreguidelines-macro-usage,
+-cppcoreguidelines-owning-memory,
+-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+-cppcoreguidelines-pro-bounds-constant-array-index,
+-cppcoreguidelines-pro-bounds-pointer-arithmetic,
+-cppcoreguidelines-pro-type-cstyle-cast,
+-cppcoreguidelines-pro-type-reinterpret-cast,
+-cppcoreguidelines-pro-type-static-cast-downcast,
+-cppcoreguidelines-pro-type-union-access,
+-cppcoreguidelines-pro-type-vararg,
+-cppcoreguidelines-special-member-functions,
+-facebook-hte-RelativeInclude,
+hicpp-exception-baseclass,
+hicpp-avoid-goto,
+modernize-*,
+-modernize-concat-nested-namespaces,
+-modernize-return-braced-init-list,
+-modernize-use-auto,
+-modernize-use-default-member-init,
+-modernize-use-using,
+-modernize-use-trailing-return-type,
+performance-*,
+-performance-noexcept-move-constructor,
+-performance-unnecessary-value-param,
+'
+HeaderFilterRegex: 'torch_ipex/csrc/.*'
+AnalyzeTemporaryDtors: false
+CheckOptions:
+...

.clang-tidy-oss

@@ -0,0 +1,43 @@
+---
+# NOTE there must be no spaces before the '-', so put the comma last.
+InheritParentConfig: true
+Checks: '
+bugprone-*,
+-bugprone-forward-declaration-namespace,
+-bugprone-macro-parentheses,
+-bugprone-lambda-function-name,
+-bugprone-reserved-identifier,
+cppcoreguidelines-*,
+-cppcoreguidelines-avoid-magic-numbers,
+-cppcoreguidelines-avoid-non-const-global-variables,
+-cppcoreguidelines-interfaces-global-init,
+-cppcoreguidelines-macro-usage,
+-cppcoreguidelines-owning-memory,
+-cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+-cppcoreguidelines-pro-bounds-constant-array-index,
+-cppcoreguidelines-pro-bounds-pointer-arithmetic,
+-cppcoreguidelines-pro-type-cstyle-cast,
+-cppcoreguidelines-pro-type-reinterpret-cast,
+-cppcoreguidelines-pro-type-static-cast-downcast,
+-cppcoreguidelines-pro-type-union-access,
+-cppcoreguidelines-pro-type-vararg,
+-cppcoreguidelines-special-member-functions,
+-facebook-hte-RelativeInclude,
+hicpp-exception-baseclass,
+hicpp-avoid-goto,
+modernize-*,
+-modernize-concat-nested-namespaces,
+-modernize-return-braced-init-list,
+-modernize-use-auto,
+-modernize-use-default-member-init,
+-modernize-use-using,
+-modernize-use-trailing-return-type,
+performance-*,
+-performance-noexcept-move-constructor,
+-performance-unnecessary-value-param,
+'
+HeaderFilterRegex: 'torch_ipex/csrc/.*'
+AnalyzeTemporaryDtors: false
+WarningsAsErrors: '*'
+CheckOptions:
+...

.flake8

@@ -0,0 +1,22 @@
+[flake8]
+select = B,C,E,F,P,T4,W,B9
+max-line-length = 120
+# C408 ignored because we like the dict keyword argument syntax
+# E501 is not flexible enough, we're using B950 instead
+ignore =
+    E203,E305,E402,E501,E721,E741,F405,F821,F841,F999,W503,W504,C408,E302,W291,E303,
+    # shebang has extra meaning in fbcode lints, so I think it's not worth trying
+    # to line this up with executable bit
+    EXE001,
+    # these ignores are from flake8-bugbear; please fix!
+    B007,B008,
+    # these ignores are from flake8-comprehensions; please fix!
+    C400,C401,C402,C403,C404,C405,C407,C411,C413,C414,C415
+per-file-ignores = __init__.py: F401 
+optional-ascii-coding = True
+exclude =
+    ./.git,
+    ./build,
+    ./scripts,
+    ./tests/cpu,
+    ./third_party,

.gitignore

@@ -207,6 +207,10 @@ gen
 .pytest_cache
 aten/build/*
 
+.clang-format-bin/
+.clang-tidy-bin/
+
+
 # Bram
 plsdontbreak
 

tools/clang_format_hash/linux64/clang-format-linux64

@@ -0,0 +1 @@
+21ca53c291a88b53dac85751b7a0203ca610ac94b7adaff3c092cf30df4168f2

tools/git-pre-commit

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+echo "Running pre-commit flake8"
+python3 tools/linter/flake8_hook.py
+
+echo "Running pre-commit clang-tidy"
+git diff HEAD > pr.diff
+python3 tools/linter/clang_tidy --diff-file "pr.diff"
+rm pr.diff
+
+echo "Running pre-commit clang-format"
+tools/linter/git-clang-format HEAD~ --force

tools/linter/__init__.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""
+A script that runs clang-format on all C/C++ files in CLANG_FORMAT_ALLOWLIST. There is
+also a diff mode which simply checks if clang-format would make any changes, which is useful for
+CI purposes.
+
+If clang-format is not available, the script also downloads a platform-appropriate binary from
+and S3 bucket and verifies it against a precommited set of blessed binary hashes.
+"""
+import argparse
+import asyncio
+import re
+import os
+import sys
+from typing import List, Set
+
+from .clang_format_utils import get_and_check_clang_format, CLANG_FORMAT_PATH
+
+# Allowlist of directories to check. All files that in that directory
+# (recursively) will be checked.
+# If you edit this, please edit the allowlist in clang_format_ci.sh as well.
+CLANG_FORMAT_ALLOWLIST = [
+    "torch_ipex/csrc"
+]
+
+# Only files with names matching this regex will be formatted.
+CPP_FILE_REGEX = re.compile(".*\\.(h|cpp|cc|c|hpp)$")
+
+
+def get_allowlisted_files() -> Set[str]:
+    """
+    Parse CLANG_FORMAT_ALLOWLIST and resolve all directories.
+    Returns the set of allowlist cpp source files.
+    """
+    matches = []
+    for dir in CLANG_FORMAT_ALLOWLIST:
+        for root, dirnames, filenames in os.walk(dir):
+            for filename in filenames:
+                if CPP_FILE_REGEX.match(filename):
+                    matches.append(os.path.join(root, filename))
+    return set(matches)
+
+
+async def run_clang_format_on_file(
+    filename: str,
+    semaphore: asyncio.Semaphore,
+    verbose: bool = False,
+) -> None:
+    """
+    Run clang-format on the provided file.
+    """
+    # -style=file picks up the closest .clang-format, -i formats the files inplace.
+    cmd = "{} -style=file -i {}".format(CLANG_FORMAT_PATH, filename)
+    async with semaphore:
+        proc = await asyncio.create_subprocess_shell(cmd)
+        _ = await proc.wait()
+    if verbose:
+        print("Formatted {}".format(filename))
+
+
+async def file_clang_formatted_correctly(
+    filename: str,
+    semaphore: asyncio.Semaphore,
+    verbose: bool = False,
+) -> bool:
+    """
+    Checks if a file is formatted correctly and returns True if so.
+    """
+    ok = True
+    # -style=file picks up the closest .clang-format
+    cmd = "{} -style=file {}".format(CLANG_FORMAT_PATH, filename)
+
+    async with semaphore:
+        proc = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE)
+        # Read back the formatted file.
+        stdout, _ = await proc.communicate()
+
+    formatted_contents = stdout.decode()
+    # Compare the formatted file to the original file.
+    with open(filename) as orig:
+        orig_contents = orig.read()
+        if formatted_contents != orig_contents:
+            ok = False
+            if verbose:
+                print("{} is not formatted correctly".format(filename))
+
+    return ok
+
+
+async def run_clang_format(
+    max_processes: int,
+    diff: bool = False,
+    verbose: bool = False,
+) -> bool:
+    """
+    Run clang-format to all files in CLANG_FORMAT_ALLOWLIST that match CPP_FILE_REGEX.
+    """
+    # Check to make sure the clang-format binary exists.
+    if not os.path.exists(CLANG_FORMAT_PATH):
+        print("clang-format binary not found")
+        return False
+
+    # Gather command-line options for clang-format.
+    args = [CLANG_FORMAT_PATH, "-style=file"]
+
+    if not diff:
+        args.append("-i")
+
+    ok = True
+
+    # Semaphore to bound the number of subprocesses that can be created at once to format files.
+    semaphore = asyncio.Semaphore(max_processes)
+
+    # Format files in parallel.
+    if diff:
+        for f in asyncio.as_completed([file_clang_formatted_correctly(f, semaphore, verbose) for f in get_allowlisted_files()]):
+            ok &= await f
+
+        if ok:
+            print("All files formatted correctly")
+        else:
+            print("Some files not formatted correctly")
+    else:
+        await asyncio.gather(*[run_clang_format_on_file(f, semaphore, verbose) for f in get_allowlisted_files()])
+
+    return ok
+
+def parse_args(args: List[str]) -> argparse.Namespace:
+    """
+    Parse and return command-line arguments.
+    """
+    parser = argparse.ArgumentParser(
+        description="Execute clang-format on your working copy changes."
+    )
+    parser.add_argument(
+        "-d",
+        "--diff",
+        action="store_true",
+        default=False,
+        help="Determine whether running clang-format would produce changes",
+    )
+    parser.add_argument("--verbose", "-v", action="store_true", default=False)
+    parser.add_argument("--max-processes", type=int, default=50,
+                        help="Maximum number of subprocesses to create to format files in parallel")
+    return parser.parse_args(args)
+
+
+def main(args: List[str]) -> bool:
+    # Parse arguments.
+    options = parse_args(args)
+    # Get clang-format and make sure it is the right binary and it is in the right place.
+    ok = get_and_check_clang_format(options.verbose)
+    # Invoke clang-format on all files in the directories in the allowlist.
+    if ok:
+        loop = asyncio.get_event_loop()
+        ok = loop.run_until_complete(run_clang_format(options.max_processes, options.diff, options.verbose))
+
+    # We have to invert because False -> 0, which is the code to be returned if everything is okay.
+    return not ok
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))