From b23c8dfcfcf96eaab4f53bc607e9fde519ae3ebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 02:31:49 +0200 Subject: [PATCH 01/96] Add custom code linter --- Scripts/find_code_issues.py | 434 ++++++++++++++++++++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 Scripts/find_code_issues.py diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py new file mode 100644 index 00000000000..b5a3aef4e09 --- /dev/null +++ b/Scripts/find_code_issues.py @@ -0,0 +1,434 @@ +#!/usr/bin/env python3 + +# Copyright 2019-2020 CERN and copyright holders of ALICE O2. +# See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +# All rights not expressly granted are reserved. +# +# This software is distributed under the terms of the GNU General Public +# License v3 (GPL Version 3), copied verbatim in the file "COPYING". +# +# In applying this license CERN does not waive the privileges and immunities +# granted to it by virtue of its status as an Intergovernmental Organization +# or submit itself to any jurisdiction. + +"""! +@brief Find issues in code +@author Vít Kučera , Inha University +@date 2024-07-14 +""" + +from abc import ABC #, abstractmethod +import argparse +import os +import sys + + +class TestSpec(ABC): + """Prototype of the test class""" + name = "Test template" # name of the test + message = "Test failed" # error message + suffixes = [] # suffixes of files to test + per_line = True # test lines separately one by one + + def file_matches(self, path: str) -> bool: + """Test whether the path matches the pattern for files to test""" + return path.endswith(tuple(self.suffixes)) if self.suffixes else True + + # @abstractmethod + def test_line(self, line: str) -> bool: + """Test a line.""" + raise NotImplementedError() + + # @abstractmethod + def test_file(self, path : str, content) -> bool: + """Test a file in a way that cannot be done line by line.""" + raise NotImplementedError() + + def run(self, path : str, content, github: bool = False) -> bool: + """Test file content.""" + # print(content) + passed = True + if not self.file_matches(path): + return passed + # print(f"Running test {self.name} for {path} with {len(content)} lines") + if self.per_line: + for i, line in enumerate(content): + line = line.strip() + # print(i + 1, line) + if not self.test_line(line): + passed = False + print(f"{path}:{i + 1}: {self.message} [{self.name}]") + if github: + print(f"::error file={path},line={i + 1},title=[{self.name}]::{self.message}") + else: + passed = self.test_file(path, content) + if not passed: + print(f"{path}: {self.message} [{self.name}]") + if github: + print(f"::error file={path},title=[{self.name}]::{self.message}") + return passed + + +########################## +# Implementations of tests +########################## + +# Bad code + +class TestIOStream(TestSpec): + name = "Include iostream" + message = "Including iostream is not allowed. Use O2 logging instead." + suffixes = [".h", ".cxx"] + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + return not line.strip().startswith("#include ") + + +class TestUsingStd(TestSpec): + name = "Importing std names" + message = "Importing std names is not allowed." + suffixes = [".h", ".cxx"] + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + return not line.strip().startswith("using std::") + + +class TestUsingDirectives(TestSpec): + name = "Using directives" + message = "Using directives are not allowed in headers." + suffixes = [".h"] + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + return not line.strip().startswith("using namespace") + + +class TestStdPrefix(TestSpec): + name = "std prefix" + message = "Use std:: prefix for STD methods." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + return not " abs(" in line + + +class TestROOT(TestSpec): + """Test use of ROOT entities""" + name = "ROOT entities" + message = "Consider replacing ROOT entities with STD C++ or O2 entities." + suffixes = [".h", ".cxx"] + keywords = ["TMath", "Double_t", "Float_t", "Int_t", "Bool_t"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and not "Macros/" in path + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + for k in self.keywords: + if k in line: + return False + return True + + +class TestPI(TestSpec): + """Test use of PI""" + name = "Use of PI" + message = "Consider using the PI constant (and its multiples) defined in o2::constants::math." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + return not "M_PI" in line + + +class TestLogging(TestSpec): + """Test logging""" + name = "Use of logging" + message = "Consider using O2 logging with LOG, LOGF, LOGP." + suffixes = [".h", ".cxx"] + keywords = ["Printf(", "printf(", "cout <", "cin >"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and not "Macros/" in path + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + for k in self.keywords: + if k in line: + return False + return True + + +# Naming conventions + + +class TestNameFunction(TestSpec): + """Test function names. + Likely to report false positives. + Can accidentally spot names of variable too but it is fine because same conventions apply. + """ + name = "function names" + message = "Use camelCase for names of functions and variables." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + # Look for "Type Name(..." + # regexp ? + words = line.split("(") + if len(words) < 2: + return True + words = words[0].split() + if len(words) != 2: + return True + if not words[1].isalnum(): + return True + if words[0] in ("return", ":", "#define", "#if", "new", "virtual", "case"): + return True + if words[0].endswith(".template"): + return True + if words[0][-1] == ",": # multiple template arguments + return True + # Extract function name. + function_name = words[1] + if "::" in function_name: + function_name = function_name.split("::")[1] + # The actual test comes here. + # if not (function_name[0].islower() and not "_" in function_name and not "-" in function_name): + # print(line, words) + # return False + # return True + return function_name[0].islower() and not "_" in function_name and not "-" in function_name + + +class TestNameMacro(TestSpec): + """Test macro names""" + name = "macro names" + message = "Use capital letters and underscores in macro names." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + if not line.startswith("#define "): + return True + # Extract macro name. + macro_name = line.split()[1] + if "(" in macro_name: + macro_name = macro_name.split("(")[0] + # The actual test comes here. + return macro_name.isupper() and not "-" in macro_name + + +class TestNameNamespace(TestSpec): + """Test names of namespaces.""" + name = "namespace names" + message = "Use lowercase letters and underscores in names of namespaces." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + if not line.startswith("namespace "): + return True + # Extract namespace name. + namespace_name = line.split()[1] + # if not namespace_name.islower(): + # print(namespace_name) + # return False + # return True + # The actual test comes here. + return namespace_name.islower() and not "-" in namespace_name + + +class TestNameUpperCamelCase(TestSpec): + """Test CamelCase of a name.""" + keyword = "key" + name = f"{keyword} CamelCase" + message = f"Use CamelCase for names of {keyword}." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + if not line.startswith(f"{self.keyword} "): + return True + # Extract object name. + words = line.split() + if not words[1].isalnum(): # "struct : ...", "enum { ..." + return True + object_name = words[1] + if words[1] == "class" and len(words) > 2: # enum class + object_name = words[2] + # if not object_name[0].isupper(): + # print(object_name) + # return False + # return True + # The actual test comes here. + return object_name[0].isupper() and not "_" in object_name and not "-" in object_name + + +class TestNameEnum(TestNameUpperCamelCase): + """Test names of enumerators. + Issue with 'enum class'""" + keyword = "enum" + name = f"{keyword} names" + message = f"Use CamelCase for names of enumerators and their values." + + +class TestNameClass(TestNameUpperCamelCase): + """Test names of classes.""" + keyword = "class" + name = f"{keyword} names" + message = f"Use CamelCase for names of classes." + + +class TestNameStruct(TestNameUpperCamelCase): + """Test names of structs.""" + keyword = "struct" + name = f"{keyword} names" + message = f"Use CamelCase for names of structs." + + +class TestNameFileCpp(TestSpec): + """Test names of C++ files.""" + name = "C++ file names" + message = "Use camelCase or CamelCase for names of C++ files. See the O2 naming conventions for details." + suffixes = [".h", ".cxx", ".C"] + per_line = False + + def test_file(self, path : str, content) -> bool: + file_name = os.path.basename(path) + return not "_" in file_name and not "-" in file_name + + +class TestNameFilePython(TestSpec): + """Test names of Python files.""" + name = "Python file names" + message = "Use lowercase letters and underscores in names of Python files." + suffixes = [".py", ".ipynb"] + per_line = False + + def test_file(self, path : str, content) -> bool: + file_name = os.path.basename(path) + return file_name.islower() and not "-" in file_name + + +# PWG-specific + +class TestHfConstAuto(TestSpec): + name = "PWGHF: const auto" + message = "Use \"const auto\" instead of \"auto const\"" + suffixes = [".h", ".cxx"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "PWGHF/" in path + + def test_line(self, line: str) -> bool: + return not "auto const" in line + + +class TestHfStructMembers(TestSpec): + """Test order of struct members.""" + name = "PWGHF: struct memners" + message = "Order struct members." + suffixes = [".h", ".cxx"] + per_line = False + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "PWGHF/" in path + + def test_file(self, path : str, content) -> bool: + return True + + +# End of test implementations + + +def main(): + """Main function""" + parser = argparse.ArgumentParser( + description="Run tests on files to find bad code." + ) + parser.add_argument("paths", type=str, nargs="+", help="File path(s)") + parser.add_argument( + "-g", + dest="github", + action="store_true", + help="Print messages as GitHub annotations", + ) + args = parser.parse_args() + + tests = [] + + # Bad code + # tests.append(TestIOStream()) + # tests.append(TestUsingDirectives()) + # tests.append(TestUsingStd()) + # tests.append(TestStdPrefix()) + # tests.append(TestROOT()) + # tests.append(TestPI()) + # tests.append(TestLogging()) + + # Naming conventions + # tests.append(TestNameFunction()) + # tests.append(TestNameMacro()) + # tests.append(TestNameNamespace()) + tests.append(TestNameEnum()) + # tests.append(TestNameClass()) + # tests.append(TestNameStruct()) + tests.append(TestNameFileCpp()) + # tests.append(TestNameFilePython()) + + # PWG + # tests.append(TestHfConstAuto()) + # tests.append(TestHfStructMembers()) + + test_names = [t.name for t in tests] + suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests + passed = True # global status + + print("Enabled tests:", test_names) + + print(f"Testing {len(args.paths)} files.") + for path in args.paths: + # Skip not tested files. + if not path.endswith(suffixes): + continue + try: + with open(path, "r") as file: + content = file.readlines() + for test in tests: + if not test.run(path, content, args.github): + passed = False + # if not passed: + # print(f"File {path} failed the test {test.name}.") + # print(f"Test {'passed' if passed else 'failed'}.") + except IOError: + print(f"Failed to open file {path}") + sys.exit(1) + + if passed: + print("All tests passed.") + else: + print("Problems have been found.") + sys.exit(1) + + +if __name__ == "__main__": + main() From 8be67cc246088d8b57108e71c8058dccd4ccb07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 02:53:16 +0200 Subject: [PATCH 02/96] Add TestHfNameStruct --- Scripts/find_code_issues.py | 79 ++++++++++++++++++++++++++----------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index b5a3aef4e09..22c99520465 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -343,9 +343,34 @@ def test_line(self, line: str) -> bool: return not "auto const" in line +class TestHfNameStruct(TestSpec): + name = "PWGHF: struct name" + message = "Names of PWGHF structs must start with \"Hf\"." + suffixes = [".h", ".cxx"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "PWGHF/" in path + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + if not line.startswith("struct "): + return True + # Extract struct name. + words = line.split() + if not words[1].isalnum(): # "struct : ...", "enum { ..." + return True + struct_name = words[1] + if words[1] == "class" and len(words) > 2: # enum class + struct_name = words[2] + # The actual test comes here. + return struct_name.startswith("Hf") + + class TestHfStructMembers(TestSpec): """Test order of struct members.""" - name = "PWGHF: struct memners" + name = "PWGHF: struct members" message = "Order struct members." suffixes = [".h", ".cxx"] per_line = False @@ -363,7 +388,7 @@ def test_file(self, path : str, content) -> bool: def main(): """Main function""" parser = argparse.ArgumentParser( - description="Run tests on files to find bad code." + description="Run tests on files to find code issues." ) parser.add_argument("paths", type=str, nargs="+", help="File path(s)") parser.add_argument( @@ -377,27 +402,37 @@ def main(): tests = [] # Bad code - # tests.append(TestIOStream()) - # tests.append(TestUsingDirectives()) - # tests.append(TestUsingStd()) - # tests.append(TestStdPrefix()) - # tests.append(TestROOT()) - # tests.append(TestPI()) - # tests.append(TestLogging()) + enable_bad_practice = True + enable_bad_practice = False + if enable_bad_practice: + tests.append(TestIOStream()) + tests.append(TestUsingDirectives()) + tests.append(TestUsingStd()) + tests.append(TestStdPrefix()) + tests.append(TestROOT()) + tests.append(TestPI()) + tests.append(TestLogging()) # Naming conventions - # tests.append(TestNameFunction()) - # tests.append(TestNameMacro()) - # tests.append(TestNameNamespace()) - tests.append(TestNameEnum()) - # tests.append(TestNameClass()) - # tests.append(TestNameStruct()) - tests.append(TestNameFileCpp()) - # tests.append(TestNameFilePython()) - - # PWG - # tests.append(TestHfConstAuto()) - # tests.append(TestHfStructMembers()) + enable_naming = True + # enable_naming = False + if enable_naming: + tests.append(TestNameFunction()) + tests.append(TestNameMacro()) + tests.append(TestNameNamespace()) + tests.append(TestNameEnum()) + tests.append(TestNameClass()) + tests.append(TestNameStruct()) + tests.append(TestNameFileCpp()) + tests.append(TestNameFilePython()) + + # PWGHF + enable_pwghf = True + enable_pwghf = False + if enable_pwghf: + tests.append(TestHfConstAuto()) + tests.append(TestHfStructMembers()) + tests.append(TestHfNameStruct()) test_names = [t.name for t in tests] suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests @@ -426,7 +461,7 @@ def main(): if passed: print("All tests passed.") else: - print("Problems have been found.") + print("Issues have been found.") sys.exit(1) From d7cf03f13b998630f2ac66a22728840b1221114d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 03:43:25 +0200 Subject: [PATCH 03/96] Add TestHfNameFileWorkflow --- Scripts/find_code_issues.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 22c99520465..33d1d7cae2b 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -331,6 +331,7 @@ def test_file(self, path : str, content) -> bool: # PWG-specific + class TestHfConstAuto(TestSpec): name = "PWGHF: const auto" message = "Use \"const auto\" instead of \"auto const\"" @@ -382,6 +383,39 @@ def test_file(self, path : str, content) -> bool: return True +class TestHfNameFileWorkflow(TestSpec): + """PWGHF: Test names of workflow files.""" + name = "PWGHF: workflow file names" + message = "Name of a workflow file must match the name of the main task in it. See the PWGHF O2 naming conventions for details." + suffixes = [".cxx"] + per_line = False + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path + + def test_file(self, path : str, content) -> bool: + file_name = os.path.basename(path).rstrip(".cxx") + if file_name[0].isupper(): + return False + base_struct_name = f"Hf{file_name[0].upper()}{file_name[1:]}" # expected base of struct names + # print(f"For file {file_name} expecting to find {base_struct_name}.") + struct_names = [] + for line in content: + if not line.startswith("struct "): + continue + # Extract struct name. + words = line.split() + if not words[1].isalnum(): # "struct : ..." + continue + struct_name = words[1] + struct_names.append(struct_name) + # print(f"Found structs: {struct_names}.") + for struct in struct_names: + if struct.startswith(base_struct_name): + return True + return False + + # End of test implementations @@ -415,7 +449,7 @@ def main(): # Naming conventions enable_naming = True - # enable_naming = False + enable_naming = False if enable_naming: tests.append(TestNameFunction()) tests.append(TestNameMacro()) @@ -433,6 +467,7 @@ def main(): tests.append(TestHfConstAuto()) tests.append(TestHfStructMembers()) tests.append(TestHfNameStruct()) + tests.append(TestHfNameFileWorkflow()) test_names = [t.name for t in tests] suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests From 93222ff753c8db59dc5c79fa6ea7541874cf67ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 05:00:45 +0200 Subject: [PATCH 04/96] Add TestNameWorkflow --- Scripts/find_code_issues.py | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 33d1d7cae2b..08850f9808e 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -329,6 +329,43 @@ def test_file(self, path : str, content) -> bool: return file_name.islower() and not "-" in file_name +class TestNameWorkflow(TestSpec): + """Test names of O2 workflows.""" + name = "O2 workflow names" + message = "Name of a workflow must consist of hyphenated lowercase words and must match the name of the workflow file." + suffixes = ["CMakeLists.txt"] + per_line = False + + def test_file(self, path : str, content) -> bool: + passed = True + workflow_name = "" + for i, line in enumerate(content): + if not line.startswith("o2physics_add_dpl_workflow("): + continue + # Extract workflow name. + workflow_name = line.strip().split("(")[1] + if not workflow_name.islower() or "_" in workflow_name: + passed = False + print(f"{path}:{i + 1}: Invalid workflow name: {workflow_name}.") + # if github: + # print(f"::error file={path},line={i + 1},title=[{self.name}]::{self.message}") + continue + # Extract workflow file name. + next_line = content[i + 1].strip() + words = next_line.split() + if words[0] != "SOURCES": + passed = False + print(f"{path}:{i + 2}: Did not find sources for workflow: {workflow_name}.") + continue + workflow_file_name = os.path.basename(words[1]) + expected_workflow_file_name = "".join([w.title() if w[0].isnumeric() else w.capitalize() for w in workflow_name.split("-")]) + ".cxx" + expected_workflow_file_name = f"{expected_workflow_file_name[0].lower()}{expected_workflow_file_name[1:]}" + if expected_workflow_file_name != workflow_file_name: + passed = False + print(f"{path}:{i + 1}: Workflow name {workflow_name} does not match the workflow file name {workflow_file_name} (expected {expected_workflow_file_name}).") + return passed + + # PWG-specific @@ -459,6 +496,7 @@ def main(): tests.append(TestNameStruct()) tests.append(TestNameFileCpp()) tests.append(TestNameFilePython()) + tests.append(TestNameWorkflow()) # PWGHF enable_pwghf = True From 33c9cf0c79b3373db04a964fddea8739b03b7220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 06:32:50 +0200 Subject: [PATCH 05/96] Implement TestHfStructMembers --- Scripts/find_code_issues.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 08850f9808e..69c80ad2468 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -410,14 +410,47 @@ class TestHfStructMembers(TestSpec): """Test order of struct members.""" name = "PWGHF: struct members" message = "Order struct members." - suffixes = [".h", ".cxx"] + suffixes = [".cxx"] per_line = False + member_order = ["Spawns<", "Builds<", "Produces<", "Configurable<", "HfHelper hfHelper", "Service<", "using ", "Filter ", "Preslice<", "Partition<", "ConfigurableAxis ", "AxisSpec ", "HistogramRegistry ", "OutputObj<", "void init(", "void process"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path def test_file(self, path : str, content) -> bool: - return True + passed = True + dic_struct = {} + struct_name = "" + for i, line in enumerate(content): + if line.startswith("//"): + continue + if line.startswith("struct "): + struct_name = line.strip().split()[1] + dic_struct[struct_name] = {} + continue + if not struct_name: + continue + for member in self.member_order: + if line.startswith(f" {member}"): + if member not in dic_struct[struct_name]: + dic_struct[struct_name][member] = [] + dic_struct[struct_name][member].append(i + 1) + break + # print(dic_struct) + last_line_last_member = 0 # number of the last line of the previous member category + index_last_member = 0 # index of the previous member category in member_order + for struct_name in dic_struct: + for i_m, member in enumerate(self.member_order): + if member not in dic_struct[struct_name]: + continue + first_line = min(dic_struct[struct_name][member]) + last_line = max(dic_struct[struct_name][member]) + if first_line < last_line_last_member: + passed = False + print(f"{path}:{first_line}: {struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).") + last_line_last_member = last_line + index_last_member = i_m + return passed class TestHfNameFileWorkflow(TestSpec): From 050346c44510e8550d577799fb8d8915aefae477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 13:20:21 +0200 Subject: [PATCH 06/96] Add functions for cases. Add TestHfNameConfigurable --- Scripts/find_code_issues.py | 108 +++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 32 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 69c80ad2468..e3601065081 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -23,6 +23,36 @@ import sys +def is_camel_case(name: str) -> bool: + """forExample or ForExample""" + return not "_" in name and not "-" in name + + +def is_upper_camel_case(name: str) -> bool: + """ForExample""" + return name[0].isupper() and is_camel_case(name) + + +def is_lower_camel_case(name: str) -> bool: + """forExample""" + return name[0].islower() and is_camel_case(name) + + +def is_kebab_case(name: str) -> bool: + """for-example""" + return name.islower() and not "_" in name + + +def is_snake_case(name: str) -> bool: + """for_example""" + return name.islower() and not "-" in name + + +def is_screaming_snake_case(name: str) -> bool: + """FOR_EXAMPLE""" + return name.isupper() and not "-" in name + + class TestSpec(ABC): """Prototype of the test class""" name = "Test template" # name of the test @@ -178,7 +208,7 @@ class TestNameFunction(TestSpec): Can accidentally spot names of variable too but it is fine because same conventions apply. """ name = "function names" - message = "Use camelCase for names of functions and variables." + message = "Use lowerCamelCase for names of functions and variables." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -206,17 +236,13 @@ def test_line(self, line: str) -> bool: if "::" in function_name: function_name = function_name.split("::")[1] # The actual test comes here. - # if not (function_name[0].islower() and not "_" in function_name and not "-" in function_name): - # print(line, words) - # return False - # return True - return function_name[0].islower() and not "_" in function_name and not "-" in function_name + return is_lower_camel_case(function_name) class TestNameMacro(TestSpec): """Test macro names""" name = "macro names" - message = "Use capital letters and underscores in macro names." + message = "Use SCREAMING_SNAKE_CASE for macro names." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -230,13 +256,13 @@ def test_line(self, line: str) -> bool: if "(" in macro_name: macro_name = macro_name.split("(")[0] # The actual test comes here. - return macro_name.isupper() and not "-" in macro_name + return is_screaming_snake_case(macro_name) class TestNameNamespace(TestSpec): """Test names of namespaces.""" name = "namespace names" - message = "Use lowercase letters and underscores in names of namespaces." + message = "Use snake_case for names of namespaces." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -247,19 +273,15 @@ def test_line(self, line: str) -> bool: return True # Extract namespace name. namespace_name = line.split()[1] - # if not namespace_name.islower(): - # print(namespace_name) - # return False - # return True # The actual test comes here. - return namespace_name.islower() and not "-" in namespace_name + return is_snake_case(namespace_name) class TestNameUpperCamelCase(TestSpec): - """Test CamelCase of a name.""" + """Test UpperCamelCase of a name.""" keyword = "key" - name = f"{keyword} CamelCase" - message = f"Use CamelCase for names of {keyword}." + name = f"{keyword} UpperCamelCase" + message = f"Use UpperCamelCase for names of {keyword}." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -275,12 +297,8 @@ def test_line(self, line: str) -> bool: object_name = words[1] if words[1] == "class" and len(words) > 2: # enum class object_name = words[2] - # if not object_name[0].isupper(): - # print(object_name) - # return False - # return True # The actual test comes here. - return object_name[0].isupper() and not "_" in object_name and not "-" in object_name + return is_upper_camel_case(object_name) class TestNameEnum(TestNameUpperCamelCase): @@ -288,51 +306,51 @@ class TestNameEnum(TestNameUpperCamelCase): Issue with 'enum class'""" keyword = "enum" name = f"{keyword} names" - message = f"Use CamelCase for names of enumerators and their values." + message = f"Use UpperCamelCase for names of enumerators and their values." class TestNameClass(TestNameUpperCamelCase): """Test names of classes.""" keyword = "class" name = f"{keyword} names" - message = f"Use CamelCase for names of classes." + message = f"Use UpperCamelCase for names of classes." class TestNameStruct(TestNameUpperCamelCase): """Test names of structs.""" keyword = "struct" name = f"{keyword} names" - message = f"Use CamelCase for names of structs." + message = f"Use UpperCamelCase for names of structs." class TestNameFileCpp(TestSpec): """Test names of C++ files.""" name = "C++ file names" - message = "Use camelCase or CamelCase for names of C++ files. See the O2 naming conventions for details." + message = "Use lowerCamelCase or UpperCamelCase for names of C++ files. See the O2 naming conventions for details." suffixes = [".h", ".cxx", ".C"] per_line = False def test_file(self, path : str, content) -> bool: file_name = os.path.basename(path) - return not "_" in file_name and not "-" in file_name + return is_camel_case(file_name) class TestNameFilePython(TestSpec): """Test names of Python files.""" name = "Python file names" - message = "Use lowercase letters and underscores in names of Python files." + message = "Use snake_case for names of Python files." suffixes = [".py", ".ipynb"] per_line = False def test_file(self, path : str, content) -> bool: file_name = os.path.basename(path) - return file_name.islower() and not "-" in file_name + return is_snake_case(file_name) class TestNameWorkflow(TestSpec): """Test names of O2 workflows.""" name = "O2 workflow names" - message = "Name of a workflow must consist of hyphenated lowercase words and must match the name of the workflow file." + message = "Use kebab-case for names of workflows and match the name of the workflow file." suffixes = ["CMakeLists.txt"] per_line = False @@ -344,7 +362,7 @@ def test_file(self, path : str, content) -> bool: continue # Extract workflow name. workflow_name = line.strip().split("(")[1] - if not workflow_name.islower() or "_" in workflow_name: + if not is_kebab_case(workflow_name): passed = False print(f"{path}:{i + 1}: Invalid workflow name: {workflow_name}.") # if github: @@ -412,7 +430,7 @@ class TestHfStructMembers(TestSpec): message = "Order struct members." suffixes = [".cxx"] per_line = False - member_order = ["Spawns<", "Builds<", "Produces<", "Configurable<", "HfHelper hfHelper", "Service<", "using ", "Filter ", "Preslice<", "Partition<", "ConfigurableAxis ", "AxisSpec ", "HistogramRegistry ", "OutputObj<", "void init(", "void process"] + member_order = ["Spawns<", "Builds<", "Produces<", "Configurable<", "HfHelper ", "SliceCache ", "Service<", "using ", "Filter ", "Preslice<", "Partition<", "ConfigurableAxis ", "AxisSpec ", "HistogramRegistry ", "OutputObj<", "void init(", "void process"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path @@ -486,6 +504,31 @@ def test_file(self, path : str, content) -> bool: return False +class TestHfNameConfigurable(TestSpec): + """PWGHF: Test names of configurables.""" + name = "PWGHF: Configurable names" + message = "Use camelCase for Configurable names and use the same name for the struct member as for the JSON string." + suffixes = [".h", ".cxx"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + if not line.startswith("Configurable"): + return True + # Extract Configurable name. + names = line.split()[1].split("{") # nameCpp{"nameJson", + name_cpp = names[0] + name_json = names[1] + if name_json[0] != "\"": # JSON name is not a literal string. + return True + name_json = name_json.strip("\",") + return is_lower_camel_case(name_cpp) and name_cpp == name_json + + # End of test implementations @@ -539,6 +582,7 @@ def main(): tests.append(TestHfStructMembers()) tests.append(TestHfNameStruct()) tests.append(TestHfNameFileWorkflow()) + tests.append(TestHfNameConfigurable()) test_names = [t.name for t in tests] suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests From 5f9ef763a06526ba5b8b8cae53adb05a617f7247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 16:01:46 +0200 Subject: [PATCH 07/96] Add TestNameConstant --- Scripts/find_code_issues.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index e3601065081..1bc6f22ad48 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -200,6 +200,7 @@ def test_line(self, line: str) -> bool: # Naming conventions +# Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/naming_formatting.html class TestNameFunction(TestSpec): @@ -240,7 +241,7 @@ def test_line(self, line: str) -> bool: class TestNameMacro(TestSpec): - """Test macro names""" + """Test macro names.""" name = "macro names" message = "Use SCREAMING_SNAKE_CASE for macro names." suffixes = [".h", ".cxx", ".C"] @@ -259,6 +260,31 @@ def test_line(self, line: str) -> bool: return is_screaming_snake_case(macro_name) +class TestNameConstant(TestSpec): + """Test constexpr constant names.""" + name = "constexpr constant names" + message = "Use UpperCamelCase for constexpr constant names. Names of special constants may be prefixed with \"k\"." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + line = line.strip() + if line.startswith("//"): + return True + words = line.split() + if not "constexpr" in words or not "=" in words: + return True + # Extract constant name. + words = words[:words.index("=")] # keep only words before "=" + constant_name = words[-1] # last word before "=" + if constant_name.endswith("]") and "[" not in constant_name: # it's an array and we do not have the name before "[" here + opens_brackets = ["[" in w for w in words] + constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" + # The actual test comes here. + if constant_name.startswith("k"): # exception for special constants + constant_name = constant_name[1:] + return is_upper_camel_case(constant_name) + + class TestNameNamespace(TestSpec): """Test names of namespaces.""" name = "namespace names" @@ -566,6 +592,7 @@ def main(): if enable_naming: tests.append(TestNameFunction()) tests.append(TestNameMacro()) + tests.append(TestNameConstant()) tests.append(TestNameNamespace()) tests.append(TestNameEnum()) tests.append(TestNameClass()) From 68bfd59f4fc39be30206aca038a3b9e9570f46ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 20:04:47 +0200 Subject: [PATCH 08/96] Extensions and documentation improvements. Extend TestStdPrefix with regex. Extend TestPI. Extend TestHfNameStruct -> TestHfNameStructClass. Improve documentation. Remove unnecessary strip. --- Scripts/find_code_issues.py | 158 ++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 69 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 1bc6f22ad48..2adf8127683 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -20,6 +20,7 @@ from abc import ABC #, abstractmethod import argparse import os +import re import sys @@ -54,14 +55,14 @@ def is_screaming_snake_case(name: str) -> bool: class TestSpec(ABC): - """Prototype of the test class""" - name = "Test template" # name of the test + """Prototype of a test class""" + name = "test template" # short name of the test message = "Test failed" # error message suffixes = [] # suffixes of files to test - per_line = True # test lines separately one by one + per_line = True # Test lines separately one by one. def file_matches(self, path: str) -> bool: - """Test whether the path matches the pattern for files to test""" + """Test whether the path matches the pattern for files to test.""" return path.endswith(tuple(self.suffixes)) if self.suffixes else True # @abstractmethod @@ -75,7 +76,7 @@ def test_file(self, path : str, content) -> bool: raise NotImplementedError() def run(self, path : str, content, github: bool = False) -> bool: - """Test file content.""" + """Run the test.""" # print(content) passed = True if not self.file_matches(path): @@ -106,7 +107,8 @@ def run(self, path : str, content, github: bool = False) -> bool: # Bad code class TestIOStream(TestSpec): - name = "Include iostream" + """Detect included iostream.""" + name = "include iostream" message = "Including iostream is not allowed. Use O2 logging instead." suffixes = [".h", ".cxx"] @@ -117,8 +119,9 @@ def test_line(self, line: str) -> bool: class TestUsingStd(TestSpec): - name = "Importing std names" - message = "Importing std names is not allowed." + """Detect importing names from the std namespace.""" + name = "import std names" + message = "Importing names from the std namespace is not allowed." suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: @@ -128,7 +131,8 @@ def test_line(self, line: str) -> bool: class TestUsingDirectives(TestSpec): - name = "Using directives" + """Detect using directives in headers.""" + name = "using directives" message = "Using directives are not allowed in headers." suffixes = [".h"] @@ -139,18 +143,27 @@ def test_line(self, line: str) -> bool: class TestStdPrefix(TestSpec): + """Detect missing std:: prefix for common names from the std namespace.""" name = "std prefix" - message = "Use std:: prefix for STD methods." + message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] + patterns = ["[^\w:.]vector<", "[^\w:.]array[<\{\()]", "[^\w:.]f?abs\(", "[^\w:.]min\(", "[^\w:.]max\(", "[^\w:.]log\(", "[^\w:.]exp\(", "[^\w:.]sin\(", "[^\w:.]cos\(", "[^\w:.]tan\(", "[^\w:.]atan\(", "[^\w:.]atan2\("] def test_line(self, line: str) -> bool: if line.startswith("//"): return True - return not " abs(" in line + for pattern in self.patterns: + if re.search(pattern, line): + return False + # occurrences = re.findall(pattern, line) + # if occurrences: + # print(occurrences) + # return False + return True class TestROOT(TestSpec): - """Test use of ROOT entities""" + """Detect use of unnecessary ROOT entities.""" name = "ROOT entities" message = "Consider replacing ROOT entities with STD C++ or O2 entities." suffixes = [".h", ".cxx"] @@ -169,21 +182,28 @@ def test_line(self, line: str) -> bool: class TestPI(TestSpec): - """Test use of PI""" - name = "Use of PI" + """Detect use of external PI.""" + name = "external PI" message = "Consider using the PI constant (and its multiples) defined in o2::constants::math." - suffixes = [".h", ".cxx", ".C"] + suffixes = [".h", ".cxx"] + keywords = ["M_PI", "TMath::Pi", "TMath::TwoPi"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: if line.startswith("//"): return True - return not "M_PI" in line + for k in self.keywords: + if k in line: + return False + return True class TestLogging(TestSpec): - """Test logging""" - name = "Use of logging" - message = "Consider using O2 logging with LOG, LOGF, LOGP." + """Detect non-O2 logging.""" + name = "logging" + message = "Consider using O2 logging (LOG, LOGF, LOGP)." suffixes = [".h", ".cxx"] keywords = ["Printf(", "printf(", "cout <", "cin >"] @@ -205,15 +225,14 @@ def test_line(self, line: str) -> bool: class TestNameFunction(TestSpec): """Test function names. - Likely to report false positives. - Can accidentally spot names of variable too but it is fine because same conventions apply. + Might report false positives. + Can accidentally spot names of variables too but it is fine because same conventions apply. """ name = "function names" message = "Use lowerCamelCase for names of functions and variables." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True # Look for "Type Name(..." @@ -247,7 +266,6 @@ class TestNameMacro(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True if not line.startswith("#define "): @@ -267,7 +285,6 @@ class TestNameConstant(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True words = line.split() @@ -281,7 +298,7 @@ def test_line(self, line: str) -> bool: constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" # The actual test comes here. if constant_name.startswith("k"): # exception for special constants - constant_name = constant_name[1:] + constant_name = constant_name[1:] # test the name without "k" return is_upper_camel_case(constant_name) @@ -292,7 +309,6 @@ class TestNameNamespace(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True if not line.startswith("namespace "): @@ -304,14 +320,13 @@ def test_line(self, line: str) -> bool: class TestNameUpperCamelCase(TestSpec): - """Test UpperCamelCase of a name.""" + """Base class for a test of UpperCamelCase names.""" keyword = "key" name = f"{keyword} UpperCamelCase" message = f"Use UpperCamelCase for names of {keyword}." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True if not line.startswith(f"{self.keyword} "): @@ -321,15 +336,14 @@ def test_line(self, line: str) -> bool: if not words[1].isalnum(): # "struct : ...", "enum { ..." return True object_name = words[1] - if words[1] == "class" and len(words) > 2: # enum class + if object_name == "class" and len(words) > 2: # enum class ... object_name = words[2] # The actual test comes here. return is_upper_camel_case(object_name) class TestNameEnum(TestNameUpperCamelCase): - """Test names of enumerators. - Issue with 'enum class'""" + """Test names of enumerators.""" keyword = "enum" name = f"{keyword} names" message = f"Use UpperCamelCase for names of enumerators and their values." @@ -401,9 +415,11 @@ def test_file(self, path : str, content) -> bool: passed = False print(f"{path}:{i + 2}: Did not find sources for workflow: {workflow_name}.") continue - workflow_file_name = os.path.basename(words[1]) + workflow_file_name = os.path.basename(words[1]) # the actual file name + # Generate the file name matching the workflow name. expected_workflow_file_name = "".join([w.title() if w[0].isnumeric() else w.capitalize() for w in workflow_name.split("-")]) + ".cxx" - expected_workflow_file_name = f"{expected_workflow_file_name[0].lower()}{expected_workflow_file_name[1:]}" + expected_workflow_file_name = f"{expected_workflow_file_name[0].lower()}{expected_workflow_file_name[1:]}" # start with lowercase letter + # Compare the actual and expected file names. if expected_workflow_file_name != workflow_file_name: passed = False print(f"{path}:{i + 1}: Workflow name {workflow_name} does not match the workflow file name {workflow_file_name} (expected {expected_workflow_file_name}).") @@ -414,46 +430,48 @@ def test_file(self, path : str, content) -> bool: class TestHfConstAuto(TestSpec): + """PWGHF: Detect swapped const auto.""" name = "PWGHF: const auto" - message = "Use \"const auto\" instead of \"auto const\"" + message = "Use \"const auto\" instead of \"auto const\"." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True return not "auto const" in line -class TestHfNameStruct(TestSpec): - name = "PWGHF: struct name" - message = "Names of PWGHF structs must start with \"Hf\"." +class TestHfNameStructClass(TestSpec): + """PWGHF: Test names of structs and classes.""" + name = "PWGHF: struct/class names" + message = "Names of PWGHF structs and classes must start with \"Hf\"." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True - if not line.startswith("struct "): + if not line.startswith(("struct ", "class ")): return True - # Extract struct name. + # Extract struct/class name. words = line.split() - if not words[1].isalnum(): # "struct : ...", "enum { ..." + if not words[1].isalnum(): # "struct : ..." return True struct_name = words[1] - if words[1] == "class" and len(words) > 2: # enum class - struct_name = words[2] # The actual test comes here. return struct_name.startswith("Hf") class TestHfStructMembers(TestSpec): - """Test order of struct members.""" - name = "PWGHF: struct members" - message = "Order struct members." + """PWGHF: Test order of struct members. + Caveat: Does not see Configurables in ConfigurableGroup.""" + name = "PWGHF: struct member order" + message = "Declare struct members in the conventional order. See the PWGHF coding guidelines." suffixes = [".cxx"] per_line = False member_order = ["Spawns<", "Builds<", "Produces<", "Configurable<", "HfHelper ", "SliceCache ", "Service<", "using ", "Filter ", "Preslice<", "Partition<", "ConfigurableAxis ", "AxisSpec ", "HistogramRegistry ", "OutputObj<", "void init(", "void process"] @@ -466,30 +484,32 @@ def test_file(self, path : str, content) -> bool: dic_struct = {} struct_name = "" for i, line in enumerate(content): - if line.startswith("//"): + if line.strip().startswith("//"): continue - if line.startswith("struct "): + if line.startswith("struct "): # expecting no indentation struct_name = line.strip().split()[1] dic_struct[struct_name] = {} continue if not struct_name: continue + # Save line numbers of members of the current struct for each category. for member in self.member_order: - if line.startswith(f" {member}"): + if line.startswith(f" {member}"): # expecting single-level indentation for direct members if member not in dic_struct[struct_name]: dic_struct[struct_name][member] = [] - dic_struct[struct_name][member].append(i + 1) + dic_struct[struct_name][member].append(i + 1) # save line number break # print(dic_struct) - last_line_last_member = 0 # number of the last line of the previous member category - index_last_member = 0 # index of the previous member category in member_order + # Detect members declared in a wrong order. + last_line_last_member = 0 # line number of the last member of the previous member category + index_last_member = 0 # index of the previous member category in the member_order list for struct_name in dic_struct: for i_m, member in enumerate(self.member_order): if member not in dic_struct[struct_name]: continue - first_line = min(dic_struct[struct_name][member]) - last_line = max(dic_struct[struct_name][member]) - if first_line < last_line_last_member: + first_line = min(dic_struct[struct_name][member]) # line number of the first member of this category + last_line = max(dic_struct[struct_name][member]) # line number of the last member of this category + if first_line < last_line_last_member: # The current category starts before the end of the previous category. passed = False print(f"{path}:{first_line}: {struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).") last_line_last_member = last_line @@ -513,7 +533,7 @@ def test_file(self, path : str, content) -> bool: return False base_struct_name = f"Hf{file_name[0].upper()}{file_name[1:]}" # expected base of struct names # print(f"For file {file_name} expecting to find {base_struct_name}.") - struct_names = [] + struct_names = [] # actual struct names in the file for line in content: if not line.startswith("struct "): continue @@ -524,8 +544,8 @@ def test_file(self, path : str, content) -> bool: struct_name = words[1] struct_names.append(struct_name) # print(f"Found structs: {struct_names}.") - for struct in struct_names: - if struct.startswith(base_struct_name): + for struct_name in struct_names: + if struct_name.startswith(base_struct_name): return True return False @@ -540,18 +560,18 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path def test_line(self, line: str) -> bool: - line = line.strip() if line.startswith("//"): return True if not line.startswith("Configurable"): return True # Extract Configurable name. - names = line.split()[1].split("{") # nameCpp{"nameJson", - name_cpp = names[0] - name_json = names[1] + names = line.split()[1].split("{") # expecting nameCpp{"nameJson", + name_cpp = names[0] # nameCpp + name_json = names[1] # expecting "nameJson", if name_json[0] != "\"": # JSON name is not a literal string. return True - name_json = name_json.strip("\",") + name_json = name_json.strip("\",") # expecting nameJson + # The actual test comes here. return is_lower_camel_case(name_cpp) and name_cpp == name_json @@ -568,11 +588,11 @@ def main(): "-g", dest="github", action="store_true", - help="Print messages as GitHub annotations", + help="Print messages also as GitHub annotations", ) args = parser.parse_args() - tests = [] + tests = [] # list of activated tests # Bad code enable_bad_practice = True @@ -607,13 +627,13 @@ def main(): if enable_pwghf: tests.append(TestHfConstAuto()) tests.append(TestHfStructMembers()) - tests.append(TestHfNameStruct()) + tests.append(TestHfNameStructClass()) tests.append(TestHfNameFileWorkflow()) tests.append(TestHfNameConfigurable()) - test_names = [t.name for t in tests] + test_names = [t.name for t in tests] # short names of activated tests suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests - passed = True # global status + passed = True # global result of all tests print("Enabled tests:", test_names) @@ -632,7 +652,7 @@ def main(): # print(f"File {path} failed the test {test.name}.") # print(f"Test {'passed' if passed else 'failed'}.") except IOError: - print(f"Failed to open file {path}") + print(f"Failed to open file {path}.") sys.exit(1) if passed: From 40a3e3dd7cc4664731e24e78dfeb74139556f29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 20:16:10 +0200 Subject: [PATCH 09/96] Fix activation order --- Scripts/find_code_issues.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 2adf8127683..457d47f3b66 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -104,7 +104,7 @@ def run(self, path : str, content, github: bool = False) -> bool: # Implementations of tests ########################## -# Bad code +# Bad practice class TestIOStream(TestSpec): """Detect included iostream.""" @@ -594,13 +594,12 @@ def main(): tests = [] # list of activated tests - # Bad code + # Bad practice enable_bad_practice = True - enable_bad_practice = False if enable_bad_practice: tests.append(TestIOStream()) - tests.append(TestUsingDirectives()) tests.append(TestUsingStd()) + tests.append(TestUsingDirectives()) tests.append(TestStdPrefix()) tests.append(TestROOT()) tests.append(TestPI()) @@ -608,7 +607,6 @@ def main(): # Naming conventions enable_naming = True - enable_naming = False if enable_naming: tests.append(TestNameFunction()) tests.append(TestNameMacro()) @@ -623,11 +621,10 @@ def main(): # PWGHF enable_pwghf = True - enable_pwghf = False if enable_pwghf: tests.append(TestHfConstAuto()) - tests.append(TestHfStructMembers()) tests.append(TestHfNameStructClass()) + tests.append(TestHfStructMembers()) tests.append(TestHfNameFileWorkflow()) tests.append(TestHfNameConfigurable()) @@ -635,9 +632,12 @@ def main(): suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests passed = True # global result of all tests + # Report overview before running. + print(f"Testing {len(args.paths)} files.") print("Enabled tests:", test_names) + print("Suffixes of tested files:", suffixes) - print(f"Testing {len(args.paths)} files.") + # Test files. for path in args.paths: # Skip not tested files. if not path.endswith(suffixes): @@ -655,6 +655,7 @@ def main(): print(f"Failed to open file {path}.") sys.exit(1) + # Report result. if passed: print("All tests passed.") else: From 3e5031320c31afe231033325cd5b862422366baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 20:35:54 +0200 Subject: [PATCH 10/96] Fix TestHfNameConfigurable --- Scripts/find_code_issues.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 457d47f3b66..3bdef3ff931 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -565,9 +565,14 @@ def test_line(self, line: str) -> bool: if not line.startswith("Configurable"): return True # Extract Configurable name. - names = line.split()[1].split("{") # expecting nameCpp{"nameJson", - name_cpp = names[0] # nameCpp - name_json = names[1] # expecting "nameJson", + words = line.split() + if words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", + name_cpp = words[1] # nameCpp + name_json = words[3][1:] # expecting "nameJson", + else: + names = words[1].split("{") # expecting Configurable... nameCpp{"nameJson", + name_cpp = names[0] # nameCpp + name_json = names[1] # expecting "nameJson", if name_json[0] != "\"": # JSON name is not a literal string. return True name_json = name_json.strip("\",") # expecting nameJson From a13c004717ab8485e4b80898b25702cb068fff5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 19 Jul 2024 21:19:30 +0200 Subject: [PATCH 11/96] Add TestConstRefInForLoop --- Scripts/find_code_issues.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 3bdef3ff931..ebffa57c929 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -219,6 +219,21 @@ def test_line(self, line: str) -> bool: return True +class TestConstRefInForLoop(TestSpec): + """Test const refs in range-based for loops.""" + name = "const ref in for loop" + message = "Use constant references for non-modified iterators in range-based for loops." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + if line.startswith("//"): + return True + if not line.startswith("for (") or " : " not in line: + return True + line = line[:line.index(" : ")] # keep only the iterator part + return True if re.search("\([\w]* ?const ?[\w]*&", line) else False + + # Naming conventions # Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/naming_formatting.html @@ -609,6 +624,7 @@ def main(): tests.append(TestROOT()) tests.append(TestPI()) tests.append(TestLogging()) + tests.append(TestConstRefInForLoop()) # Naming conventions enable_naming = True From 3c29ae2a6dc982eaddc0d67451b9d436713f1c48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 20 Jul 2024 01:47:28 +0200 Subject: [PATCH 12/96] Add TestConstRefInSubscription --- Scripts/find_code_issues.py | 49 +++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index ebffa57c929..fc8149dc1c3 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -234,6 +234,54 @@ def test_line(self, line: str) -> bool: return True if re.search("\([\w]* ?const ?[\w]*&", line) else False +class TestConstRefInSubscription(TestSpec): + """Test const refs in process function subscriptions.""" + name = "const ref in process" + message = "Use constant references for table subscriptions in process functions." + suffixes = [".cxx"] + # suffixes = [".h"] + per_line = False + + def test_file(self, path : str, content) -> bool: + passed = True + n_parens_opened = 0 # number of opened parentheses + arguments = "" # process function arguments + line_process = 0 + for i, line in enumerate(content): + line = line.strip() + if line.startswith("//"): + continue + if re.search("^void process[\w]*\(", line): + line_process = (i + 1) + i_closing = line.rfind(")") + i_start = line.find("(") + 1 + i_end = i_closing if i_closing != -1 else len(line) + arguments = line[i_start:i_end] # get arguments between parentheses + n_parens_opened = line.count("(") - line.count(")") + elif n_parens_opened > 0: + i_closing = line.rfind(")") + i_start = 0 + i_end = i_closing if i_closing != -1 else len(line) + arguments += " " + line[i_start:i_end] # get arguments between parentheses + n_parens_opened += line.count("(") - line.count(")") + if line_process > 0 and n_parens_opened == 0: + # process arguments + # sanitise template arguments + template_args = re.findall("<[\w:, ]*>", arguments) + if template_args: + for arg in template_args: + if ", " in arg: + arguments = arguments.replace(arg, arg.replace(", ", ":")) + words = arguments.split(", ") + # test + for arg in words: + if not re.search("[\w<>:]* ?const ?[\w<>:]*&", arg): + passed = False + print(f"{path}:{i + 1}: Argument {arg} is not const&.") + line_process = 0 + return passed + + # Naming conventions # Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/naming_formatting.html @@ -625,6 +673,7 @@ def main(): tests.append(TestPI()) tests.append(TestLogging()) tests.append(TestConstRefInForLoop()) + tests.append(TestConstRefInSubscription()) # Naming conventions enable_naming = True From 65062c2026f632b0b5bc49b48ef691269a631019 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 20 Jul 2024 02:31:52 +0200 Subject: [PATCH 13/96] Unify error printing --- Scripts/find_code_issues.py | 38 ++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index fc8149dc1c3..fd3bd99f2ca 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -24,6 +24,8 @@ import sys +github_mode = False # GitHub mode + def is_camel_case(name: str) -> bool: """forExample or ForExample""" return not "_" in name and not "-" in name @@ -54,6 +56,15 @@ def is_screaming_snake_case(name: str) -> bool: return name.isupper() and not "-" in name +def print_error(path: str, line: int, title: str, message: str) -> str: + """Format and print error message.""" + str_line = "" if line is None else f"{line}:" + print(f"{path}:{str_line} {message} [{title}]") # terminal format + if github_mode: + str_line = "" if line is None else f",line={line}" + print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub action format + + class TestSpec(ABC): """Prototype of a test class""" name = "test template" # short name of the test @@ -75,7 +86,7 @@ def test_file(self, path : str, content) -> bool: """Test a file in a way that cannot be done line by line.""" raise NotImplementedError() - def run(self, path : str, content, github: bool = False) -> bool: + def run(self, path : str, content) -> bool: """Run the test.""" # print(content) passed = True @@ -88,15 +99,11 @@ def run(self, path : str, content, github: bool = False) -> bool: # print(i + 1, line) if not self.test_line(line): passed = False - print(f"{path}:{i + 1}: {self.message} [{self.name}]") - if github: - print(f"::error file={path},line={i + 1},title=[{self.name}]::{self.message}") + print_error(path, i + 1, self.name, self.message) else: passed = self.test_file(path, content) if not passed: - print(f"{path}: {self.message} [{self.name}]") - if github: - print(f"::error file={path},title=[{self.name}]::{self.message}") + print_error(path, None, self.name, self.message) return passed @@ -277,7 +284,7 @@ def test_file(self, path : str, content) -> bool: for arg in words: if not re.search("[\w<>:]* ?const ?[\w<>:]*&", arg): passed = False - print(f"{path}:{i + 1}: Argument {arg} is not const&.") + print_error(path, i + 1, self.name, f"Argument {arg} is not const&.") line_process = 0 return passed @@ -467,16 +474,14 @@ def test_file(self, path : str, content) -> bool: workflow_name = line.strip().split("(")[1] if not is_kebab_case(workflow_name): passed = False - print(f"{path}:{i + 1}: Invalid workflow name: {workflow_name}.") - # if github: - # print(f"::error file={path},line={i + 1},title=[{self.name}]::{self.message}") + print_error(path, i + 1, self.name, f"Invalid workflow name: {workflow_name}.") continue # Extract workflow file name. next_line = content[i + 1].strip() words = next_line.split() if words[0] != "SOURCES": passed = False - print(f"{path}:{i + 2}: Did not find sources for workflow: {workflow_name}.") + print_error(path, i + 2, self.name, f"Did not find sources for workflow: {workflow_name}.") continue workflow_file_name = os.path.basename(words[1]) # the actual file name # Generate the file name matching the workflow name. @@ -485,7 +490,7 @@ def test_file(self, path : str, content) -> bool: # Compare the actual and expected file names. if expected_workflow_file_name != workflow_file_name: passed = False - print(f"{path}:{i + 1}: Workflow name {workflow_name} does not match the workflow file name {workflow_file_name} (expected {expected_workflow_file_name}).") + print_error(path, i + 1, self.name, f"Workflow name {workflow_name} does not match the workflow file name {workflow_file_name} (expected {expected_workflow_file_name}).") return passed @@ -574,7 +579,7 @@ def test_file(self, path : str, content) -> bool: last_line = max(dic_struct[struct_name][member]) # line number of the last member of this category if first_line < last_line_last_member: # The current category starts before the end of the previous category. passed = False - print(f"{path}:{first_line}: {struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).") + print_error(path, first_line, self.name, f"{struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).") last_line_last_member = last_line index_last_member = i_m return passed @@ -659,6 +664,9 @@ def main(): help="Print messages also as GitHub annotations", ) args = parser.parse_args() + if args.github: + global github_mode + github_mode = True tests = [] # list of activated tests @@ -716,7 +724,7 @@ def main(): with open(path, "r") as file: content = file.readlines() for test in tests: - if not test.run(path, content, args.github): + if not test.run(path, content): passed = False # if not passed: # print(f"File {path} failed the test {test.name}.") From b3e1fb5b83fa185dc548fade2581a568f3ce12bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 19:32:59 +0200 Subject: [PATCH 14/96] Add is_comment_cpp. Skip empty lines. Improve TestNameFunction. --- Scripts/find_code_issues.py | 145 ++++++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 46 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index fd3bd99f2ca..b2f4c318b8b 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -58,6 +58,7 @@ def is_screaming_snake_case(name: str) -> bool: def print_error(path: str, line: int, title: str, message: str) -> str: """Format and print error message.""" + # return # Use to suppress error message when counting speed. str_line = "" if line is None else f"{line}:" print(f"{path}:{str_line} {message} [{title}]") # terminal format if github_mode: @@ -65,6 +66,11 @@ def print_error(path: str, line: int, title: str, message: str) -> str: print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub action format +def is_comment_cpp(line: str) -> bool: + """Test whether a line is a C++ comment.""" + return line.startswith(("//", "/*")) + + class TestSpec(ABC): """Prototype of a test class""" name = "test template" # short name of the test @@ -96,6 +102,8 @@ def run(self, path : str, content) -> bool: if self.per_line: for i, line in enumerate(content): line = line.strip() + if not line: + continue # print(i + 1, line) if not self.test_line(line): passed = False @@ -120,7 +128,7 @@ class TestIOStream(TestSpec): suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True return not line.strip().startswith("#include ") @@ -132,7 +140,7 @@ class TestUsingStd(TestSpec): suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True return not line.strip().startswith("using std::") @@ -144,7 +152,7 @@ class TestUsingDirectives(TestSpec): suffixes = [".h"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True return not line.strip().startswith("using namespace") @@ -155,9 +163,10 @@ class TestStdPrefix(TestSpec): message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] patterns = ["[^\w:.]vector<", "[^\w:.]array[<\{\()]", "[^\w:.]f?abs\(", "[^\w:.]min\(", "[^\w:.]max\(", "[^\w:.]log\(", "[^\w:.]exp\(", "[^\w:.]sin\(", "[^\w:.]cos\(", "[^\w:.]tan\(", "[^\w:.]atan\(", "[^\w:.]atan2\("] + # Add sqrt, pow def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True for pattern in self.patterns: if re.search(pattern, line): @@ -180,7 +189,7 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True for k in self.keywords: if k in line: @@ -199,7 +208,7 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True for k in self.keywords: if k in line: @@ -218,7 +227,7 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True for k in self.keywords: if k in line: @@ -233,7 +242,7 @@ class TestConstRefInForLoop(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True if not line.startswith("for (") or " : " not in line: return True @@ -256,7 +265,7 @@ def test_file(self, path : str, content) -> bool: line_process = 0 for i, line in enumerate(content): line = line.strip() - if line.startswith("//"): + if is_comment_cpp(line): continue if re.search("^void process[\w]*\(", line): line_process = (i + 1) @@ -294,39 +303,83 @@ def test_file(self, path : str, content) -> bool: class TestNameFunction(TestSpec): - """Test function names. + """Test names of functions and of some variables. Might report false positives. - Can accidentally spot names of variables too but it is fine because same conventions apply. """ - name = "function names" + name = "function/variable names" message = "Use lowerCamelCase for names of functions and variables." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True - # Look for "Type Name(..." - # regexp ? - words = line.split("(") - if len(words) < 2: + # Look for declarations of functions and variables. + + # a not enough successful attempt to use regex + # if not re.search("^([a-z]+ )?(const[\*\&]? )?([\w][\w:<>\[\]\*\& ]* )(const[\*\&]? )?([\w]+)[\(\{]", line): + # return True + + # Any function declaration has "(". + if not "(" in line: # does not accept simple variable declarations "type name;" or "type name{value};" return True - words = words[0].split() - if len(words) != 2: + + # Strip away irrelevant remainders of the line after the object name. + # For functions, stripping after "(" is enough but this way we also identify many declarations of variables. + for keyword in ("(", "{", ";", " = ", "//", "/*"): + if keyword in line: + line = line[:line.find(keyword)] + + # Check the words. + words = line.split() + + # number of words + if len(words) < 2: return True - if not words[1].isalnum(): + + # First word starts with a letter. (rejects "} else") + if not words[0][0].isalpha(): return True - if words[0] in ("return", ":", "#define", "#if", "new", "virtual", "case"): + + # Reject false positives with same structure. + if words[0] in ("return", "if", "else", "new", "case"): return True - if words[0].endswith(".template"): + + # if words[0].endswith(".template"): + # return True + + # multiple template arguments + # if words[0][-1] == ",": + # return True + + # All words before the name start with an alphanumeric character (underscores not allowed). + # Rejects expressions, e.g. * = += << }, but accepts numbers in array declarations. + if not all(w[0].isalnum() for w in words[:-1]): + # if not all(re.search("\w", w[0]) for w in words): return True - if words[0][-1] == ",": # multiple template arguments + + # Extract function/variable name. + funval_name = words[-1] # expecting the name in the last word + if "[" in funval_name: # remove brackets for arrays + funval_name = funval_name[:funval_name.find("[")] + if "::" in funval_name: # methods with the class prefix + funval_name = funval_name.split("::")[-1] + + # Check the name candidate. + + # names of variables and functions are alphanumeric strings + # if not funval_name.isalnum(): + + # names of variables and functions are alphanumeric strings (consider arrays) + # if not re.search("^[\w]+(\[[\w]+\])*$", funval_name): + + # Names of variables and functions are identifiers. + if not funval_name.isidentifier(): # should be same as ^[\w]+$ return True - # Extract function name. - function_name = words[1] - if "::" in function_name: - function_name = function_name.split("::")[1] + + print(f"{line} -> {funval_name}") + return True # The actual test comes here. - return is_lower_camel_case(function_name) + return is_lower_camel_case(funval_name) class TestNameMacro(TestSpec): @@ -336,7 +389,7 @@ class TestNameMacro(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True if not line.startswith("#define "): return True @@ -355,7 +408,7 @@ class TestNameConstant(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True words = line.split() if not "constexpr" in words or not "=" in words: @@ -379,7 +432,7 @@ class TestNameNamespace(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True if not line.startswith("namespace "): return True @@ -397,7 +450,7 @@ class TestNameUpperCamelCase(TestSpec): suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True if not line.startswith(f"{self.keyword} "): return True @@ -507,7 +560,7 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True return not "auto const" in line @@ -522,7 +575,7 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True if not line.startswith(("struct ", "class ")): return True @@ -628,7 +681,7 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path def test_line(self, line: str) -> bool: - if line.startswith("//"): + if is_comment_cpp(line): return True if not line.startswith("Configurable"): return True @@ -671,7 +724,7 @@ def main(): tests = [] # list of activated tests # Bad practice - enable_bad_practice = True + enable_bad_practice = False if enable_bad_practice: tests.append(TestIOStream()) tests.append(TestUsingStd()) @@ -687,18 +740,18 @@ def main(): enable_naming = True if enable_naming: tests.append(TestNameFunction()) - tests.append(TestNameMacro()) - tests.append(TestNameConstant()) - tests.append(TestNameNamespace()) - tests.append(TestNameEnum()) - tests.append(TestNameClass()) - tests.append(TestNameStruct()) - tests.append(TestNameFileCpp()) - tests.append(TestNameFilePython()) - tests.append(TestNameWorkflow()) + # tests.append(TestNameMacro()) + # tests.append(TestNameConstant()) + # tests.append(TestNameNamespace()) + # tests.append(TestNameEnum()) + # tests.append(TestNameClass()) + # tests.append(TestNameStruct()) + # tests.append(TestNameFileCpp()) + # tests.append(TestNameFilePython()) + # tests.append(TestNameWorkflow()) # PWGHF - enable_pwghf = True + enable_pwghf = False if enable_pwghf: tests.append(TestHfConstAuto()) tests.append(TestHfNameStructClass()) From 144f7d74ddcd3e2bf095f0a7534036bf52a1a764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 20:11:31 +0200 Subject: [PATCH 15/96] Spot more false positives in TestNameFunction --- Scripts/find_code_issues.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index b2f4c318b8b..41e053c489b 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -304,6 +304,7 @@ def test_file(self, path : str, content) -> bool: class TestNameFunction(TestSpec): """Test names of functions and of some variables. + Detects variable declarations with "(". Might report false positives. """ name = "function/variable names" @@ -320,7 +321,8 @@ def test_line(self, line: str) -> bool: # return True # Any function declaration has "(". - if not "(" in line: # does not accept simple variable declarations "type name;" or "type name{value};" + # This does not accept variable declarations without "(", e.g. "type name;" or "type name = value;" or "type name{value};" + if not "(" in line: return True # Strip away irrelevant remainders of the line after the object name. @@ -341,8 +343,11 @@ def test_line(self, line: str) -> bool: return True # Reject false positives with same structure. - if words[0] in ("return", "if", "else", "new", "case"): + if words[0] in ("return", "if", "else", "new", "case", "typename", "using"): return True + if len(words) > 2: + if words[1] in ("typename"): + return True # if words[0].endswith(".template"): # return True From 1cd5491ca8e8f12f4fcbd33a0313886cf51d6679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 21:02:17 +0200 Subject: [PATCH 16/96] TestNameFunction: Drop ( condition and improve false positive detection. --- Scripts/find_code_issues.py | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 41e053c489b..82357843022 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -303,9 +303,10 @@ def test_file(self, path : str, content) -> bool: class TestNameFunction(TestSpec): - """Test names of functions and of some variables. + """Test names of functions and of most variables. Detects variable declarations with "(". Might report false positives. + Cannot detect multiple declarations "type name1, name2;" """ name = "function/variable names" message = "Use lowerCamelCase for names of functions and variables." @@ -316,15 +317,6 @@ def test_line(self, line: str) -> bool: return True # Look for declarations of functions and variables. - # a not enough successful attempt to use regex - # if not re.search("^([a-z]+ )?(const[\*\&]? )?([\w][\w:<>\[\]\*\& ]* )(const[\*\&]? )?([\w]+)[\(\{]", line): - # return True - - # Any function declaration has "(". - # This does not accept variable declarations without "(", e.g. "type name;" or "type name = value;" or "type name{value};" - if not "(" in line: - return True - # Strip away irrelevant remainders of the line after the object name. # For functions, stripping after "(" is enough but this way we also identify many declarations of variables. for keyword in ("(", "{", ";", " = ", "//", "/*"): @@ -343,40 +335,25 @@ def test_line(self, line: str) -> bool: return True # Reject false positives with same structure. - if words[0] in ("return", "if", "else", "new", "case", "typename", "using"): + if words[0] in ("return", "if", "else", "new", "delete", "case", "typename", "using", "typedef", "enum", "namespace", "struct", "class"): + return True + if len(words) > 2 and words[1] in ("typename", "class", "struct"): return True - if len(words) > 2: - if words[1] in ("typename"): - return True - - # if words[0].endswith(".template"): - # return True - - # multiple template arguments - # if words[0][-1] == ",": - # return True # All words before the name start with an alphanumeric character (underscores not allowed). # Rejects expressions, e.g. * = += << }, but accepts numbers in array declarations. if not all(w[0].isalnum() for w in words[:-1]): - # if not all(re.search("\w", w[0]) for w in words): return True # Extract function/variable name. funval_name = words[-1] # expecting the name in the last word - if "[" in funval_name: # remove brackets for arrays + if "[" in funval_name: # Remove brackets for arrays. funval_name = funval_name[:funval_name.find("[")] - if "::" in funval_name: # methods with the class prefix + if "::" in funval_name: # Remove the class prefix for methods. funval_name = funval_name.split("::")[-1] # Check the name candidate. - # names of variables and functions are alphanumeric strings - # if not funval_name.isalnum(): - - # names of variables and functions are alphanumeric strings (consider arrays) - # if not re.search("^[\w]+(\[[\w]+\])*$", funval_name): - # Names of variables and functions are identifiers. if not funval_name.isidentifier(): # should be same as ^[\w]+$ return True From 5ec0e67ea6c19aefc5320c6b4e4120e79cd73157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 21:19:59 +0200 Subject: [PATCH 17/96] TestNameFunction: Catch more false positives --- Scripts/find_code_issues.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 82357843022..eb6b0af57e0 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -306,7 +306,8 @@ class TestNameFunction(TestSpec): """Test names of functions and of most variables. Detects variable declarations with "(". Might report false positives. - Cannot detect multiple declarations "type name1, name2;" + Does not detect multiple declarations "type name1, name2;" + Does not detect function arguments on the same line as the function declaration. """ name = "function/variable names" message = "Use lowerCamelCase for names of functions and variables." @@ -330,12 +331,12 @@ def test_line(self, line: str) -> bool: if len(words) < 2: return True - # First word starts with a letter. (rejects "} else") + # First word starts with a letter. if not words[0][0].isalpha(): return True # Reject false positives with same structure. - if words[0] in ("return", "if", "else", "new", "delete", "case", "typename", "using", "typedef", "enum", "namespace", "struct", "class"): + if words[0] in ("return", "if", "else", "new", "delete", "delete[]", "case", "typename", "using", "typedef", "enum", "namespace", "struct", "class"): return True if len(words) > 2 and words[1] in ("typename", "class", "struct"): return True From d6ed725a0ae7398565c7faae7a0869ca772fc0b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 22:21:45 +0200 Subject: [PATCH 18/96] TestNameFunction: Treat properly constexpr and arrays. --- Scripts/find_code_issues.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index eb6b0af57e0..d1279756d58 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -341,13 +341,22 @@ def test_line(self, line: str) -> bool: if len(words) > 2 and words[1] in ("typename", "class", "struct"): return True + # Identify the position of the name for cases "name[n + m]". + funval_name = words[-1] # expecting the name in the last word + if funval_name.endswith("]") and "[" not in funval_name: # it's an array and we do not have the name before "[" here + opens_brackets = ["[" in w for w in words] + index_name = opens_brackets.index(True) # the name is in the first element with "[" + funval_name = words[index_name] + words = words[:(index_name + 1)] # Strip away words after the name. + if len(words) < 2: # Check the adjusted number of words. + return True + # All words before the name start with an alphanumeric character (underscores not allowed). # Rejects expressions, e.g. * = += << }, but accepts numbers in array declarations. if not all(w[0].isalnum() for w in words[:-1]): return True # Extract function/variable name. - funval_name = words[-1] # expecting the name in the last word if "[" in funval_name: # Remove brackets for arrays. funval_name = funval_name[:funval_name.find("[")] if "::" in funval_name: # Remove the class prefix for methods. @@ -362,6 +371,8 @@ def test_line(self, line: str) -> bool: print(f"{line} -> {funval_name}") return True # The actual test comes here. + if "constexpr" in words[:-1]: + return is_camel_case(funval_name) return is_lower_camel_case(funval_name) From 01e8058a63c00633054244461f3cabdf1542c7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 22:37:23 +0200 Subject: [PATCH 19/96] Improve TestStdPrefix --- Scripts/find_code_issues.py | 48 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index d1279756d58..ca790b505aa 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -162,14 +162,14 @@ class TestStdPrefix(TestSpec): name = "std prefix" message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] - patterns = ["[^\w:.]vector<", "[^\w:.]array[<\{\()]", "[^\w:.]f?abs\(", "[^\w:.]min\(", "[^\w:.]max\(", "[^\w:.]log\(", "[^\w:.]exp\(", "[^\w:.]sin\(", "[^\w:.]cos\(", "[^\w:.]tan\(", "[^\w:.]atan\(", "[^\w:.]atan2\("] - # Add sqrt, pow + prefix_bad = "[^\w:.]" + patterns = ["vector<", "array[<\{\(]", "f?abs\(", "sqrt\(", "pow\(", "min\(", "max\(", "log\(", "exp\(", "sin\(", "cos\(", "tan\(", "atan\(", "atan2\("] def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True for pattern in self.patterns: - if re.search(pattern, line): + if re.search(f"{self.prefix_bad}{pattern}", line): return False # occurrences = re.findall(pattern, line) # if occurrences: @@ -368,8 +368,8 @@ def test_line(self, line: str) -> bool: if not funval_name.isidentifier(): # should be same as ^[\w]+$ return True - print(f"{line} -> {funval_name}") - return True + # print(f"{line} -> {funval_name}") + # return True # The actual test comes here. if "constexpr" in words[:-1]: return is_camel_case(funval_name) @@ -718,31 +718,31 @@ def main(): tests = [] # list of activated tests # Bad practice - enable_bad_practice = False + enable_bad_practice = True if enable_bad_practice: - tests.append(TestIOStream()) - tests.append(TestUsingStd()) - tests.append(TestUsingDirectives()) + # tests.append(TestIOStream()) + # tests.append(TestUsingStd()) + # tests.append(TestUsingDirectives()) tests.append(TestStdPrefix()) - tests.append(TestROOT()) - tests.append(TestPI()) - tests.append(TestLogging()) - tests.append(TestConstRefInForLoop()) - tests.append(TestConstRefInSubscription()) + # tests.append(TestROOT()) + # tests.append(TestPI()) + # tests.append(TestLogging()) + # tests.append(TestConstRefInForLoop()) + # tests.append(TestConstRefInSubscription()) # Naming conventions - enable_naming = True + enable_naming = False if enable_naming: tests.append(TestNameFunction()) - # tests.append(TestNameMacro()) - # tests.append(TestNameConstant()) - # tests.append(TestNameNamespace()) - # tests.append(TestNameEnum()) - # tests.append(TestNameClass()) - # tests.append(TestNameStruct()) - # tests.append(TestNameFileCpp()) - # tests.append(TestNameFilePython()) - # tests.append(TestNameWorkflow()) + tests.append(TestNameMacro()) + tests.append(TestNameConstant()) + tests.append(TestNameNamespace()) + tests.append(TestNameEnum()) + tests.append(TestNameClass()) + tests.append(TestNameStruct()) + tests.append(TestNameFileCpp()) + tests.append(TestNameFilePython()) + tests.append(TestNameWorkflow()) # PWGHF enable_pwghf = False From d170e1261fdf89510edcaf34dde74cea60f571e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 22:42:30 +0200 Subject: [PATCH 20/96] Rename TestNameFunction -> TestNameFunctionVariable. --- Scripts/find_code_issues.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index ca790b505aa..4d796bbfabd 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -302,12 +302,12 @@ def test_file(self, path : str, content) -> bool: # Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/naming_formatting.html -class TestNameFunction(TestSpec): +class TestNameFunctionVariable(TestSpec): """Test names of functions and of most variables. - Detects variable declarations with "(". Might report false positives. - Does not detect multiple declarations "type name1, name2;" + Does not detect multiple variable declarations, i.e. "type name1, name2;" Does not detect function arguments on the same line as the function declaration. + Does not check capitalisation for constexpr because of special rules for constants. See TestNameConstant. """ name = "function/variable names" message = "Use lowerCamelCase for names of functions and variables." @@ -733,7 +733,7 @@ def main(): # Naming conventions enable_naming = False if enable_naming: - tests.append(TestNameFunction()) + tests.append(TestNameFunctionVariable()) tests.append(TestNameMacro()) tests.append(TestNameConstant()) tests.append(TestNameNamespace()) From dcf8e93da79e4682a2b7b4fa6c05d2d7dbb01f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 23:14:41 +0200 Subject: [PATCH 21/96] Fix TestLogging --- Scripts/find_code_issues.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 4d796bbfabd..28e28e99b82 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -221,7 +221,7 @@ class TestLogging(TestSpec): name = "logging" message = "Consider using O2 logging (LOG, LOGF, LOGP)." suffixes = [".h", ".cxx"] - keywords = ["Printf(", "printf(", "cout <", "cin >"] + keywords = ["Printf(", "printf(", "cout <", "std::cout <"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path @@ -230,7 +230,7 @@ def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True for k in self.keywords: - if k in line: + if line.startswith(k): return False return True @@ -723,10 +723,10 @@ def main(): # tests.append(TestIOStream()) # tests.append(TestUsingStd()) # tests.append(TestUsingDirectives()) - tests.append(TestStdPrefix()) + # tests.append(TestStdPrefix()) # tests.append(TestROOT()) # tests.append(TestPI()) - # tests.append(TestLogging()) + tests.append(TestLogging()) # tests.append(TestConstRefInForLoop()) # tests.append(TestConstRefInSubscription()) From 13735a6d9811756a7f12b5a597af2b0ad595c468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sun, 21 Jul 2024 23:35:27 +0200 Subject: [PATCH 22/96] Improve TestIOStream, TestUsingStd, TestUsingDirectives --- Scripts/find_code_issues.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 28e28e99b82..8b5c6431662 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -124,25 +124,25 @@ def run(self, path : str, content) -> bool: class TestIOStream(TestSpec): """Detect included iostream.""" name = "include iostream" - message = "Including iostream is not allowed. Use O2 logging instead." + message = "Including iostream is discouraged. Use O2 logging instead." suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - return not line.strip().startswith("#include ") + return not line.startswith("#include ") class TestUsingStd(TestSpec): """Detect importing names from the std namespace.""" name = "import std names" - message = "Importing names from the std namespace is not allowed." - suffixes = [".h", ".cxx"] + message = "Importing names from the std namespace is not allowed in headers." + suffixes = [".h"] def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - return not line.strip().startswith("using std::") + return not line.startswith("using std::") class TestUsingDirectives(TestSpec): @@ -154,7 +154,7 @@ class TestUsingDirectives(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - return not line.strip().startswith("using namespace") + return not line.startswith("using namespace") class TestStdPrefix(TestSpec): @@ -720,13 +720,13 @@ def main(): # Bad practice enable_bad_practice = True if enable_bad_practice: - # tests.append(TestIOStream()) - # tests.append(TestUsingStd()) - # tests.append(TestUsingDirectives()) + tests.append(TestIOStream()) + tests.append(TestUsingStd()) + tests.append(TestUsingDirectives()) # tests.append(TestStdPrefix()) # tests.append(TestROOT()) # tests.append(TestPI()) - tests.append(TestLogging()) + # tests.append(TestLogging()) # tests.append(TestConstRefInForLoop()) # tests.append(TestConstRefInSubscription()) From 000e01049c5982ad0fd391f7b1fc96e5748b8701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 01:09:28 +0200 Subject: [PATCH 23/96] Add TestPdgDatabase, TestPdgCode --- Scripts/find_code_issues.py | 47 ++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 8b5c6431662..dec2f647aa8 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -216,6 +216,45 @@ def test_line(self, line: str) -> bool: return True +class TestPdgDatabase(TestSpec): + """Detect use of TDatabasePDG.""" + name = "pdg database" + message = "Direct use of TDatabasePDG is not allowed. Use o2::constants::physics::Mass... or Service." + suffixes = [".h", ".cxx"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and not "Macros/" in path + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + if "TDatabasePDG" in line: + return False + return True + + +class TestPdgCode(TestSpec): + """Detect use of hard-coded PDG codes.""" + name = "explicit pdg code" + message = "Avoid using hard-coded PDG codes. Use named values from PDG_t or o2::constants::physics::Pdg instead." + suffixes = [".h", ".cxx", ".C"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + if re.search("->(GetParticle|Mass)\([+-]?[0-9]+\)", line): + return False + match = re.search("[Pp][Dd][Gg][\w]* = [+-]?([0-9]+);", line) + if match: + code = match.group(1) + if code not in ("0", "1", "999"): + return False + return True + + class TestLogging(TestSpec): """Detect non-O2 logging.""" name = "logging" @@ -720,12 +759,14 @@ def main(): # Bad practice enable_bad_practice = True if enable_bad_practice: - tests.append(TestIOStream()) - tests.append(TestUsingStd()) - tests.append(TestUsingDirectives()) + # tests.append(TestIOStream()) + # tests.append(TestUsingStd()) + # tests.append(TestUsingDirectives()) # tests.append(TestStdPrefix()) # tests.append(TestROOT()) # tests.append(TestPI()) + # tests.append(TestPdgDatabase()) + tests.append(TestPdgCode()) # tests.append(TestLogging()) # tests.append(TestConstRefInForLoop()) # tests.append(TestConstRefInSubscription()) From ea7c984e5a55180ac78cfa0499fc0b6df27e72cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 02:00:48 +0200 Subject: [PATCH 24/96] Add TestPdgMass --- Scripts/find_code_issues.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index dec2f647aa8..793bd7bd2af 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -239,9 +239,6 @@ class TestPdgCode(TestSpec): message = "Avoid using hard-coded PDG codes. Use named values from PDG_t or o2::constants::physics::Pdg instead." suffixes = [".h", ".cxx", ".C"] - def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) - def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True @@ -255,6 +252,23 @@ def test_line(self, line: str) -> bool: return True +class TestPdgMass(TestSpec): + """Detect unnecessary call of Mass() for a known PDG code.""" + name = "known pdg mass" + message = "Consider using o2::constants::physics::Mass... instead of calling a database method for a known PDG code." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + pattern_pdg_code = "[+-]?(k[A-Z][a-zA-Z0-9]*|[0-9]+)" + if re.search(f"->GetParticle\({pattern_pdg_code}\)->Mass\(\)", line): + return False + if re.search(f"->Mass\({pattern_pdg_code}\)", line): + return False + return True + + class TestLogging(TestSpec): """Detect non-O2 logging.""" name = "logging" @@ -766,7 +780,8 @@ def main(): # tests.append(TestROOT()) # tests.append(TestPI()) # tests.append(TestPdgDatabase()) - tests.append(TestPdgCode()) + # tests.append(TestPdgCode()) + tests.append(TestPdgMass()) # tests.append(TestLogging()) # tests.append(TestConstRefInForLoop()) # tests.append(TestConstRefInSubscription()) From 829413b747ca92eeaa6f30d928e2e2294450d348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 03:44:39 +0200 Subject: [PATCH 25/96] Add test for documentation --- Scripts/find_code_issues.py | 74 ++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/Scripts/find_code_issues.py b/Scripts/find_code_issues.py index 793bd7bd2af..fe164224857 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/find_code_issues.py @@ -351,6 +351,49 @@ def test_file(self, path : str, content) -> bool: return passed +# Documentation +# Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/comments_guidelines.html + + +class TestDocumentationFile(TestSpec): + """Test mandatory documentation of C++ files.""" + name = "file documentation" + message = "Provide mandatory file documentation." + suffixes = [".h", ".cxx", ".C"] + per_line = False + + def test_file(self, path : str, content) -> bool: + passed = False + doc_items = [] + doc_items.append({"keyword": "file", "pattern": f"{os.path.basename(path)}$", "found": False}) + doc_items.append({"keyword": "brief", "pattern": "[\w]+", "found": False}) + doc_items.append({"keyword": "author", "pattern": "[\w]+", "found": False}) + doc_prefix = "///" + n_lines_copyright = 11 + last_doc_line = n_lines_copyright + + for i, line in enumerate(content): + if i < n_lines_copyright: # Skip copyright lines. + continue + if line.strip() and not line.startswith(doc_prefix): # Stop at the first non-empty non-doc line. + break + if line.startswith(doc_prefix): + last_doc_line = (i + 1) + for item in doc_items: + if re.search(f"^{doc_prefix} [\\\\@]{item['keyword']} {item['pattern']}", line): + item["found"] = True + # print_error(path, i + 1, self.name, f"Found \{item['keyword']}.") + break + if all(item["found"] for item in doc_items): # All items have been found. + passed = True + break + if not passed: + for item in doc_items: + if not item["found"]: + print_error(path, last_doc_line, self.name, f"Documentation for \{item['keyword']} is missing or incorrect.") + return passed + + # Naming conventions # Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/naming_formatting.html @@ -773,21 +816,26 @@ def main(): # Bad practice enable_bad_practice = True if enable_bad_practice: - # tests.append(TestIOStream()) - # tests.append(TestUsingStd()) - # tests.append(TestUsingDirectives()) - # tests.append(TestStdPrefix()) - # tests.append(TestROOT()) - # tests.append(TestPI()) - # tests.append(TestPdgDatabase()) - # tests.append(TestPdgCode()) + tests.append(TestIOStream()) + tests.append(TestUsingStd()) + tests.append(TestUsingDirectives()) + tests.append(TestStdPrefix()) + tests.append(TestROOT()) + tests.append(TestPI()) + tests.append(TestPdgDatabase()) + tests.append(TestPdgCode()) tests.append(TestPdgMass()) - # tests.append(TestLogging()) - # tests.append(TestConstRefInForLoop()) - # tests.append(TestConstRefInSubscription()) + tests.append(TestLogging()) + tests.append(TestConstRefInForLoop()) + tests.append(TestConstRefInSubscription()) + + # Documentation + enable_documentation = True + if enable_documentation: + tests.append(TestDocumentationFile()) # Naming conventions - enable_naming = False + enable_naming = True if enable_naming: tests.append(TestNameFunctionVariable()) tests.append(TestNameMacro()) @@ -801,7 +849,7 @@ def main(): tests.append(TestNameWorkflow()) # PWGHF - enable_pwghf = False + enable_pwghf = True if enable_pwghf: tests.append(TestHfConstAuto()) tests.append(TestHfNameStructClass()) From e9f49404ae9a293dfc4d4538045c9fb584196cfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 03:48:55 +0200 Subject: [PATCH 26/96] Rename linter --- Scripts/{find_code_issues.py => o2_linter.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename Scripts/{find_code_issues.py => o2_linter.py} (99%) diff --git a/Scripts/find_code_issues.py b/Scripts/o2_linter.py similarity index 99% rename from Scripts/find_code_issues.py rename to Scripts/o2_linter.py index fe164224857..44d3363f806 100644 --- a/Scripts/find_code_issues.py +++ b/Scripts/o2_linter.py @@ -12,7 +12,7 @@ # or submit itself to any jurisdiction. """! -@brief Find issues in code +@brief O2 linter (Find issues in O2 code) @author Vít Kučera , Inha University @date 2024-07-14 """ From 17ef3bce1e7567b4369a41dad16bcb6740681975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 04:42:00 +0200 Subject: [PATCH 27/96] TestDocumentationFile: Allow multiple spaces after keyword. --- Scripts/o2_linter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 44d3363f806..5d29193f8c4 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -380,7 +380,7 @@ def test_file(self, path : str, content) -> bool: if line.startswith(doc_prefix): last_doc_line = (i + 1) for item in doc_items: - if re.search(f"^{doc_prefix} [\\\\@]{item['keyword']} {item['pattern']}", line): + if re.search(f"^{doc_prefix} [\\\\@]{item['keyword']} +{item['pattern']}", line): item["found"] = True # print_error(path, i + 1, self.name, f"Found \{item['keyword']}.") break @@ -390,7 +390,7 @@ def test_file(self, path : str, content) -> bool: if not passed: for item in doc_items: if not item["found"]: - print_error(path, last_doc_line, self.name, f"Documentation for \{item['keyword']} is missing or incorrect.") + print_error(path, last_doc_line, self.name, f"Documentation for \{item['keyword']} is missing, incorrect or misplaced.") return passed @@ -764,7 +764,7 @@ def test_file(self, path : str, content) -> bool: class TestHfNameConfigurable(TestSpec): """PWGHF: Test names of configurables.""" name = "PWGHF: Configurable names" - message = "Use camelCase for Configurable names and use the same name for the struct member as for the JSON string." + message = "Use lowerCamelCase for Configurable names and use the same name for the struct member as for the JSON string." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: From fbd0cf89e9b8815aeedcb6dd468f425996040546 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 05:54:33 +0200 Subject: [PATCH 28/96] Add workflow file --- .github/workflows/o2-linter.yml | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/o2-linter.yml diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml new file mode 100644 index 00000000000..57a72c6dd19 --- /dev/null +++ b/.github/workflows/o2-linter.yml @@ -0,0 +1,34 @@ +--- +# Find code issues in modified text files +name: O2 linter + +'on': [pull_request, push] +permissions: {} +env: + MAIN_BRANCH: master + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +jobs: + o2-linter: + name: Lint O2 code + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # needed to get the full history + # - name: Fetch upstream + # run: | + # # Fetch the main upstream branch to find the common ancestor + # git config --global user.name "Nemo" # required on some servers + # git remote add upstream https://github.com/AliceO2Group/Run3Analysisvalidation.git || exit 1 + # git fetch upstream ${{ env.MAIN_BRANCH }} || exit 1 + - name: Run tests + run: | + # Diff against the common ancestor of the source branch and the target branch. + files=$(git diff --diff-filter d --name-only ${{ env.MAIN_BRANCH }}...) + python Scripts/o2_linter.py -g $files + done From 593a2d109537195dc40fca0d86938f1fcb2ffea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 06:04:14 +0200 Subject: [PATCH 29/96] Print remote --- .github/workflows/o2-linter.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index 57a72c6dd19..24d48f32eec 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -28,6 +28,7 @@ jobs: # git fetch upstream ${{ env.MAIN_BRANCH }} || exit 1 - name: Run tests run: | + git remote -v # Diff against the common ancestor of the source branch and the target branch. files=$(git diff --diff-filter d --name-only ${{ env.MAIN_BRANCH }}...) python Scripts/o2_linter.py -g $files From 6b8bca38f6033bd6f818fb150534346fdc36f891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 06:06:15 +0200 Subject: [PATCH 30/96] Fetch master --- .github/workflows/o2-linter.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index 24d48f32eec..6e4b8958f2a 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -20,15 +20,15 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 # needed to get the full history - # - name: Fetch upstream - # run: | - # # Fetch the main upstream branch to find the common ancestor - # git config --global user.name "Nemo" # required on some servers - # git remote add upstream https://github.com/AliceO2Group/Run3Analysisvalidation.git || exit 1 - # git fetch upstream ${{ env.MAIN_BRANCH }} || exit 1 + - name: Fetch upstream + run: | + # Fetch the main upstream branch to find the common ancestor + # git config --global user.name "Nemo" # required on some servers + # git remote add upstream https://github.com/AliceO2Group/Run3Analysisvalidation.git || exit 1 + # git fetch upstream ${{ env.MAIN_BRANCH }} || exit 1 + git fetch origin ${{ env.MAIN_BRANCH }} || exit 1 - name: Run tests run: | - git remote -v # Diff against the common ancestor of the source branch and the target branch. files=$(git diff --diff-filter d --name-only ${{ env.MAIN_BRANCH }}...) python Scripts/o2_linter.py -g $files From 30910e91b407e8653165cf7f2fe2532040fa39cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 06:16:13 +0200 Subject: [PATCH 31/96] Add origin --- .github/workflows/o2-linter.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index 6e4b8958f2a..714f6257696 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -26,10 +26,10 @@ jobs: # git config --global user.name "Nemo" # required on some servers # git remote add upstream https://github.com/AliceO2Group/Run3Analysisvalidation.git || exit 1 # git fetch upstream ${{ env.MAIN_BRANCH }} || exit 1 - git fetch origin ${{ env.MAIN_BRANCH }} || exit 1 + # git fetch origin ${{ env.MAIN_BRANCH }} || exit 1 - name: Run tests run: | # Diff against the common ancestor of the source branch and the target branch. - files=$(git diff --diff-filter d --name-only ${{ env.MAIN_BRANCH }}...) + files=$(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) python Scripts/o2_linter.py -g $files done From 96315eea9446eb713df5022cb00ef09c59afbd5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 06:29:14 +0200 Subject: [PATCH 32/96] Annotate only on pull requests. --- .github/workflows/o2-linter.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index 714f6257696..71fa21fde3c 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -31,5 +31,6 @@ jobs: run: | # Diff against the common ancestor of the source branch and the target branch. files=$(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) - python Scripts/o2_linter.py -g $files + [ ${{ github.event_name }} == 'pull_request' ] && options="-g" + python Scripts/o2_linter.py $options $files done From 6c2ced2036872b82a39603613e932ebc65d15f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 06:43:31 +0200 Subject: [PATCH 33/96] Simplify --- .github/workflows/o2-linter.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index 71fa21fde3c..cab05bd35b9 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -20,17 +20,9 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 # needed to get the full history - - name: Fetch upstream - run: | - # Fetch the main upstream branch to find the common ancestor - # git config --global user.name "Nemo" # required on some servers - # git remote add upstream https://github.com/AliceO2Group/Run3Analysisvalidation.git || exit 1 - # git fetch upstream ${{ env.MAIN_BRANCH }} || exit 1 - # git fetch origin ${{ env.MAIN_BRANCH }} || exit 1 - name: Run tests run: | # Diff against the common ancestor of the source branch and the target branch. files=$(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) [ ${{ github.event_name }} == 'pull_request' ] && options="-g" python Scripts/o2_linter.py $options $files - done From 033ae2e7d9e2ebec98f31060adac4c0659a919fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 14:31:48 +0200 Subject: [PATCH 34/96] Hyphenate test names --- Scripts/o2_linter.py | 58 ++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 5d29193f8c4..cea5f278847 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -123,7 +123,7 @@ def run(self, path : str, content) -> bool: class TestIOStream(TestSpec): """Detect included iostream.""" - name = "include iostream" + name = "include-iostream" message = "Including iostream is discouraged. Use O2 logging instead." suffixes = [".h", ".cxx"] @@ -135,7 +135,7 @@ def test_line(self, line: str) -> bool: class TestUsingStd(TestSpec): """Detect importing names from the std namespace.""" - name = "import std names" + name = "import-std-name" message = "Importing names from the std namespace is not allowed in headers." suffixes = [".h"] @@ -147,7 +147,7 @@ def test_line(self, line: str) -> bool: class TestUsingDirectives(TestSpec): """Detect using directives in headers.""" - name = "using directives" + name = "using-directive" message = "Using directives are not allowed in headers." suffixes = [".h"] @@ -159,7 +159,7 @@ def test_line(self, line: str) -> bool: class TestStdPrefix(TestSpec): """Detect missing std:: prefix for common names from the std namespace.""" - name = "std prefix" + name = "std-prefix" message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] prefix_bad = "[^\w:.]" @@ -180,7 +180,7 @@ def test_line(self, line: str) -> bool: class TestROOT(TestSpec): """Detect use of unnecessary ROOT entities.""" - name = "ROOT entities" + name = "root-entity" message = "Consider replacing ROOT entities with STD C++ or O2 entities." suffixes = [".h", ".cxx"] keywords = ["TMath", "Double_t", "Float_t", "Int_t", "Bool_t"] @@ -199,7 +199,7 @@ def test_line(self, line: str) -> bool: class TestPI(TestSpec): """Detect use of external PI.""" - name = "external PI" + name = "external-pi" message = "Consider using the PI constant (and its multiples) defined in o2::constants::math." suffixes = [".h", ".cxx"] keywords = ["M_PI", "TMath::Pi", "TMath::TwoPi"] @@ -218,7 +218,7 @@ def test_line(self, line: str) -> bool: class TestPdgDatabase(TestSpec): """Detect use of TDatabasePDG.""" - name = "pdg database" + name = "pdg/database" message = "Direct use of TDatabasePDG is not allowed. Use o2::constants::physics::Mass... or Service." suffixes = [".h", ".cxx"] @@ -235,7 +235,7 @@ def test_line(self, line: str) -> bool: class TestPdgCode(TestSpec): """Detect use of hard-coded PDG codes.""" - name = "explicit pdg code" + name = "pdg/explicit-code" message = "Avoid using hard-coded PDG codes. Use named values from PDG_t or o2::constants::physics::Pdg instead." suffixes = [".h", ".cxx", ".C"] @@ -254,7 +254,7 @@ def test_line(self, line: str) -> bool: class TestPdgMass(TestSpec): """Detect unnecessary call of Mass() for a known PDG code.""" - name = "known pdg mass" + name = "pdg/known-mass" message = "Consider using o2::constants::physics::Mass... instead of calling a database method for a known PDG code." suffixes = [".h", ".cxx", ".C"] @@ -290,7 +290,7 @@ def test_line(self, line: str) -> bool: class TestConstRefInForLoop(TestSpec): """Test const refs in range-based for loops.""" - name = "const ref in for loop" + name = "const-ref-in-for-loop" message = "Use constant references for non-modified iterators in range-based for loops." suffixes = [".h", ".cxx", ".C"] @@ -305,7 +305,7 @@ def test_line(self, line: str) -> bool: class TestConstRefInSubscription(TestSpec): """Test const refs in process function subscriptions.""" - name = "const ref in process" + name = "const-ref-in-process" message = "Use constant references for table subscriptions in process functions." suffixes = [".cxx"] # suffixes = [".h"] @@ -357,7 +357,7 @@ def test_file(self, path : str, content) -> bool: class TestDocumentationFile(TestSpec): """Test mandatory documentation of C++ files.""" - name = "file documentation" + name = "doc/file" message = "Provide mandatory file documentation." suffixes = [".h", ".cxx", ".C"] per_line = False @@ -405,7 +405,7 @@ class TestNameFunctionVariable(TestSpec): Does not detect function arguments on the same line as the function declaration. Does not check capitalisation for constexpr because of special rules for constants. See TestNameConstant. """ - name = "function/variable names" + name = "name/function-variable" message = "Use lowerCamelCase for names of functions and variables." suffixes = [".h", ".cxx", ".C"] @@ -474,7 +474,7 @@ def test_line(self, line: str) -> bool: class TestNameMacro(TestSpec): """Test macro names.""" - name = "macro names" + name = "name/macro" message = "Use SCREAMING_SNAKE_CASE for macro names." suffixes = [".h", ".cxx", ".C"] @@ -493,7 +493,7 @@ def test_line(self, line: str) -> bool: class TestNameConstant(TestSpec): """Test constexpr constant names.""" - name = "constexpr constant names" + name = "name/constexpr-constant" message = "Use UpperCamelCase for constexpr constant names. Names of special constants may be prefixed with \"k\"." suffixes = [".h", ".cxx", ".C"] @@ -517,7 +517,7 @@ def test_line(self, line: str) -> bool: class TestNameNamespace(TestSpec): """Test names of namespaces.""" - name = "namespace names" + name = "name/namespace" message = "Use snake_case for names of namespaces." suffixes = [".h", ".cxx", ".C"] @@ -535,7 +535,7 @@ def test_line(self, line: str) -> bool: class TestNameUpperCamelCase(TestSpec): """Base class for a test of UpperCamelCase names.""" keyword = "key" - name = f"{keyword} UpperCamelCase" + name = f"name/{keyword}" message = f"Use UpperCamelCase for names of {keyword}." suffixes = [".h", ".cxx", ".C"] @@ -558,27 +558,27 @@ def test_line(self, line: str) -> bool: class TestNameEnum(TestNameUpperCamelCase): """Test names of enumerators.""" keyword = "enum" - name = f"{keyword} names" + name = f"name/enum" message = f"Use UpperCamelCase for names of enumerators and their values." class TestNameClass(TestNameUpperCamelCase): """Test names of classes.""" keyword = "class" - name = f"{keyword} names" + name = f"name/class" message = f"Use UpperCamelCase for names of classes." class TestNameStruct(TestNameUpperCamelCase): """Test names of structs.""" keyword = "struct" - name = f"{keyword} names" + name = f"name/struct" message = f"Use UpperCamelCase for names of structs." class TestNameFileCpp(TestSpec): """Test names of C++ files.""" - name = "C++ file names" + name = "name/file-cpp" message = "Use lowerCamelCase or UpperCamelCase for names of C++ files. See the O2 naming conventions for details." suffixes = [".h", ".cxx", ".C"] per_line = False @@ -590,7 +590,7 @@ def test_file(self, path : str, content) -> bool: class TestNameFilePython(TestSpec): """Test names of Python files.""" - name = "Python file names" + name = "name/file-python" message = "Use snake_case for names of Python files." suffixes = [".py", ".ipynb"] per_line = False @@ -602,7 +602,7 @@ def test_file(self, path : str, content) -> bool: class TestNameWorkflow(TestSpec): """Test names of O2 workflows.""" - name = "O2 workflow names" + name = "name/o2-workflow" message = "Use kebab-case for names of workflows and match the name of the workflow file." suffixes = ["CMakeLists.txt"] per_line = False @@ -637,12 +637,12 @@ def test_file(self, path : str, content) -> bool: return passed -# PWG-specific +# PWG-HF class TestHfConstAuto(TestSpec): """PWGHF: Detect swapped const auto.""" - name = "PWGHF: const auto" + name = "pwghf/const-auto" message = "Use \"const auto\" instead of \"auto const\"." suffixes = [".h", ".cxx"] @@ -657,7 +657,7 @@ def test_line(self, line: str) -> bool: class TestHfNameStructClass(TestSpec): """PWGHF: Test names of structs and classes.""" - name = "PWGHF: struct/class names" + name = "pwghf/name/struct-class" message = "Names of PWGHF structs and classes must start with \"Hf\"." suffixes = [".h", ".cxx"] @@ -681,7 +681,7 @@ def test_line(self, line: str) -> bool: class TestHfStructMembers(TestSpec): """PWGHF: Test order of struct members. Caveat: Does not see Configurables in ConfigurableGroup.""" - name = "PWGHF: struct member order" + name = "pwghf/struct member order" message = "Declare struct members in the conventional order. See the PWGHF coding guidelines." suffixes = [".cxx"] per_line = False @@ -730,7 +730,7 @@ def test_file(self, path : str, content) -> bool: class TestHfNameFileWorkflow(TestSpec): """PWGHF: Test names of workflow files.""" - name = "PWGHF: workflow file names" + name = "pwghf/name/workflow-file" message = "Name of a workflow file must match the name of the main task in it. See the PWGHF O2 naming conventions for details." suffixes = [".cxx"] per_line = False @@ -763,7 +763,7 @@ def test_file(self, path : str, content) -> bool: class TestHfNameConfigurable(TestSpec): """PWGHF: Test names of configurables.""" - name = "PWGHF: Configurable names" + name = "pwghf/name/configurable" message = "Use lowerCamelCase for Configurable names and use the same name for the struct member as for the JSON string." suffixes = [".h", ".cxx"] From 7b7377e444bcfc01c52126033dd35d45f90ae45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 15:18:07 +0200 Subject: [PATCH 35/96] Minor edits --- .github/workflows/o2-linter.yml | 2 +- Scripts/o2_linter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index cab05bd35b9..f8c5c712982 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -13,7 +13,7 @@ concurrency: jobs: o2-linter: - name: Lint O2 code + name: O2 linter runs-on: ubuntu-latest steps: - name: Checkout Code diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index cea5f278847..cf6d228a1f9 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -68,7 +68,7 @@ def print_error(path: str, line: int, title: str, message: str) -> str: def is_comment_cpp(line: str) -> bool: """Test whether a line is a C++ comment.""" - return line.startswith(("//", "/*")) + return line.strip().startswith(("//", "/*")) class TestSpec(ABC): From 12de8858832ac8d375dbb16770ed645ecb6c2234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 16:18:39 +0200 Subject: [PATCH 36/96] Add option to disable test --- Scripts/o2_linter.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index cf6d228a1f9..7a79ea81c5b 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -17,7 +17,7 @@ @date 2024-07-14 """ -from abc import ABC #, abstractmethod +from abc import ABC import argparse import os import re @@ -73,7 +73,7 @@ def is_comment_cpp(line: str) -> bool: class TestSpec(ABC): """Prototype of a test class""" - name = "test template" # short name of the test + name = "test-template" # short name of the test message = "Test failed" # error message suffixes = [] # suffixes of files to test per_line = True # Test lines separately one by one. @@ -82,12 +82,21 @@ def file_matches(self, path: str) -> bool: """Test whether the path matches the pattern for files to test.""" return path.endswith(tuple(self.suffixes)) if self.suffixes else True - # @abstractmethod + def is_disabled(self, line: str, prefix_comment="//") -> bool: + """Detect whether the test is explicitly disabled.""" + prefix_disable = "o2-linter: disable=" + for prefix in [prefix_comment, prefix_disable]: + if not prefix in line: + return False + line = line[(line.find(prefix) + len(prefix)):] # Strip away part before prefix. + if self.name in line: + return True + return False + def test_line(self, line: str) -> bool: """Test a line.""" raise NotImplementedError() - # @abstractmethod def test_file(self, path : str, content) -> bool: """Test a file in a way that cannot be done line by line.""" raise NotImplementedError() @@ -105,6 +114,8 @@ def run(self, path : str, content) -> bool: if not line: continue # print(i + 1, line) + if self.is_disabled(line): + continue if not self.test_line(line): passed = False print_error(path, i + 1, self.name, self.message) @@ -308,7 +319,6 @@ class TestConstRefInSubscription(TestSpec): name = "const-ref-in-process" message = "Use constant references for table subscriptions in process functions." suffixes = [".cxx"] - # suffixes = [".h"] per_line = False def test_file(self, path : str, content) -> bool: @@ -320,6 +330,8 @@ def test_file(self, path : str, content) -> bool: line = line.strip() if is_comment_cpp(line): continue + if self.is_disabled(line): + continue if re.search("^void process[\w]*\(", line): line_process = (i + 1) i_closing = line.rfind(")") @@ -613,6 +625,8 @@ def test_file(self, path : str, content) -> bool: for i, line in enumerate(content): if not line.startswith("o2physics_add_dpl_workflow("): continue + if self.is_disabled(line, "#"): + continue # Extract workflow name. workflow_name = line.strip().split("(")[1] if not is_kebab_case(workflow_name): From 55b4edfcf98589ebe365159dee0918d1d924859d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 20:38:02 +0200 Subject: [PATCH 37/96] Minor fixes --- Scripts/o2_linter.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 7a79ea81c5b..e006711664e 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -28,7 +28,7 @@ def is_camel_case(name: str) -> bool: """forExample or ForExample""" - return not "_" in name and not "-" in name + return not "_" in name and not "-" in name and not " " in name def is_upper_camel_case(name: str) -> bool: @@ -43,17 +43,17 @@ def is_lower_camel_case(name: str) -> bool: def is_kebab_case(name: str) -> bool: """for-example""" - return name.islower() and not "_" in name + return name.islower() and not "_" in name and not " " in name def is_snake_case(name: str) -> bool: """for_example""" - return name.islower() and not "-" in name + return name.islower() and not "-" in name and not " " in name def is_screaming_snake_case(name: str) -> bool: """FOR_EXAMPLE""" - return name.isupper() and not "-" in name + return name.isupper() and not "-" in name and not " " in name def print_error(path: str, line: int, title: str, message: str) -> str: @@ -628,7 +628,7 @@ def test_file(self, path : str, content) -> bool: if self.is_disabled(line, "#"): continue # Extract workflow name. - workflow_name = line.strip().split("(")[1] + workflow_name = line.strip().split("(")[1].split()[0] if not is_kebab_case(workflow_name): passed = False print_error(path, i + 1, self.name, f"Invalid workflow name: {workflow_name}.") From 97a661ed2e61d278e842dcdf0e8e6a3121296552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 22 Jul 2024 22:41:27 +0200 Subject: [PATCH 38/96] Improve regex --- Scripts/o2_linter.py | 51 +++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index e006711664e..6f1776144af 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -173,14 +173,14 @@ class TestStdPrefix(TestSpec): name = "std-prefix" message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] - prefix_bad = "[^\w:.]" - patterns = ["vector<", "array[<\{\(]", "f?abs\(", "sqrt\(", "pow\(", "min\(", "max\(", "log\(", "exp\(", "sin\(", "cos\(", "tan\(", "atan\(", "atan2\("] + prefix_bad = r"[^\w:\.]" + patterns = [r"vector<", r"array[<\{\(]", r"f?abs\(", r"sqrt\(", r"pow\(", r"min\(", r"max\(", r"log\(", r"exp\(", r"sin\(", r"cos\(", r"tan\(", r"atan\(", r"atan2\("] def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True for pattern in self.patterns: - if re.search(f"{self.prefix_bad}{pattern}", line): + if re.search(fr"{self.prefix_bad}{pattern}", line): return False # occurrences = re.findall(pattern, line) # if occurrences: @@ -194,18 +194,15 @@ class TestROOT(TestSpec): name = "root-entity" message = "Consider replacing ROOT entities with STD C++ or O2 entities." suffixes = [".h", ".cxx"] - keywords = ["TMath", "Double_t", "Float_t", "Int_t", "Bool_t"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: + pattern = r"TMath|(U?(Int|Char|Short)|Double(32)?|Float(16)?|U?Long(64)?|Bool)_t" if is_comment_cpp(line): return True - for k in self.keywords: - if k in line: - return False - return True + return re.search(pattern, line) is None class TestPI(TestSpec): @@ -213,18 +210,15 @@ class TestPI(TestSpec): name = "external-pi" message = "Consider using the PI constant (and its multiples) defined in o2::constants::math." suffixes = [".h", ".cxx"] - keywords = ["M_PI", "TMath::Pi", "TMath::TwoPi"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: + pattern = r"M_PI|TMath::(Two)?Pi" if is_comment_cpp(line): return True - for k in self.keywords: - if k in line: - return False - return True + return re.search(pattern, line) is None class TestPdgDatabase(TestSpec): @@ -239,9 +233,7 @@ def file_matches(self, path: str) -> bool: def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - if "TDatabasePDG" in line: - return False - return True + return not "TDatabasePDG" in line class TestPdgCode(TestSpec): @@ -253,9 +245,9 @@ class TestPdgCode(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - if re.search("->(GetParticle|Mass)\([+-]?[0-9]+\)", line): + if re.search(r"->(GetParticle|Mass)\([+-]?[0-9]+\)", line): return False - match = re.search("[Pp][Dd][Gg][\w]* = [+-]?([0-9]+);", line) + match = re.search(r"[Pp][Dd][Gg][\w]* = [+-]?([0-9]+);", line) if match: code = match.group(1) if code not in ("0", "1", "999"): @@ -272,10 +264,10 @@ class TestPdgMass(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - pattern_pdg_code = "[+-]?(k[A-Z][a-zA-Z0-9]*|[0-9]+)" - if re.search(f"->GetParticle\({pattern_pdg_code}\)->Mass\(\)", line): + pattern_pdg_code = r"[+-]?(k[A-Z][a-zA-Z0-9]*|[0-9]+)" + if re.search(fr"->GetParticle\({pattern_pdg_code}\)->Mass\(\)", line): return False - if re.search(f"->Mass\({pattern_pdg_code}\)", line): + if re.search(fr"->Mass\({pattern_pdg_code}\)", line): return False return True @@ -285,18 +277,15 @@ class TestLogging(TestSpec): name = "logging" message = "Consider using O2 logging (LOG, LOGF, LOGP)." suffixes = [".h", ".cxx"] - keywords = ["Printf(", "printf(", "cout <", "std::cout <"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and not "Macros/" in path def test_line(self, line: str) -> bool: + pattern = r"^([Pp]rintf\(|(std::)?cout <)" if is_comment_cpp(line): return True - for k in self.keywords: - if line.startswith(k): - return False - return True + return re.search(pattern, line) is None class TestConstRefInForLoop(TestSpec): @@ -311,7 +300,7 @@ def test_line(self, line: str) -> bool: if not line.startswith("for (") or " : " not in line: return True line = line[:line.index(" : ")] # keep only the iterator part - return True if re.search("\([\w]* ?const ?[\w]*&", line) else False + return True if re.search(r"\([\w]* ?const ?[\w]*&", line) else False class TestConstRefInSubscription(TestSpec): @@ -332,7 +321,7 @@ def test_file(self, path : str, content) -> bool: continue if self.is_disabled(line): continue - if re.search("^void process[\w]*\(", line): + if re.search(r"^void process[\w]*\(", line): line_process = (i + 1) i_closing = line.rfind(")") i_start = line.find("(") + 1 @@ -348,7 +337,7 @@ def test_file(self, path : str, content) -> bool: if line_process > 0 and n_parens_opened == 0: # process arguments # sanitise template arguments - template_args = re.findall("<[\w:, ]*>", arguments) + template_args = re.findall(r"<[\w:, ]*>", arguments) if template_args: for arg in template_args: if ", " in arg: @@ -356,7 +345,7 @@ def test_file(self, path : str, content) -> bool: words = arguments.split(", ") # test for arg in words: - if not re.search("[\w<>:]* ?const ?[\w<>:]*&", arg): + if not re.search(r"[\w<>:]* ?const ?[\w<>:]*&", arg): passed = False print_error(path, i + 1, self.name, f"Argument {arg} is not const&.") line_process = 0 @@ -392,7 +381,7 @@ def test_file(self, path : str, content) -> bool: if line.startswith(doc_prefix): last_doc_line = (i + 1) for item in doc_items: - if re.search(f"^{doc_prefix} [\\\\@]{item['keyword']} +{item['pattern']}", line): + if re.search(fr"^{doc_prefix} [\\@]{item['keyword']} +{item['pattern']}", line): item["found"] = True # print_error(path, i + 1, self.name, f"Found \{item['keyword']}.") break From 57fd08c897a2d13f15da666f80d2eb1d98ee31e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 04:53:18 +0200 Subject: [PATCH 39/96] TestPdgCode, TestDocumentationFile: Improve pattern. TestNameConstant: Improve name extraction. --- Scripts/o2_linter.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 6f1776144af..307b14c5bff 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -247,7 +247,7 @@ def test_line(self, line: str) -> bool: return True if re.search(r"->(GetParticle|Mass)\([+-]?[0-9]+\)", line): return False - match = re.search(r"[Pp][Dd][Gg][\w]* = [+-]?([0-9]+);", line) + match = re.search(r"[Pp][Dd][Gg][\w]* ={1,2} [+-]?([0-9]+);", line) if match: code = match.group(1) if code not in ("0", "1", "999"): @@ -366,9 +366,9 @@ class TestDocumentationFile(TestSpec): def test_file(self, path : str, content) -> bool: passed = False doc_items = [] - doc_items.append({"keyword": "file", "pattern": f"{os.path.basename(path)}$", "found": False}) - doc_items.append({"keyword": "brief", "pattern": "[\w]+", "found": False}) - doc_items.append({"keyword": "author", "pattern": "[\w]+", "found": False}) + doc_items.append({"keyword": "file", "pattern": fr"{os.path.basename(path)}$", "found": False}) + doc_items.append({"keyword": "brief", "pattern": r"\w.* \w", "found": False}) + doc_items.append({"keyword": "author", "pattern": r"[\w]+", "found": False}) doc_prefix = "///" n_lines_copyright = 11 last_doc_line = n_lines_copyright @@ -510,6 +510,10 @@ def test_line(self, line: str) -> bool: if constant_name.endswith("]") and "[" not in constant_name: # it's an array and we do not have the name before "[" here opens_brackets = ["[" in w for w in words] constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" + if "[" in constant_name: # Remove brackets for arrays. + constant_name = constant_name[:constant_name.find("[")] + if "::" in constant_name: # Remove the class prefix for methods. + constant_name = constant_name.split("::")[-1] # The actual test comes here. if constant_name.startswith("k"): # exception for special constants constant_name = constant_name[1:] # test the name without "k" From 79155ed7d9b9ba0748ad4aab30e948ce61c69515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 06:03:21 +0200 Subject: [PATCH 40/96] TestConstRefInForLoop: Fix regex --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 307b14c5bff..41235c85f6d 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -300,7 +300,7 @@ def test_line(self, line: str) -> bool: if not line.startswith("for (") or " : " not in line: return True line = line[:line.index(" : ")] # keep only the iterator part - return True if re.search(r"\([\w]* ?const ?[\w]*&", line) else False + return re.search(r"(\w const|const \w+)& ", line) is not None class TestConstRefInSubscription(TestSpec): From d66ad1ec4389c8fe2560980c4e5b5afca0dee5c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 06:18:18 +0200 Subject: [PATCH 41/96] TestConstRefInSubscriptionL Improve regex --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 41235c85f6d..f177766454d 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -345,7 +345,7 @@ def test_file(self, path : str, content) -> bool: words = arguments.split(", ") # test for arg in words: - if not re.search(r"[\w<>:]* ?const ?[\w<>:]*&", arg): + if not re.search(r"([\w>] const|const [\w<>:]+)&", arg): passed = False print_error(path, i + 1, self.name, f"Argument {arg} is not const&.") line_process = 0 From 5ca100b8cce9c173e06ac76b11a5a03dfa2fb60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 18:13:38 +0200 Subject: [PATCH 42/96] TestStdPrefix: Fix false positives in comments and strings --- Scripts/o2_linter.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index f177766454d..3509d1269d1 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -71,6 +71,14 @@ def is_comment_cpp(line: str) -> bool: return line.strip().startswith(("//", "/*")) +def remove_comment_cpp(line: str) -> str: + """Remove C++ comments from the end of a line.""" + for keyword in ("//", "/*"): + if keyword in line: + line = line[:line.find(keyword)] + return line + + class TestSpec(ABC): """Prototype of a test class""" name = "test-template" # short name of the test @@ -173,19 +181,27 @@ class TestStdPrefix(TestSpec): name = "std-prefix" message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] - prefix_bad = r"[^\w:\.]" + prefix_bad = r"[^\w:\.\"]" patterns = [r"vector<", r"array[<\{\(]", r"f?abs\(", r"sqrt\(", r"pow\(", r"min\(", r"max\(", r"log\(", r"exp\(", r"sin\(", r"cos\(", r"tan\(", r"atan\(", r"atan2\("] def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) for pattern in self.patterns: - if re.search(fr"{self.prefix_bad}{pattern}", line): + iterators = re.finditer(fr"{self.prefix_bad}{pattern}", line) + matches = [(it.start(), it.group()) for it in iterators] + if not matches: + continue + if not "\"" in line: # Found a match which cannot be inside a string. + return False + # Ignore matches inside strings. + for match in matches: + n_quotes_before = line.count("\"", 0, match[0]) # Count quotation marks before the match. + if n_quotes_before % 2: # If odd, we are inside a string and we should ignore this match. + continue + # We are not inside a string and this match is valid. return False - # occurrences = re.findall(pattern, line) - # if occurrences: - # print(occurrences) - # return False return True @@ -855,7 +871,7 @@ def main(): tests.append(TestNameFilePython()) tests.append(TestNameWorkflow()) - # PWGHF + # PWG-HF enable_pwghf = True if enable_pwghf: tests.append(TestHfConstAuto()) From c84c21ae64f5df111178964c8e4d4073ef6c0201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 19:57:07 +0200 Subject: [PATCH 43/96] Remove comments from lines before scanning --- Scripts/o2_linter.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 3509d1269d1..7e0c7e3a234 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -218,6 +218,7 @@ def test_line(self, line: str) -> bool: pattern = r"TMath|(U?(Int|Char|Short)|Double(32)?|Float(16)?|U?Long(64)?|Bool)_t" if is_comment_cpp(line): return True + line = remove_comment_cpp(line) return re.search(pattern, line) is None @@ -234,6 +235,7 @@ def test_line(self, line: str) -> bool: pattern = r"M_PI|TMath::(Two)?Pi" if is_comment_cpp(line): return True + line = remove_comment_cpp(line) return re.search(pattern, line) is None @@ -249,6 +251,7 @@ def file_matches(self, path: str) -> bool: def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) return not "TDatabasePDG" in line @@ -261,6 +264,7 @@ class TestPdgCode(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) if re.search(r"->(GetParticle|Mass)\([+-]?[0-9]+\)", line): return False match = re.search(r"[Pp][Dd][Gg][\w]* ={1,2} [+-]?([0-9]+);", line) @@ -280,6 +284,7 @@ class TestPdgMass(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) pattern_pdg_code = r"[+-]?(k[A-Z][a-zA-Z0-9]*|[0-9]+)" if re.search(fr"->GetParticle\({pattern_pdg_code}\)->Mass\(\)", line): return False @@ -301,6 +306,7 @@ def test_line(self, line: str) -> bool: pattern = r"^([Pp]rintf\(|(std::)?cout <)" if is_comment_cpp(line): return True + line = remove_comment_cpp(line) return re.search(pattern, line) is None @@ -313,6 +319,7 @@ class TestConstRefInForLoop(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) if not line.startswith("for (") or " : " not in line: return True line = line[:line.index(" : ")] # keep only the iterator part @@ -337,6 +344,8 @@ def test_file(self, path : str, content) -> bool: continue if self.is_disabled(line): continue + if "//" in line: # Remove comment. (Ignore /* to avoid truncating at /*parameter*/.) + line = line[:line.find("//")] if re.search(r"^void process[\w]*\(", line): line_process = (i + 1) i_closing = line.rfind(")") @@ -517,6 +526,7 @@ class TestNameConstant(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) words = line.split() if not "constexpr" in words or not "=" in words: return True @@ -565,6 +575,7 @@ def test_line(self, line: str) -> bool: return True if not line.startswith(f"{self.keyword} "): return True + line = remove_comment_cpp(line) # Extract object name. words = line.split() if not words[1].isalnum(): # "struct : ...", "enum { ..." @@ -675,6 +686,7 @@ def file_matches(self, path: str) -> bool: def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True + line = remove_comment_cpp(line) return not "auto const" in line @@ -692,6 +704,7 @@ def test_line(self, line: str) -> bool: return True if not line.startswith(("struct ", "class ")): return True + line = remove_comment_cpp(line) # Extract struct/class name. words = line.split() if not words[1].isalnum(): # "struct : ..." @@ -718,9 +731,10 @@ def test_file(self, path : str, content) -> bool: dic_struct = {} struct_name = "" for i, line in enumerate(content): - if line.strip().startswith("//"): + if is_comment_cpp(line): continue if line.startswith("struct "): # expecting no indentation + line = remove_comment_cpp(line) struct_name = line.strip().split()[1] dic_struct[struct_name] = {} continue From 7cecbdc575b06ade9331eab553438095f712974f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 20:12:54 +0200 Subject: [PATCH 44/96] TestConstRefInForLoop: Improve matching --- Scripts/o2_linter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 7e0c7e3a234..0993dd229a3 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -320,9 +320,9 @@ def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True line = remove_comment_cpp(line) - if not line.startswith("for (") or " : " not in line: + if not re.match("for \(.* :", line): return True - line = line[:line.index(" : ")] # keep only the iterator part + line = line[:line.index(" :")] # keep only the iterator part return re.search(r"(\w const|const \w+)& ", line) is not None From 2436c7ca0f69b8b2e487fae5f21133a4ab4c42ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 20:16:02 +0200 Subject: [PATCH 45/96] Replace find with index --- Scripts/o2_linter.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 0993dd229a3..3ea57eb9853 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -75,7 +75,7 @@ def remove_comment_cpp(line: str) -> str: """Remove C++ comments from the end of a line.""" for keyword in ("//", "/*"): if keyword in line: - line = line[:line.find(keyword)] + line = line[:line.index(keyword)] return line @@ -96,7 +96,7 @@ def is_disabled(self, line: str, prefix_comment="//") -> bool: for prefix in [prefix_comment, prefix_disable]: if not prefix in line: return False - line = line[(line.find(prefix) + len(prefix)):] # Strip away part before prefix. + line = line[(line.index(prefix) + len(prefix)):] # Strip away part before prefix. if self.name in line: return True return False @@ -345,11 +345,11 @@ def test_file(self, path : str, content) -> bool: if self.is_disabled(line): continue if "//" in line: # Remove comment. (Ignore /* to avoid truncating at /*parameter*/.) - line = line[:line.find("//")] + line = line[:line.index("//")] if re.search(r"^void process[\w]*\(", line): line_process = (i + 1) i_closing = line.rfind(")") - i_start = line.find("(") + 1 + i_start = line.index("(") + 1 i_end = i_closing if i_closing != -1 else len(line) arguments = line[i_start:i_end] # get arguments between parentheses n_parens_opened = line.count("(") - line.count(")") @@ -444,7 +444,7 @@ def test_line(self, line: str) -> bool: # For functions, stripping after "(" is enough but this way we also identify many declarations of variables. for keyword in ("(", "{", ";", " = ", "//", "/*"): if keyword in line: - line = line[:line.find(keyword)] + line = line[:line.index(keyword)] # Check the words. words = line.split() @@ -480,7 +480,7 @@ def test_line(self, line: str) -> bool: # Extract function/variable name. if "[" in funval_name: # Remove brackets for arrays. - funval_name = funval_name[:funval_name.find("[")] + funval_name = funval_name[:funval_name.index("[")] if "::" in funval_name: # Remove the class prefix for methods. funval_name = funval_name.split("::")[-1] @@ -537,7 +537,7 @@ def test_line(self, line: str) -> bool: opens_brackets = ["[" in w for w in words] constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" if "[" in constant_name: # Remove brackets for arrays. - constant_name = constant_name[:constant_name.find("[")] + constant_name = constant_name[:constant_name.index("[")] if "::" in constant_name: # Remove the class prefix for methods. constant_name = constant_name.split("::")[-1] # The actual test comes here. From bdbb4547f6ac05fd8d2be59d5c58bac260670d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 21:49:14 +0200 Subject: [PATCH 46/96] TestNameFileCpp: Add more specific rules --- Scripts/o2_linter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 3ea57eb9853..6ca3844330e 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -392,7 +392,7 @@ def test_file(self, path : str, content) -> bool: passed = False doc_items = [] doc_items.append({"keyword": "file", "pattern": fr"{os.path.basename(path)}$", "found": False}) - doc_items.append({"keyword": "brief", "pattern": r"\w.* \w", "found": False}) + doc_items.append({"keyword": "brief", "pattern": r"\w.* \w", "found": False}) # at least two words doc_items.append({"keyword": "author", "pattern": r"[\w]+", "found": False}) doc_prefix = "///" n_lines_copyright = 11 @@ -617,6 +617,15 @@ class TestNameFileCpp(TestSpec): def test_file(self, path : str, content) -> bool: file_name = os.path.basename(path) + # workflow file + if re.search("(TableProducer|Tasks)/.*\.cxx", path): + return is_lower_camel_case(file_name) + # data model file + if "DataModel/" in path: + return is_upper_camel_case(file_name) + # utility file + if re.search("[Uu]til(ity|ities|s)?", file_name): + return is_lower_camel_case(file_name) return is_camel_case(file_name) From f0efb31a10ff5360bbd7b5f4f0df52b051505ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 23 Jul 2024 21:53:38 +0200 Subject: [PATCH 47/96] Rephrase error messages. --- Scripts/o2_linter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 6ca3844330e..b87cd1c0c55 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -501,7 +501,7 @@ def test_line(self, line: str) -> bool: class TestNameMacro(TestSpec): """Test macro names.""" name = "name/macro" - message = "Use SCREAMING_SNAKE_CASE for macro names." + message = "Use SCREAMING_SNAKE_CASE for names of macros." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -520,7 +520,7 @@ def test_line(self, line: str) -> bool: class TestNameConstant(TestSpec): """Test constexpr constant names.""" name = "name/constexpr-constant" - message = "Use UpperCamelCase for constexpr constant names. Names of special constants may be prefixed with \"k\"." + message = "Use UpperCamelCase for names of constexpr constants. Names of special constants may be prefixed with \"k\"." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -676,7 +676,7 @@ def test_file(self, path : str, content) -> bool: # Compare the actual and expected file names. if expected_workflow_file_name != workflow_file_name: passed = False - print_error(path, i + 1, self.name, f"Workflow name {workflow_name} does not match the workflow file name {workflow_file_name} (expected {expected_workflow_file_name}).") + print_error(path, i + 1, self.name, f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. (Matches {expected_workflow_file_name}.)") return passed From 9328d0adf062672c765eb34cf423a8d48bc00c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Wed, 24 Jul 2024 00:29:46 +0200 Subject: [PATCH 48/96] Add TestNameColumn, TestNameTable --- Scripts/o2_linter.py | 61 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index b87cd1c0c55..cd44da00df7 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -76,7 +76,7 @@ def remove_comment_cpp(line: str) -> str: for keyword in ("//", "/*"): if keyword in line: line = line[:line.index(keyword)] - return line + return line.strip() class TestSpec(ABC): @@ -546,6 +546,63 @@ def test_line(self, line: str) -> bool: return is_upper_camel_case(constant_name) +class TestNameColumn(TestSpec): + """Test names of O2 columns.""" + name = "name/o2-column" + message = "Use UpperCamelCase for names of O2 columns and matching lowerCamelCase names for their getters." + suffixes = [".h", ".cxx"] + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + if not (match := re.match("DECLARE(_[A-Z]+)*_COLUMN(_[A-Z]+)*\(", line)): + return True + # Extract names of the column type and getter. + line = remove_comment_cpp(line) + line = line[len(match.group()):].strip() # Extract part after "(". + if not (match := re.match("([^,]+), ([^,\) ]+)", line)): + print(f"Failed to extract column type and getter from \"{line}\".") + return False + column_type_name = match.group(1) + column_getter_name = match.group(2) + # print(f"Got \"{column_type_name}\" \"{column_getter_name}\"") + # return True + if column_type_name[0] == "_": # probably a macro variable + return True + # The actual test comes here. + if not is_upper_camel_case(column_type_name): + return False + if not is_lower_camel_case(column_getter_name): + return False + return f"{column_type_name[0].lower()}{column_type_name[1:]}" == column_getter_name + + +class TestNameTable(TestSpec): + """Test names of O2 tables.""" + name = "name/o2-table" + message = "Use UpperCamelCase for names of O2 tables." + suffixes = [".h", ".cxx"] + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + if not (match := re.match("DECLARE(_[A-Z]+)*_TABLES?(_[A-Z]+)*\(", line)): + return True + # Extract names of the column type and getter. + line = remove_comment_cpp(line) + line = line[len(match.group()):].strip() # Extract part after "(". + if not (match := re.match("([^,\) ]+)", line)): + print(f"Failed to extract table type from \"{line}\".") + return False + table_type_name = match.group(1) + # print(f"Got \"{table_type_name}\"") + # return True + if table_type_name[0] == "_": # probably a macro variable + return True + # The actual test comes here. + return is_upper_camel_case(table_type_name) + + class TestNameNamespace(TestSpec): """Test names of namespaces.""" name = "name/namespace" @@ -886,6 +943,8 @@ def main(): tests.append(TestNameFunctionVariable()) tests.append(TestNameMacro()) tests.append(TestNameConstant()) + tests.append(TestNameColumn()) + tests.append(TestNameTable()) tests.append(TestNameNamespace()) tests.append(TestNameEnum()) tests.append(TestNameClass()) From 49e05cc718e5012cedde7faa9c5b202356e5d7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Wed, 24 Jul 2024 02:57:14 +0200 Subject: [PATCH 49/96] Fixes --- .github/workflows/o2-linter.yml | 7 +++-- Scripts/o2_linter.py | 49 +++++++++++++++++---------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index f8c5c712982..c8ced2d0e10 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -1,5 +1,5 @@ --- -# Find code issues in modified text files +# Find issues in O2 code name: O2 linter 'on': [pull_request, push] @@ -23,6 +23,7 @@ jobs: - name: Run tests run: | # Diff against the common ancestor of the source branch and the target branch. - files=$(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) + readarray -t files < <(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) [ ${{ github.event_name }} == 'pull_request' ] && options="-g" - python Scripts/o2_linter.py $options $files + # shellcheck disable=SC2086 # Ignore unquoted options. + python Scripts/o2_linter.py $options "${files[@]}" diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index cd44da00df7..98217794569 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -22,7 +22,7 @@ import os import re import sys - +from typing import Union github_mode = False # GitHub mode @@ -56,14 +56,14 @@ def is_screaming_snake_case(name: str) -> bool: return name.isupper() and not "-" in name and not " " in name -def print_error(path: str, line: int, title: str, message: str) -> str: +def print_error(path: str, line: Union[int, None], title: str, message: str): """Format and print error message.""" # return # Use to suppress error message when counting speed. str_line = "" if line is None else f"{line}:" print(f"{path}:{str_line} {message} [{title}]") # terminal format if github_mode: str_line = "" if line is None else f",line={line}" - print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub action format + print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub annotation format def is_comment_cpp(line: str) -> bool: @@ -83,7 +83,7 @@ class TestSpec(ABC): """Prototype of a test class""" name = "test-template" # short name of the test message = "Test failed" # error message - suffixes = [] # suffixes of files to test + suffixes: "list[str]" = [] # suffixes of files to test per_line = True # Test lines separately one by one. def file_matches(self, path: str) -> bool: @@ -320,7 +320,7 @@ def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True line = remove_comment_cpp(line) - if not re.match("for \(.* :", line): + if not re.match(r"for \(.* :", line): return True line = line[:line.index(" :")] # keep only the iterator part return re.search(r"(\w const|const \w+)& ", line) is not None @@ -416,7 +416,7 @@ def test_file(self, path : str, content) -> bool: if not passed: for item in doc_items: if not item["found"]: - print_error(path, last_doc_line, self.name, f"Documentation for \{item['keyword']} is missing, incorrect or misplaced.") + print_error(path, last_doc_line, self.name, f"Documentation for \\{item['keyword']} is missing, incorrect or misplaced.") return passed @@ -555,12 +555,12 @@ class TestNameColumn(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - if not (match := re.match("DECLARE(_[A-Z]+)*_COLUMN(_[A-Z]+)*\(", line)): + if not (match := re.match(r"DECLARE(_[A-Z]+)*_COLUMN(_[A-Z]+)*\(", line)): return True # Extract names of the column type and getter. line = remove_comment_cpp(line) line = line[len(match.group()):].strip() # Extract part after "(". - if not (match := re.match("([^,]+), ([^,\) ]+)", line)): + if not (match := re.match(r"([^,]+), ([^,\) ]+)", line)): print(f"Failed to extract column type and getter from \"{line}\".") return False column_type_name = match.group(1) @@ -586,12 +586,12 @@ class TestNameTable(TestSpec): def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True - if not (match := re.match("DECLARE(_[A-Z]+)*_TABLES?(_[A-Z]+)*\(", line)): + if not (match := re.match(r"DECLARE(_[A-Z]+)*_TABLES?(_[A-Z]+)*\(", line)): return True # Extract names of the column type and getter. line = remove_comment_cpp(line) line = line[len(match.group()):].strip() # Extract part after "(". - if not (match := re.match("([^,\) ]+)", line)): + if not (match := re.match(r"([^,\) ]+)", line)): print(f"Failed to extract table type from \"{line}\".") return False table_type_name = match.group(1) @@ -675,13 +675,13 @@ class TestNameFileCpp(TestSpec): def test_file(self, path : str, content) -> bool: file_name = os.path.basename(path) # workflow file - if re.search("(TableProducer|Tasks)/.*\.cxx", path): + if re.search(r"(TableProducer|Tasks)/.*\.cxx", path): return is_lower_camel_case(file_name) - # data model file + # data model file if "DataModel/" in path: return is_upper_camel_case(file_name) - # utility file - if re.search("[Uu]til(ity|ities|s)?", file_name): + # utility file + if re.search(r"[Uu]til(ity|ities|s)?", file_name): return is_lower_camel_case(file_name) return is_camel_case(file_name) @@ -783,7 +783,7 @@ def test_line(self, line: str) -> bool: class TestHfStructMembers(TestSpec): """PWGHF: Test order of struct members. Caveat: Does not see Configurables in ConfigurableGroup.""" - name = "pwghf/struct member order" + name = "pwghf/struct-member-order" message = "Declare struct members in the conventional order. See the PWGHF coding guidelines." suffixes = [".cxx"] per_line = False @@ -794,7 +794,7 @@ def file_matches(self, path: str) -> bool: def test_file(self, path : str, content) -> bool: passed = True - dic_struct = {} + dic_struct:dict[str, dict] = {} struct_name = "" for i, line in enumerate(content): if is_comment_cpp(line): @@ -867,7 +867,7 @@ def test_file(self, path : str, content) -> bool: class TestHfNameConfigurable(TestSpec): """PWGHF: Test names of configurables.""" name = "pwghf/name/configurable" - message = "Use lowerCamelCase for Configurable names and use the same name for the struct member as for the JSON string." + message = "Use lowerCamelCase for names of configurables and use the same name for the struct member as for the JSON string." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: @@ -968,28 +968,31 @@ def main(): # Report overview before running. print(f"Testing {len(args.paths)} files.") + # print(args.paths) print("Enabled tests:", test_names) print("Suffixes of tested files:", suffixes) + # print(f"Github annotations: {github_mode}.") # Test files. for path in args.paths: + # print(f"Processing path \"{path}\".") # Skip not tested files. if not path.endswith(suffixes): + # print(f"Skipping path \"{path}\".") continue try: with open(path, "r") as file: content = file.readlines() for test in tests: - if not test.run(path, content): + result = test.run(path, content) + if not result: passed = False - # if not passed: - # print(f"File {path} failed the test {test.name}.") - # print(f"Test {'passed' if passed else 'failed'}.") + # print(f"File \"{path}\" {'passed' if result else 'failed'} the test {test.name}.") except IOError: - print(f"Failed to open file {path}.") + print(f"Failed to open file \"{path}\".") sys.exit(1) - # Report result. + # Report global result. if passed: print("All tests passed.") else: From 4d0fda40c6e374202d107d71268258e4c3d4e4c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 25 Jul 2024 04:45:00 +0200 Subject: [PATCH 50/96] Add TestNameDevice --- Scripts/o2_linter.py | 91 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 98217794569..aff3c4823c5 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -56,6 +56,17 @@ def is_screaming_snake_case(name: str) -> bool: return name.isupper() and not "-" in name and not " " in name +def kebab_case_to_camel_case_u(line: str) -> str: + """Convert kebab-case string to UpperCamelCase string.""" + return "".join([w.title() if w[0].isnumeric() else w.capitalize() for w in line.split("-")]) + + +def kebab_case_to_camel_case_l(line: str) -> str: + """Convert kebab-case string to lowerCamelCase string.""" + new_line = kebab_case_to_camel_case_u(line) + return f"{new_line[0].lower()}{new_line[1:]}" # start with lowercase letter + + def print_error(path: str, line: Union[int, None], title: str, message: str): """Format and print error message.""" # return # Use to suppress error message when counting speed. @@ -728,8 +739,7 @@ def test_file(self, path : str, content) -> bool: continue workflow_file_name = os.path.basename(words[1]) # the actual file name # Generate the file name matching the workflow name. - expected_workflow_file_name = "".join([w.title() if w[0].isnumeric() else w.capitalize() for w in workflow_name.split("-")]) + ".cxx" - expected_workflow_file_name = f"{expected_workflow_file_name[0].lower()}{expected_workflow_file_name[1:]}" # start with lowercase letter + expected_workflow_file_name = kebab_case_to_camel_case_l(workflow_name) + ".cxx" # Compare the actual and expected file names. if expected_workflow_file_name != workflow_file_name: passed = False @@ -737,6 +747,82 @@ def test_file(self, path : str, content) -> bool: return passed +class TestNameDevice(TestSpec): + """Test names of devices.""" + name = "name/o2-device" + message = "Specify device name only when it cannot be derived from the struct name. Only append to the default name." + suffixes = [".cxx"] + per_line = False + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) + + def test_file(self, path : str, content) -> bool: + is_inside_define = False + is_inside_adapt = False + struct_name = "" + n_parens_opened = 0 # number of opened parentheses + passed = True + for i, line in enumerate(content): + if not line.strip(): + continue + if self.is_disabled(line, "#"): + continue + if is_comment_cpp(line): + continue + line = remove_comment_cpp(line) + # Wait for defineDataProcessing. + if not is_inside_define: + if not re.match(r"(o2::)?(framework::)?WorkflowSpec defineDataProcessing\(", line): + continue + # print(f"{i + 1}: Entering define.") + is_inside_define = True + # Return at the end of defineDataProcessing. + if is_inside_define and line[0] == "}": + # print(f"{i + 1}: Exiting define.") + break + # Wait for adaptAnalysisTask. + if not is_inside_adapt: + if (index := line.find("adaptAnalysisTask<")) == -1: + continue + # print(f"{i + 1}: Entering adapt.") + is_inside_adapt = True + line = line[(index + len("adaptAnalysisTask<")):] + # Extract struct name. + if not (match := re.match(r"([^<>]+)", line)): + print_error(path, i + 1, self.name, f"Failed to extract struct name from \"{line}\".") + return False + struct_name = match.group(1) + # print(f"{i + 1}: Got struct name {struct_name}") + line = line[(line.index(struct_name) + len(struct_name)):] + if is_inside_adapt: + n_parens_opened += line.count("(") - line.count(")") + # print(f"{i + 1}: {n_parens_opened} opened parens") + if n_parens_opened <= 0: + # print(f"{i + 1}: Exiting adapt.") + is_inside_adapt = False + # Find device name. + if not "TaskName{" in line: + continue + passed = False + # Extract device name. + if not (match := re.search(r"TaskName\{\"([^\}]+)\"\}", line)): + print_error(path, i + 1, self.name, f"Failed to extract device name from \"{line}\".") + return False + device_name = match.group(1) + # print(f"{i + 1}: Got struct \"{struct_name}\" with device name \"{device_name}\".") + # Test device name. + expected_struct_name = kebab_case_to_camel_case_u(device_name) + if struct_name == expected_struct_name: + print_error(path, i + 1, self.name, f"Specified device name {device_name} matches exactly the struct name {struct_name}. TaskName seems unnecessary.") + # if template, check that device name is extension of struct name + elif expected_struct_name.startswith(struct_name): + print_error(path, i + 1, self.name, f"Specified device name {device_name} is an extension of the struct name {struct_name}. Is the struct templated?") + else: + print_error(path, i + 1, self.name, f"Specified device name {device_name} does not match the struct name {struct_name}. (Matching {expected_struct_name})") + return passed + + # PWG-HF @@ -952,6 +1038,7 @@ def main(): tests.append(TestNameFileCpp()) tests.append(TestNameFilePython()) tests.append(TestNameWorkflow()) + tests.append(TestNameDevice()) # PWG-HF enable_pwghf = True From b164618bdd1bac76b57638ca072d39f0aa946d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 25 Jul 2024 05:11:18 +0200 Subject: [PATCH 51/96] TestNameDevice: Support templated structs --- Scripts/o2_linter.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index aff3c4823c5..9647a5f9840 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -761,6 +761,7 @@ def test_file(self, path : str, content) -> bool: is_inside_define = False is_inside_adapt = False struct_name = "" + struct_templated = False # Is the struct templated? n_parens_opened = 0 # number of opened parentheses passed = True for i, line in enumerate(content): @@ -789,10 +790,14 @@ def test_file(self, path : str, content) -> bool: is_inside_adapt = True line = line[(index + len("adaptAnalysisTask<")):] # Extract struct name. - if not (match := re.match(r"([^<>]+)", line)): + if not (match := re.match(r"([^>]+)", line)): print_error(path, i + 1, self.name, f"Failed to extract struct name from \"{line}\".") return False struct_name = match.group(1) + if (index := struct_name.find("<")) > -1: + struct_templated = True + # print(f"{i + 1}: Got templated struct name {struct_name}") + struct_name = struct_name[:index] # print(f"{i + 1}: Got struct name {struct_name}") line = line[(line.index(struct_name) + len(struct_name)):] if is_inside_adapt: @@ -817,7 +822,7 @@ def test_file(self, path : str, content) -> bool: print_error(path, i + 1, self.name, f"Specified device name {device_name} matches exactly the struct name {struct_name}. TaskName seems unnecessary.") # if template, check that device name is extension of struct name elif expected_struct_name.startswith(struct_name): - print_error(path, i + 1, self.name, f"Specified device name {device_name} is an extension of the struct name {struct_name}. Is the struct templated?") + print_error(path, i + 1, self.name, f"Specified device name {device_name} is an extension of the struct name {struct_name}. Is the struct templated? {struct_templated}") else: print_error(path, i + 1, self.name, f"Specified device name {device_name} does not match the struct name {struct_name}. (Matching {expected_struct_name})") return passed From 4cfb3e9e7c7a187a8f86d6a2b24d8a5b672ae41f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 25 Jul 2024 20:08:01 +0200 Subject: [PATCH 52/96] TestNameDevice -> TestNameTask: Improve treatment of special cases. --- Scripts/o2_linter.py | 68 +++++++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 9647a5f9840..cfa5fde13e6 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -67,6 +67,20 @@ def kebab_case_to_camel_case_l(line: str) -> str: return f"{new_line[0].lower()}{new_line[1:]}" # start with lowercase letter +def camel_case_to_kebab_case(line: str) -> str: + """Convert CamelCase string to kebab-case string. + As done in O2/Framework/Foundation/include/Framework/TypeIdHelpers.h:type_to_task_name + """ + if not line.strip(): + return line + new_line = [] + for i, c in enumerate(line): + if i > 0 and c.isupper() and line[i - 1] != "-": + new_line.append("-") + new_line.append(c.lower()) + return "".join(new_line) + + def print_error(path: str, line: Union[int, None], title: str, message: str): """Format and print error message.""" # return # Use to suppress error message when counting speed. @@ -747,10 +761,10 @@ def test_file(self, path : str, content) -> bool: return passed -class TestNameDevice(TestSpec): - """Test names of devices.""" - name = "name/o2-device" - message = "Specify device name only when it cannot be derived from the struct name. Only append to the default name." +class TestNameTask(TestSpec): + """Test explicit task names.""" + name = "name/o2-task" + message = "Specify task name only when it cannot be derived from the struct name. Only append to the default name." suffixes = [".cxx"] per_line = False @@ -758,8 +772,8 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) def test_file(self, path : str, content) -> bool: - is_inside_define = False - is_inside_adapt = False + is_inside_define = False # Are we inside defineDataProcessing? + is_inside_adapt = False # Are we inside adaptAnalysisTask? struct_name = "" struct_templated = False # Is the struct templated? n_parens_opened = 0 # number of opened parentheses @@ -806,25 +820,39 @@ def test_file(self, path : str, content) -> bool: if n_parens_opened <= 0: # print(f"{i + 1}: Exiting adapt.") is_inside_adapt = False - # Find device name. + # Find explicit task name. if not "TaskName{" in line: continue passed = False - # Extract device name. + # Extract explicit task name. if not (match := re.search(r"TaskName\{\"([^\}]+)\"\}", line)): - print_error(path, i + 1, self.name, f"Failed to extract device name from \"{line}\".") + print_error(path, i + 1, self.name, f"Failed to extract explicit task name from \"{line}\".") return False - device_name = match.group(1) - # print(f"{i + 1}: Got struct \"{struct_name}\" with device name \"{device_name}\".") - # Test device name. - expected_struct_name = kebab_case_to_camel_case_u(device_name) - if struct_name == expected_struct_name: - print_error(path, i + 1, self.name, f"Specified device name {device_name} matches exactly the struct name {struct_name}. TaskName seems unnecessary.") - # if template, check that device name is extension of struct name - elif expected_struct_name.startswith(struct_name): - print_error(path, i + 1, self.name, f"Specified device name {device_name} is an extension of the struct name {struct_name}. Is the struct templated? {struct_templated}") + task_name = match.group(1) + # print(f"{i + 1}: Got struct \"{struct_name}\" with task name \"{task_name}\".") + # Test explicit task name. + device_name_from_struct_name = camel_case_to_kebab_case(struct_name) + device_name_from_task_name = camel_case_to_kebab_case(task_name) + struct_name_from_device_name = kebab_case_to_camel_case_u(device_name_from_task_name) + if device_name_from_struct_name == device_name_from_task_name: + # If the task name results in the same device name as the struct name would, TaskName is redundant and should be removed. + print_error(path, i + 1, self.name, f"Specified task name {task_name} and the struct name {struct_name} produce the same device name {device_name_from_struct_name}. TaskName is redundant.") + passed = False + elif device_name_from_struct_name.replace("-", "") == device_name_from_task_name.replace("-", ""): + # If the device names generated from the task name and from the struct name differ in hyphenation, capitalisation of the struct name should be fixed and TaskName should be removed. (special cases: alice3-, -2prong) + print_error(path, i + 1, self.name, f"Device names {device_name_from_task_name}, {device_name_from_struct_name} generated from the specified task name {task_name} and from the struct name {struct_name} differ in hyphenation. Consider fixing capitalisation of the struct name to {struct_name_from_device_name} and removing TaskName.") + passed = False + elif device_name_from_task_name.startswith(device_name_from_struct_name): + # If the device name generated from the task name is an extension of the device name generated from the struct name, accept it if the struct is templated. If the struct is not templated, extension is acceptable if adaptTask is called multiple times for the same struct. + if struct_templated: + print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} and the struct is templated. All good") + else: + print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} but the struct is not templated. Is it adapted multiple times?") + passed = False else: - print_error(path, i + 1, self.name, f"Specified device name {device_name} does not match the struct name {struct_name}. (Matching {expected_struct_name})") + # Other cases should be rejected. + print_error(path, i + 1, self.name, f"Specified task name {task_name} produces device name {device_name_from_task_name} which does not match the device name {device_name_from_struct_name} from the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})") + passed = False return passed @@ -1043,7 +1071,7 @@ def main(): tests.append(TestNameFileCpp()) tests.append(TestNameFilePython()) tests.append(TestNameWorkflow()) - tests.append(TestNameDevice()) + tests.append(TestNameTask()) # PWG-HF enable_pwghf = True From bb4c0a64aeb4f2612a4a9c0159873d18ce0e5995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 25 Jul 2024 21:49:43 +0200 Subject: [PATCH 53/96] Minor fixes --- Scripts/o2_linter.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index cfa5fde13e6..a856be0d941 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -362,7 +362,7 @@ def test_file(self, path : str, content) -> bool: passed = True n_parens_opened = 0 # number of opened parentheses arguments = "" # process function arguments - line_process = 0 + line_process = 0 # line number of the process function for i, line in enumerate(content): line = line.strip() if is_comment_cpp(line): @@ -372,7 +372,7 @@ def test_file(self, path : str, content) -> bool: if "//" in line: # Remove comment. (Ignore /* to avoid truncating at /*parameter*/.) line = line[:line.index("//")] if re.search(r"^void process[\w]*\(", line): - line_process = (i + 1) + line_process = i + 1 i_closing = line.rfind(")") i_start = line.index("(") + 1 i_end = i_closing if i_closing != -1 else len(line) @@ -762,15 +762,13 @@ def test_file(self, path : str, content) -> bool: class TestNameTask(TestSpec): - """Test explicit task names.""" + """Test explicit task names. + Detect usage of TaskName, check whether it is needed and whether the task name matches the struct name.""" name = "name/o2-task" message = "Specify task name only when it cannot be derived from the struct name. Only append to the default name." suffixes = [".cxx"] per_line = False - def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) - def test_file(self, path : str, content) -> bool: is_inside_define = False # Are we inside defineDataProcessing? is_inside_adapt = False # Are we inside adaptAnalysisTask? @@ -781,7 +779,7 @@ def test_file(self, path : str, content) -> bool: for i, line in enumerate(content): if not line.strip(): continue - if self.is_disabled(line, "#"): + if self.is_disabled(line): continue if is_comment_cpp(line): continue @@ -831,24 +829,24 @@ def test_file(self, path : str, content) -> bool: task_name = match.group(1) # print(f"{i + 1}: Got struct \"{struct_name}\" with task name \"{task_name}\".") # Test explicit task name. - device_name_from_struct_name = camel_case_to_kebab_case(struct_name) - device_name_from_task_name = camel_case_to_kebab_case(task_name) - struct_name_from_device_name = kebab_case_to_camel_case_u(device_name_from_task_name) + device_name_from_struct_name = camel_case_to_kebab_case(struct_name) # default device name, in absence of TaskName + device_name_from_task_name = camel_case_to_kebab_case(task_name) # actual device name, generated from TaskName + struct_name_from_device_name = kebab_case_to_camel_case_u(device_name_from_task_name) # struct name matching the TaskName if device_name_from_struct_name == device_name_from_task_name: # If the task name results in the same device name as the struct name would, TaskName is redundant and should be removed. print_error(path, i + 1, self.name, f"Specified task name {task_name} and the struct name {struct_name} produce the same device name {device_name_from_struct_name}. TaskName is redundant.") passed = False elif device_name_from_struct_name.replace("-", "") == device_name_from_task_name.replace("-", ""): # If the device names generated from the task name and from the struct name differ in hyphenation, capitalisation of the struct name should be fixed and TaskName should be removed. (special cases: alice3-, -2prong) - print_error(path, i + 1, self.name, f"Device names {device_name_from_task_name}, {device_name_from_struct_name} generated from the specified task name {task_name} and from the struct name {struct_name} differ in hyphenation. Consider fixing capitalisation of the struct name to {struct_name_from_device_name} and removing TaskName.") + print_error(path, i + 1, self.name, f"Device names {device_name_from_task_name} and {device_name_from_struct_name} generated from the specified task name {task_name} and from the struct name {struct_name}, respectively, differ in hyphenation. Consider fixing capitalisation of the struct name to {struct_name_from_device_name} and removing TaskName.") passed = False elif device_name_from_task_name.startswith(device_name_from_struct_name): - # If the device name generated from the task name is an extension of the device name generated from the struct name, accept it if the struct is templated. If the struct is not templated, extension is acceptable if adaptTask is called multiple times for the same struct. - if struct_templated: - print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} and the struct is templated. All good") - else: + # If the device name generated from the task name is an extension of the device name generated from the struct name, accept it if the struct is templated. If the struct is not templated, extension is acceptable if adaptAnalysisTask is called multiple times for the same struct. + if not struct_templated: print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} but the struct is not templated. Is it adapted multiple times?") passed = False + # else: + # print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} and the struct is templated. All good") else: # Other cases should be rejected. print_error(path, i + 1, self.name, f"Specified task name {task_name} produces device name {device_name_from_task_name} which does not match the device name {device_name_from_struct_name} from the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})") From 00bf0e9a140afc7e0bf39be896fe92dfeb130215 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 26 Jul 2024 15:56:46 +0200 Subject: [PATCH 54/96] Add more protections and detections of special cases. Add protection against empty string in is_case functions. TestConstRefInSubscription: Add support for nested <> blocks in argument types (block_ranges). TestNameFunctionVariable, TestNameConstant: Add protection against multi-line declarations of arrays. TestNameConstant: Add protection against constant named k. TestNameNamespace: Add protection against anonymous namespaces. TestNameUpperCamelCase: Do not remove comment (keep the space ). Detect enum struct. --- .github/workflows/o2-linter.yml | 2 +- Scripts/o2_linter.py | 76 ++++++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index c8ced2d0e10..f6127db25cc 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -22,7 +22,7 @@ jobs: fetch-depth: 0 # needed to get the full history - name: Run tests run: | - # Diff against the common ancestor of the source branch and the target branch. + # Diff against the common ancestor of the source branch and the main branch. readarray -t files < <(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) [ ${{ github.event_name }} == 'pull_request' ] && options="-g" # shellcheck disable=SC2086 # Ignore unquoted options. diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index a856be0d941..1cfb2f613ca 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -33,26 +33,36 @@ def is_camel_case(name: str) -> bool: def is_upper_camel_case(name: str) -> bool: """ForExample""" + if not name: + return False return name[0].isupper() and is_camel_case(name) def is_lower_camel_case(name: str) -> bool: """forExample""" + if not name: + return False return name[0].islower() and is_camel_case(name) def is_kebab_case(name: str) -> bool: """for-example""" + if not name: + return False return name.islower() and not "_" in name and not " " in name def is_snake_case(name: str) -> bool: """for_example""" + if not name: + return False return name.islower() and not "-" in name and not " " in name def is_screaming_snake_case(name: str) -> bool: """FOR_EXAMPLE""" + if not name: + return False return name.isupper() and not "-" in name and not " " in name @@ -104,6 +114,43 @@ def remove_comment_cpp(line: str) -> str: return line.strip() +def block_ranges(line: str, c_open: str, c_close: str) -> "list[list[int]]": + """Get list of index ranges of longest blocks opened with str_open and closed with str_close.""" + # print(f"Looking for {c_open}{c_close} blocks in \"{line}\".") + # print(line) + list_ranges = [] + if not all((line, len(c_open) == 1, len(c_close) == 1)): + return list_ranges + def direction(c:str) -> int: + if c == c_open: + return 1 + if c == c_close: + return -1 + return 0 + list_levels = [] # list of block levels (net number of opened blocks) + level = 0 # current block level (sum or previous directions) + for c in line: + list_levels.append(level := level + direction(c)) + level_min = min(list_levels) # minimum level (!= 0 if there are opened blocks) + # Look for openings (level_min + 1) and closings (level_min). + index_start = -1 + is_opened = False + # print(list_levels) + for i, l in enumerate(list_levels): + if not is_opened and l > level_min: + is_opened = True + index_start = i + # print(f"Opening at {i}") + elif is_opened and (l == level_min or i == len(list_levels) -1): + is_opened = False + list_ranges.append([index_start, i]) + # print(f"Closing at {i}") + # print(f"block_ranges: Found {len(list_ranges)} blocks: {list_ranges}.") + if is_opened: + print("block_ranges: Block left opened.") + return list_ranges + + class TestSpec(ABC): """Prototype of a test class""" name = "test-template" # short name of the test @@ -385,15 +432,16 @@ def test_file(self, path : str, content) -> bool: arguments += " " + line[i_start:i_end] # get arguments between parentheses n_parens_opened += line.count("(") - line.count(")") if line_process > 0 and n_parens_opened == 0: - # process arguments - # sanitise template arguments - template_args = re.findall(r"<[\w:, ]*>", arguments) - if template_args: - for arg in template_args: - if ", " in arg: - arguments = arguments.replace(arg, arg.replace(", ", ":")) + # Process arguments. + # Sanitise arguments with spaces between <>. + for start, end in block_ranges(arguments, "<", ">"): + arg = arguments[start:(end + 1)] + # print(f"Found argument \"{arg}\" in [{start}, {end}]") + if ", " in arg: + arguments = arguments.replace(arg, arg.replace(", ", "__")) + # Extract arguments. words = arguments.split(", ") - # test + # Test each argument. for arg in words: if not re.search(r"([\w>] const|const [\w<>:]+)&", arg): passed = False @@ -454,6 +502,7 @@ class TestNameFunctionVariable(TestSpec): Might report false positives. Does not detect multiple variable declarations, i.e. "type name1, name2;" Does not detect function arguments on the same line as the function declaration. + Does not detect multi-line declarations. Does not check capitalisation for constexpr because of special rules for constants. See TestNameConstant. """ name = "name/function-variable" @@ -492,6 +541,8 @@ def test_line(self, line: str) -> bool: funval_name = words[-1] # expecting the name in the last word if funval_name.endswith("]") and "[" not in funval_name: # it's an array and we do not have the name before "[" here opens_brackets = ["[" in w for w in words] + if not any(opens_brackets): # The opening "[" is not on this line. We have to give up. + return True index_name = opens_brackets.index(True) # the name is in the first element with "[" funval_name = words[index_name] words = words[:(index_name + 1)] # Strip away words after the name. @@ -560,13 +611,15 @@ def test_line(self, line: str) -> bool: constant_name = words[-1] # last word before "=" if constant_name.endswith("]") and "[" not in constant_name: # it's an array and we do not have the name before "[" here opens_brackets = ["[" in w for w in words] + if not any(opens_brackets): # The opening "[" is not on this line. We have to give up. + return True constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" if "[" in constant_name: # Remove brackets for arrays. constant_name = constant_name[:constant_name.index("[")] if "::" in constant_name: # Remove the class prefix for methods. constant_name = constant_name.split("::")[-1] # The actual test comes here. - if constant_name.startswith("k"): # exception for special constants + if constant_name.startswith("k") and len(constant_name) > 1: # exception for special constants constant_name = constant_name[1:] # test the name without "k" return is_upper_camel_case(constant_name) @@ -641,6 +694,8 @@ def test_line(self, line: str) -> bool: return True # Extract namespace name. namespace_name = line.split()[1] + if namespace_name == "{": # ignore anonymous namespaces + return True # The actual test comes here. return is_snake_case(namespace_name) @@ -657,13 +712,12 @@ def test_line(self, line: str) -> bool: return True if not line.startswith(f"{self.keyword} "): return True - line = remove_comment_cpp(line) # Extract object name. words = line.split() if not words[1].isalnum(): # "struct : ...", "enum { ..." return True object_name = words[1] - if object_name == "class" and len(words) > 2: # enum class ... + if object_name in ("class", "struct") and len(words) > 2: # enum class ... or enum struct object_name = words[2] # The actual test comes here. return is_upper_camel_case(object_name) From edb666efb7773a3c53502fadd4bee408f24e3174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 26 Jul 2024 16:36:32 +0200 Subject: [PATCH 55/96] Add TestWorkflowOptions --- Scripts/o2_linter.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 1cfb2f613ca..c9297b66d63 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -450,6 +450,39 @@ def test_file(self, path : str, content) -> bool: return passed +class TestWorkflowOptions(TestSpec): + """Detect usage of workflow options in defineDataProcessing. (Not supported on AliHyperloop.)""" + name = "o2-workflow-options" + message = "Do not use workflow options to customise workflow topology composition in defineDataProcessing. Use process function switches or metadata instead." + suffixes = [".cxx"] + per_line = False + + def test_file(self, path : str, content) -> bool: + is_inside_define = False # Are we inside defineDataProcessing? + for i, line in enumerate(content): + if not line.strip(): + continue + if self.is_disabled(line): + continue + if is_comment_cpp(line): + continue + line = remove_comment_cpp(line) + # Wait for defineDataProcessing. + if not is_inside_define: + if not re.match(r"(o2::)?(framework::)?WorkflowSpec defineDataProcessing\(", line): + continue + # print(f"{i + 1}: Entering define.") + is_inside_define = True + # Return at the end of defineDataProcessing. + if is_inside_define and line[0] == "}": + # print(f"{i + 1}: Exiting define.") + break + # Detect options. + if ".options()" in line: + return False + return True + + # Documentation # Reference: https://rawgit.com/AliceO2Group/CodingGuidelines/master/comments_guidelines.html @@ -1102,6 +1135,7 @@ def main(): tests.append(TestLogging()) tests.append(TestConstRefInForLoop()) tests.append(TestConstRefInSubscription()) + tests.append(TestWorkflowOptions()) # Documentation enable_documentation = True From b05ec0474c95553feecf89e57a35623e8595c11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 26 Jul 2024 20:27:59 +0200 Subject: [PATCH 56/96] Linting block_ranges --- Scripts/o2_linter.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index c9297b66d63..fbd2c71f6e0 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -114,34 +114,34 @@ def remove_comment_cpp(line: str) -> str: return line.strip() -def block_ranges(line: str, c_open: str, c_close: str) -> "list[list[int]]": - """Get list of index ranges of longest blocks opened with str_open and closed with str_close.""" - # print(f"Looking for {c_open}{c_close} blocks in \"{line}\".") +def block_ranges(line: str, char_open: str, char_close: str) -> "list[list[int]]": + """Get list of index ranges of longest blocks opened with char_open and closed with char_close.""" + # print(f"Looking for {char_open}{char_close} blocks in \"{line}\".") # print(line) - list_ranges = [] - if not all((line, len(c_open) == 1, len(c_close) == 1)): + list_ranges: list[list[int]] = [] + if not all((line, len(char_open) == 1, len(char_close) == 1)): return list_ranges - def direction(c:str) -> int: - if c == c_open: + def direction(char:str) -> int: + if char == char_open: return 1 - if c == c_close: + if char == char_close: return -1 return 0 list_levels = [] # list of block levels (net number of opened blocks) - level = 0 # current block level (sum or previous directions) - for c in line: - list_levels.append(level := level + direction(c)) - level_min = min(list_levels) # minimum level (!= 0 if there are opened blocks) + level_sum = 0 # current block level (sum of previous directions) + for char in line: + list_levels.append(level_sum := level_sum + direction(char)) + level_min = min(list_levels) # minimum level (!= 0 if line has opened blocks) # Look for openings (level_min + 1) and closings (level_min). index_start = -1 is_opened = False # print(list_levels) - for i, l in enumerate(list_levels): - if not is_opened and l > level_min: + for i, level in enumerate(list_levels): + if not is_opened and level > level_min: is_opened = True index_start = i # print(f"Opening at {i}") - elif is_opened and (l == level_min or i == len(list_levels) -1): + elif is_opened and (level == level_min or i == len(list_levels) - 1): is_opened = False list_ranges.append([index_start, i]) # print(f"Closing at {i}") From 94b125efbcb097760155e8c5d05a7c13ea9876ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 26 Jul 2024 23:33:19 +0200 Subject: [PATCH 57/96] Add TestTwoPiAddedSubtracted, TestPiMultipliedDivided --- Scripts/o2_linter.py | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index fbd2c71f6e0..39de8901edb 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -294,10 +294,10 @@ def test_line(self, line: str) -> bool: return re.search(pattern, line) is None -class TestPI(TestSpec): - """Detect use of external PI.""" +class TestPi(TestSpec): + """Detect use of external pi.""" name = "external-pi" - message = "Consider using the PI constant (and its multiples) defined in o2::constants::math." + message = "Consider using the PI constant (and its multiples and fractions) defined in o2::constants::math." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: @@ -311,6 +311,38 @@ def test_line(self, line: str) -> bool: return re.search(pattern, line) is None +class TestTwoPiAddSubtract(TestSpec): + """Detect adding/subtracting of 2 pi.""" + name = "two-pi-add-subtract" + message = "Consider using RecoDecay::constrainAngle to restrict angle to a given range." + suffixes = [".h", ".cxx"] + + def test_line(self, line: str) -> bool: + pattern_two_pi = r"(2(\.0*f?)? \* (M_PI|TMath::Pi\(\)|(((o2::)?constants::)?math::)?PI)|(((o2::)?constants::)?math::)?TwoPI|TMath::TwoPi\(\))" + pattern = fr"[\+-]=? {pattern_two_pi}" + if is_comment_cpp(line): + return True + line = remove_comment_cpp(line) + return re.search(pattern, line) is None + + +class TestPiMultipleFraction(TestSpec): + """Detect multiplying/dividing of pi for existing equivalent constants.""" + name = "pi-multiple-fraction" + message = "Consider using multiples/fractions of PI defined in o2::constants::math." + suffixes = [".h", ".cxx"] + + def test_line(self, line: str) -> bool: + pattern_pi = r"(M_PI|TMath::(Two)?Pi\(\)|(((o2::)?constants::)?math::)?(Two)?PI)" + pattern_multiple = r"(2(\.0*f?)?|0\.2?5f?) \* " # * 2, 0.25, 0.5 + pattern_fraction = r" / ((2|3|4)([ ,;\)]|\.0*f?))" # / 2, 3, 4 + pattern = fr"{pattern_multiple}{pattern_pi}|{pattern_pi}{pattern_fraction}" + if is_comment_cpp(line): + return True + line = remove_comment_cpp(line) + return re.search(pattern, line) is None + + class TestPdgDatabase(TestSpec): """Detect use of TDatabasePDG.""" name = "pdg/database" @@ -1128,7 +1160,9 @@ def main(): tests.append(TestUsingDirectives()) tests.append(TestStdPrefix()) tests.append(TestROOT()) - tests.append(TestPI()) + tests.append(TestPi()) + tests.append(TestTwoPiAddSubtract()) + tests.append(TestPiMultipleFraction()) tests.append(TestPdgDatabase()) tests.append(TestPdgCode()) tests.append(TestPdgMass()) From c019c03055ba4db74773b58a2e66d8c5b7da5109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Fri, 26 Jul 2024 23:47:01 +0200 Subject: [PATCH 58/96] Extend description --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 39de8901edb..fac84a98254 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -12,7 +12,7 @@ # or submit itself to any jurisdiction. """! -@brief O2 linter (Find issues in O2 code) +@brief O2 linter (Find O2-specific issues in O2 code) @author Vít Kučera , Inha University @date 2024-07-14 """ From c8a1be002cc5b6c7b7a8807a70b37b3875bb3ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 27 Jul 2024 03:23:08 +0200 Subject: [PATCH 59/96] Format --- Scripts/o2_linter.py | 413 ++++++++++++++++++++++++++++--------------- 1 file changed, 275 insertions(+), 138 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index fac84a98254..d79307fa07c 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -17,14 +17,15 @@ @date 2024-07-14 """ -from abc import ABC import argparse import os import re import sys +from abc import ABC from typing import Union -github_mode = False # GitHub mode +github_mode = False # GitHub mode + def is_camel_case(name: str) -> bool: """forExample or ForExample""" @@ -74,7 +75,7 @@ def kebab_case_to_camel_case_u(line: str) -> str: def kebab_case_to_camel_case_l(line: str) -> str: """Convert kebab-case string to lowerCamelCase string.""" new_line = kebab_case_to_camel_case_u(line) - return f"{new_line[0].lower()}{new_line[1:]}" # start with lowercase letter + return f"{new_line[0].lower()}{new_line[1:]}" # start with lowercase letter def camel_case_to_kebab_case(line: str) -> str: @@ -95,10 +96,10 @@ def print_error(path: str, line: Union[int, None], title: str, message: str): """Format and print error message.""" # return # Use to suppress error message when counting speed. str_line = "" if line is None else f"{line}:" - print(f"{path}:{str_line} {message} [{title}]") # terminal format + print(f"{path}:{str_line} {message} [{title}]") # terminal format if github_mode: str_line = "" if line is None else f",line={line}" - print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub annotation format + print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub annotation format def is_comment_cpp(line: str) -> bool: @@ -110,7 +111,7 @@ def remove_comment_cpp(line: str) -> str: """Remove C++ comments from the end of a line.""" for keyword in ("//", "/*"): if keyword in line: - line = line[:line.index(keyword)] + line = line[: line.index(keyword)] return line.strip() @@ -121,17 +122,19 @@ def block_ranges(line: str, char_open: str, char_close: str) -> "list[list[int]] list_ranges: list[list[int]] = [] if not all((line, len(char_open) == 1, len(char_close) == 1)): return list_ranges - def direction(char:str) -> int: + + def direction(char: str) -> int: if char == char_open: return 1 if char == char_close: return -1 return 0 - list_levels = [] # list of block levels (net number of opened blocks) - level_sum = 0 # current block level (sum of previous directions) + + list_levels = [] # list of block levels (net number of opened blocks) + level_sum = 0 # current block level (sum of previous directions) for char in line: list_levels.append(level_sum := level_sum + direction(char)) - level_min = min(list_levels) # minimum level (!= 0 if line has opened blocks) + level_min = min(list_levels) # minimum level (!= 0 if line has opened blocks) # Look for openings (level_min + 1) and closings (level_min). index_start = -1 is_opened = False @@ -153,10 +156,11 @@ def direction(char:str) -> int: class TestSpec(ABC): """Prototype of a test class""" - name = "test-template" # short name of the test - message = "Test failed" # error message - suffixes: "list[str]" = [] # suffixes of files to test - per_line = True # Test lines separately one by one. + + name = "test-template" # short name of the test + message = "Test failed" # error message + suffixes: "list[str]" = [] # suffixes of files to test + per_line = True # Test lines separately one by one. def file_matches(self, path: str) -> bool: """Test whether the path matches the pattern for files to test.""" @@ -168,7 +172,7 @@ def is_disabled(self, line: str, prefix_comment="//") -> bool: for prefix in [prefix_comment, prefix_disable]: if not prefix in line: return False - line = line[(line.index(prefix) + len(prefix)):] # Strip away part before prefix. + line = line[(line.index(prefix) + len(prefix)) :] # Strip away part before prefix. if self.name in line: return True return False @@ -177,11 +181,11 @@ def test_line(self, line: str) -> bool: """Test a line.""" raise NotImplementedError() - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: """Test a file in a way that cannot be done line by line.""" raise NotImplementedError() - def run(self, path : str, content) -> bool: + def run(self, path: str, content) -> bool: """Run the test.""" # print(content) passed = True @@ -212,8 +216,10 @@ def run(self, path : str, content) -> bool: # Bad practice + class TestIOStream(TestSpec): """Detect included iostream.""" + name = "include-iostream" message = "Including iostream is discouraged. Use O2 logging instead." suffixes = [".h", ".cxx"] @@ -226,6 +232,7 @@ def test_line(self, line: str) -> bool: class TestUsingStd(TestSpec): """Detect importing names from the std namespace.""" + name = "import-std-name" message = "Importing names from the std namespace is not allowed in headers." suffixes = [".h"] @@ -238,6 +245,7 @@ def test_line(self, line: str) -> bool: class TestUsingDirectives(TestSpec): """Detect using directives in headers.""" + name = "using-directive" message = "Using directives are not allowed in headers." suffixes = [".h"] @@ -250,27 +258,43 @@ def test_line(self, line: str) -> bool: class TestStdPrefix(TestSpec): """Detect missing std:: prefix for common names from the std namespace.""" + name = "std-prefix" message = "Use std:: prefix for names from the std namespace." suffixes = [".h", ".cxx", ".C"] prefix_bad = r"[^\w:\.\"]" - patterns = [r"vector<", r"array[<\{\(]", r"f?abs\(", r"sqrt\(", r"pow\(", r"min\(", r"max\(", r"log\(", r"exp\(", r"sin\(", r"cos\(", r"tan\(", r"atan\(", r"atan2\("] + patterns = [ + r"vector<", + r"array[<\{\(]", + r"f?abs\(", + r"sqrt\(", + r"pow\(", + r"min\(", + r"max\(", + r"log\(", + r"exp\(", + r"sin\(", + r"cos\(", + r"tan\(", + r"atan\(", + r"atan2\(", + ] def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True line = remove_comment_cpp(line) for pattern in self.patterns: - iterators = re.finditer(fr"{self.prefix_bad}{pattern}", line) + iterators = re.finditer(rf"{self.prefix_bad}{pattern}", line) matches = [(it.start(), it.group()) for it in iterators] if not matches: continue - if not "\"" in line: # Found a match which cannot be inside a string. + if not '"' in line: # Found a match which cannot be inside a string. return False # Ignore matches inside strings. for match in matches: - n_quotes_before = line.count("\"", 0, match[0]) # Count quotation marks before the match. - if n_quotes_before % 2: # If odd, we are inside a string and we should ignore this match. + n_quotes_before = line.count('"', 0, match[0]) # Count quotation marks before the match. + if n_quotes_before % 2: # If odd, we are inside a string and we should ignore this match. continue # We are not inside a string and this match is valid. return False @@ -279,6 +303,7 @@ def test_line(self, line: str) -> bool: class TestROOT(TestSpec): """Detect use of unnecessary ROOT entities.""" + name = "root-entity" message = "Consider replacing ROOT entities with STD C++ or O2 entities." suffixes = [".h", ".cxx"] @@ -296,6 +321,7 @@ def test_line(self, line: str) -> bool: class TestPi(TestSpec): """Detect use of external pi.""" + name = "external-pi" message = "Consider using the PI constant (and its multiples and fractions) defined in o2::constants::math." suffixes = [".h", ".cxx"] @@ -313,13 +339,14 @@ def test_line(self, line: str) -> bool: class TestTwoPiAddSubtract(TestSpec): """Detect adding/subtracting of 2 pi.""" + name = "two-pi-add-subtract" message = "Consider using RecoDecay::constrainAngle to restrict angle to a given range." suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: pattern_two_pi = r"(2(\.0*f?)? \* (M_PI|TMath::Pi\(\)|(((o2::)?constants::)?math::)?PI)|(((o2::)?constants::)?math::)?TwoPI|TMath::TwoPi\(\))" - pattern = fr"[\+-]=? {pattern_two_pi}" + pattern = rf"[\+-]=? {pattern_two_pi}" if is_comment_cpp(line): return True line = remove_comment_cpp(line) @@ -328,15 +355,16 @@ def test_line(self, line: str) -> bool: class TestPiMultipleFraction(TestSpec): """Detect multiplying/dividing of pi for existing equivalent constants.""" + name = "pi-multiple-fraction" message = "Consider using multiples/fractions of PI defined in o2::constants::math." suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: pattern_pi = r"(M_PI|TMath::(Two)?Pi\(\)|(((o2::)?constants::)?math::)?(Two)?PI)" - pattern_multiple = r"(2(\.0*f?)?|0\.2?5f?) \* " # * 2, 0.25, 0.5 - pattern_fraction = r" / ((2|3|4)([ ,;\)]|\.0*f?))" # / 2, 3, 4 - pattern = fr"{pattern_multiple}{pattern_pi}|{pattern_pi}{pattern_fraction}" + pattern_multiple = r"(2(\.0*f?)?|0\.2?5f?) \* " # * 2, 0.25, 0.5 + pattern_fraction = r" / ((2|3|4)([ ,;\)]|\.0*f?))" # / 2, 3, 4 + pattern = rf"{pattern_multiple}{pattern_pi}|{pattern_pi}{pattern_fraction}" if is_comment_cpp(line): return True line = remove_comment_cpp(line) @@ -345,6 +373,7 @@ def test_line(self, line: str) -> bool: class TestPdgDatabase(TestSpec): """Detect use of TDatabasePDG.""" + name = "pdg/database" message = "Direct use of TDatabasePDG is not allowed. Use o2::constants::physics::Mass... or Service." suffixes = [".h", ".cxx"] @@ -361,6 +390,7 @@ def test_line(self, line: str) -> bool: class TestPdgCode(TestSpec): """Detect use of hard-coded PDG codes.""" + name = "pdg/explicit-code" message = "Avoid using hard-coded PDG codes. Use named values from PDG_t or o2::constants::physics::Pdg instead." suffixes = [".h", ".cxx", ".C"] @@ -381,8 +411,11 @@ def test_line(self, line: str) -> bool: class TestPdgMass(TestSpec): """Detect unnecessary call of Mass() for a known PDG code.""" + name = "pdg/known-mass" - message = "Consider using o2::constants::physics::Mass... instead of calling a database method for a known PDG code." + message = ( + "Consider using o2::constants::physics::Mass... instead of calling a database method for a known PDG code." + ) suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -390,15 +423,16 @@ def test_line(self, line: str) -> bool: return True line = remove_comment_cpp(line) pattern_pdg_code = r"[+-]?(k[A-Z][a-zA-Z0-9]*|[0-9]+)" - if re.search(fr"->GetParticle\({pattern_pdg_code}\)->Mass\(\)", line): + if re.search(rf"->GetParticle\({pattern_pdg_code}\)->Mass\(\)", line): return False - if re.search(fr"->Mass\({pattern_pdg_code}\)", line): + if re.search(rf"->Mass\({pattern_pdg_code}\)", line): return False return True class TestLogging(TestSpec): """Detect non-O2 logging.""" + name = "logging" message = "Consider using O2 logging (LOG, LOGF, LOGP)." suffixes = [".h", ".cxx"] @@ -416,6 +450,7 @@ def test_line(self, line: str) -> bool: class TestConstRefInForLoop(TestSpec): """Test const refs in range-based for loops.""" + name = "const-ref-in-for-loop" message = "Use constant references for non-modified iterators in range-based for loops." suffixes = [".h", ".cxx", ".C"] @@ -426,48 +461,49 @@ def test_line(self, line: str) -> bool: line = remove_comment_cpp(line) if not re.match(r"for \(.* :", line): return True - line = line[:line.index(" :")] # keep only the iterator part + line = line[: line.index(" :")] # keep only the iterator part return re.search(r"(\w const|const \w+)& ", line) is not None class TestConstRefInSubscription(TestSpec): """Test const refs in process function subscriptions.""" + name = "const-ref-in-process" message = "Use constant references for table subscriptions in process functions." suffixes = [".cxx"] per_line = False - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: passed = True - n_parens_opened = 0 # number of opened parentheses - arguments = "" # process function arguments - line_process = 0 # line number of the process function + n_parens_opened = 0 # number of opened parentheses + arguments = "" # process function arguments + line_process = 0 # line number of the process function for i, line in enumerate(content): line = line.strip() if is_comment_cpp(line): continue if self.is_disabled(line): continue - if "//" in line: # Remove comment. (Ignore /* to avoid truncating at /*parameter*/.) - line = line[:line.index("//")] + if "//" in line: # Remove comment. (Ignore /* to avoid truncating at /*parameter*/.) + line = line[: line.index("//")] if re.search(r"^void process[\w]*\(", line): line_process = i + 1 i_closing = line.rfind(")") i_start = line.index("(") + 1 i_end = i_closing if i_closing != -1 else len(line) - arguments = line[i_start:i_end] # get arguments between parentheses + arguments = line[i_start:i_end] # get arguments between parentheses n_parens_opened = line.count("(") - line.count(")") elif n_parens_opened > 0: i_closing = line.rfind(")") i_start = 0 i_end = i_closing if i_closing != -1 else len(line) - arguments += " " + line[i_start:i_end] # get arguments between parentheses + arguments += " " + line[i_start:i_end] # get arguments between parentheses n_parens_opened += line.count("(") - line.count(")") if line_process > 0 and n_parens_opened == 0: # Process arguments. # Sanitise arguments with spaces between <>. for start, end in block_ranges(arguments, "<", ">"): - arg = arguments[start:(end + 1)] + arg = arguments[start : (end + 1)] # print(f"Found argument \"{arg}\" in [{start}, {end}]") if ", " in arg: arguments = arguments.replace(arg, arg.replace(", ", "__")) @@ -484,13 +520,14 @@ def test_file(self, path : str, content) -> bool: class TestWorkflowOptions(TestSpec): """Detect usage of workflow options in defineDataProcessing. (Not supported on AliHyperloop.)""" + name = "o2-workflow-options" message = "Do not use workflow options to customise workflow topology composition in defineDataProcessing. Use process function switches or metadata instead." suffixes = [".cxx"] per_line = False - def test_file(self, path : str, content) -> bool: - is_inside_define = False # Are we inside defineDataProcessing? + def test_file(self, path: str, content) -> bool: + is_inside_define = False # Are we inside defineDataProcessing? for i, line in enumerate(content): if not line.strip(): continue @@ -521,40 +558,46 @@ def test_file(self, path : str, content) -> bool: class TestDocumentationFile(TestSpec): """Test mandatory documentation of C++ files.""" + name = "doc/file" message = "Provide mandatory file documentation." suffixes = [".h", ".cxx", ".C"] per_line = False - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: passed = False doc_items = [] - doc_items.append({"keyword": "file", "pattern": fr"{os.path.basename(path)}$", "found": False}) - doc_items.append({"keyword": "brief", "pattern": r"\w.* \w", "found": False}) # at least two words + doc_items.append({"keyword": "file", "pattern": rf"{os.path.basename(path)}$", "found": False}) + doc_items.append({"keyword": "brief", "pattern": r"\w.* \w", "found": False}) # at least two words doc_items.append({"keyword": "author", "pattern": r"[\w]+", "found": False}) doc_prefix = "///" n_lines_copyright = 11 last_doc_line = n_lines_copyright for i, line in enumerate(content): - if i < n_lines_copyright: # Skip copyright lines. + if i < n_lines_copyright: # Skip copyright lines. continue - if line.strip() and not line.startswith(doc_prefix): # Stop at the first non-empty non-doc line. + if line.strip() and not line.startswith(doc_prefix): # Stop at the first non-empty non-doc line. break if line.startswith(doc_prefix): - last_doc_line = (i + 1) + last_doc_line = i + 1 for item in doc_items: - if re.search(fr"^{doc_prefix} [\\@]{item['keyword']} +{item['pattern']}", line): + if re.search(rf"^{doc_prefix} [\\@]{item['keyword']} +{item['pattern']}", line): item["found"] = True # print_error(path, i + 1, self.name, f"Found \{item['keyword']}.") break - if all(item["found"] for item in doc_items): # All items have been found. + if all(item["found"] for item in doc_items): # All items have been found. passed = True break if not passed: for item in doc_items: if not item["found"]: - print_error(path, last_doc_line, self.name, f"Documentation for \\{item['keyword']} is missing, incorrect or misplaced.") + print_error( + path, + last_doc_line, + self.name, + f"Documentation for \\{item['keyword']} is missing, incorrect or misplaced.", + ) return passed @@ -570,6 +613,7 @@ class TestNameFunctionVariable(TestSpec): Does not detect multi-line declarations. Does not check capitalisation for constexpr because of special rules for constants. See TestNameConstant. """ + name = "name/function-variable" message = "Use lowerCamelCase for names of functions and variables." suffixes = [".h", ".cxx", ".C"] @@ -583,7 +627,7 @@ def test_line(self, line: str) -> bool: # For functions, stripping after "(" is enough but this way we also identify many declarations of variables. for keyword in ("(", "{", ";", " = ", "//", "/*"): if keyword in line: - line = line[:line.index(keyword)] + line = line[: line.index(keyword)] # Check the words. words = line.split() @@ -597,21 +641,38 @@ def test_line(self, line: str) -> bool: return True # Reject false positives with same structure. - if words[0] in ("return", "if", "else", "new", "delete", "delete[]", "case", "typename", "using", "typedef", "enum", "namespace", "struct", "class"): + if words[0] in ( + "return", + "if", + "else", + "new", + "delete", + "delete[]", + "case", + "typename", + "using", + "typedef", + "enum", + "namespace", + "struct", + "class", + ): return True if len(words) > 2 and words[1] in ("typename", "class", "struct"): return True # Identify the position of the name for cases "name[n + m]". - funval_name = words[-1] # expecting the name in the last word - if funval_name.endswith("]") and "[" not in funval_name: # it's an array and we do not have the name before "[" here + funval_name = words[-1] # expecting the name in the last word + if ( + funval_name.endswith("]") and "[" not in funval_name + ): # it's an array and we do not have the name before "[" here opens_brackets = ["[" in w for w in words] - if not any(opens_brackets): # The opening "[" is not on this line. We have to give up. + if not any(opens_brackets): # The opening "[" is not on this line. We have to give up. return True - index_name = opens_brackets.index(True) # the name is in the first element with "[" + index_name = opens_brackets.index(True) # the name is in the first element with "[" funval_name = words[index_name] - words = words[:(index_name + 1)] # Strip away words after the name. - if len(words) < 2: # Check the adjusted number of words. + words = words[: (index_name + 1)] # Strip away words after the name. + if len(words) < 2: # Check the adjusted number of words. return True # All words before the name start with an alphanumeric character (underscores not allowed). @@ -620,15 +681,15 @@ def test_line(self, line: str) -> bool: return True # Extract function/variable name. - if "[" in funval_name: # Remove brackets for arrays. - funval_name = funval_name[:funval_name.index("[")] - if "::" in funval_name: # Remove the class prefix for methods. + if "[" in funval_name: # Remove brackets for arrays. + funval_name = funval_name[: funval_name.index("[")] + if "::" in funval_name: # Remove the class prefix for methods. funval_name = funval_name.split("::")[-1] # Check the name candidate. # Names of variables and functions are identifiers. - if not funval_name.isidentifier(): # should be same as ^[\w]+$ + if not funval_name.isidentifier(): # should be same as ^[\w]+$ return True # print(f"{line} -> {funval_name}") @@ -641,6 +702,7 @@ def test_line(self, line: str) -> bool: class TestNameMacro(TestSpec): """Test macro names.""" + name = "name/macro" message = "Use SCREAMING_SNAKE_CASE for names of macros." suffixes = [".h", ".cxx", ".C"] @@ -660,8 +722,11 @@ def test_line(self, line: str) -> bool: class TestNameConstant(TestSpec): """Test constexpr constant names.""" + name = "name/constexpr-constant" - message = "Use UpperCamelCase for names of constexpr constants. Names of special constants may be prefixed with \"k\"." + message = ( + 'Use UpperCamelCase for names of constexpr constants. Names of special constants may be prefixed with "k".' + ) suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -672,25 +737,28 @@ def test_line(self, line: str) -> bool: if not "constexpr" in words or not "=" in words: return True # Extract constant name. - words = words[:words.index("=")] # keep only words before "=" - constant_name = words[-1] # last word before "=" - if constant_name.endswith("]") and "[" not in constant_name: # it's an array and we do not have the name before "[" here + words = words[: words.index("=")] # keep only words before "=" + constant_name = words[-1] # last word before "=" + if ( + constant_name.endswith("]") and "[" not in constant_name + ): # it's an array and we do not have the name before "[" here opens_brackets = ["[" in w for w in words] - if not any(opens_brackets): # The opening "[" is not on this line. We have to give up. + if not any(opens_brackets): # The opening "[" is not on this line. We have to give up. return True - constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" - if "[" in constant_name: # Remove brackets for arrays. - constant_name = constant_name[:constant_name.index("[")] - if "::" in constant_name: # Remove the class prefix for methods. + constant_name = words[opens_brackets.index(True)] # the name is in the first element with "[" + if "[" in constant_name: # Remove brackets for arrays. + constant_name = constant_name[: constant_name.index("[")] + if "::" in constant_name: # Remove the class prefix for methods. constant_name = constant_name.split("::")[-1] # The actual test comes here. - if constant_name.startswith("k") and len(constant_name) > 1: # exception for special constants - constant_name = constant_name[1:] # test the name without "k" + if constant_name.startswith("k") and len(constant_name) > 1: # exception for special constants + constant_name = constant_name[1:] # test the name without "k" return is_upper_camel_case(constant_name) class TestNameColumn(TestSpec): """Test names of O2 columns.""" + name = "name/o2-column" message = "Use UpperCamelCase for names of O2 columns and matching lowerCamelCase names for their getters." suffixes = [".h", ".cxx"] @@ -702,15 +770,15 @@ def test_line(self, line: str) -> bool: return True # Extract names of the column type and getter. line = remove_comment_cpp(line) - line = line[len(match.group()):].strip() # Extract part after "(". + line = line[len(match.group()) :].strip() # Extract part after "(". if not (match := re.match(r"([^,]+), ([^,\) ]+)", line)): - print(f"Failed to extract column type and getter from \"{line}\".") + print(f'Failed to extract column type and getter from "{line}".') return False column_type_name = match.group(1) column_getter_name = match.group(2) # print(f"Got \"{column_type_name}\" \"{column_getter_name}\"") # return True - if column_type_name[0] == "_": # probably a macro variable + if column_type_name[0] == "_": # probably a macro variable return True # The actual test comes here. if not is_upper_camel_case(column_type_name): @@ -722,6 +790,7 @@ def test_line(self, line: str) -> bool: class TestNameTable(TestSpec): """Test names of O2 tables.""" + name = "name/o2-table" message = "Use UpperCamelCase for names of O2 tables." suffixes = [".h", ".cxx"] @@ -733,14 +802,14 @@ def test_line(self, line: str) -> bool: return True # Extract names of the column type and getter. line = remove_comment_cpp(line) - line = line[len(match.group()):].strip() # Extract part after "(". + line = line[len(match.group()) :].strip() # Extract part after "(". if not (match := re.match(r"([^,\) ]+)", line)): - print(f"Failed to extract table type from \"{line}\".") + print(f'Failed to extract table type from "{line}".') return False table_type_name = match.group(1) # print(f"Got \"{table_type_name}\"") # return True - if table_type_name[0] == "_": # probably a macro variable + if table_type_name[0] == "_": # probably a macro variable return True # The actual test comes here. return is_upper_camel_case(table_type_name) @@ -748,6 +817,7 @@ def test_line(self, line: str) -> bool: class TestNameNamespace(TestSpec): """Test names of namespaces.""" + name = "name/namespace" message = "Use snake_case for names of namespaces." suffixes = [".h", ".cxx", ".C"] @@ -759,7 +829,7 @@ def test_line(self, line: str) -> bool: return True # Extract namespace name. namespace_name = line.split()[1] - if namespace_name == "{": # ignore anonymous namespaces + if namespace_name == "{": # ignore anonymous namespaces return True # The actual test comes here. return is_snake_case(namespace_name) @@ -767,6 +837,7 @@ def test_line(self, line: str) -> bool: class TestNameUpperCamelCase(TestSpec): """Base class for a test of UpperCamelCase names.""" + keyword = "key" name = f"name/{keyword}" message = f"Use UpperCamelCase for names of {keyword}." @@ -779,10 +850,10 @@ def test_line(self, line: str) -> bool: return True # Extract object name. words = line.split() - if not words[1].isalnum(): # "struct : ...", "enum { ..." + if not words[1].isalnum(): # "struct : ...", "enum { ..." return True object_name = words[1] - if object_name in ("class", "struct") and len(words) > 2: # enum class ... or enum struct + if object_name in ("class", "struct") and len(words) > 2: # enum class ... or enum struct object_name = words[2] # The actual test comes here. return is_upper_camel_case(object_name) @@ -790,6 +861,7 @@ def test_line(self, line: str) -> bool: class TestNameEnum(TestNameUpperCamelCase): """Test names of enumerators.""" + keyword = "enum" name = f"name/enum" message = f"Use UpperCamelCase for names of enumerators and their values." @@ -797,6 +869,7 @@ class TestNameEnum(TestNameUpperCamelCase): class TestNameClass(TestNameUpperCamelCase): """Test names of classes.""" + keyword = "class" name = f"name/class" message = f"Use UpperCamelCase for names of classes." @@ -804,6 +877,7 @@ class TestNameClass(TestNameUpperCamelCase): class TestNameStruct(TestNameUpperCamelCase): """Test names of structs.""" + keyword = "struct" name = f"name/struct" message = f"Use UpperCamelCase for names of structs." @@ -811,12 +885,13 @@ class TestNameStruct(TestNameUpperCamelCase): class TestNameFileCpp(TestSpec): """Test names of C++ files.""" + name = "name/file-cpp" message = "Use lowerCamelCase or UpperCamelCase for names of C++ files. See the O2 naming conventions for details." suffixes = [".h", ".cxx", ".C"] per_line = False - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path) # workflow file if re.search(r"(TableProducer|Tasks)/.*\.cxx", path): @@ -832,24 +907,26 @@ def test_file(self, path : str, content) -> bool: class TestNameFilePython(TestSpec): """Test names of Python files.""" + name = "name/file-python" message = "Use snake_case for names of Python files." suffixes = [".py", ".ipynb"] per_line = False - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path) return is_snake_case(file_name) class TestNameWorkflow(TestSpec): """Test names of O2 workflows.""" + name = "name/o2-workflow" message = "Use kebab-case for names of workflows and match the name of the workflow file." suffixes = ["CMakeLists.txt"] per_line = False - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: passed = True workflow_name = "" for i, line in enumerate(content): @@ -870,30 +947,36 @@ def test_file(self, path : str, content) -> bool: passed = False print_error(path, i + 2, self.name, f"Did not find sources for workflow: {workflow_name}.") continue - workflow_file_name = os.path.basename(words[1]) # the actual file name + workflow_file_name = os.path.basename(words[1]) # the actual file name # Generate the file name matching the workflow name. expected_workflow_file_name = kebab_case_to_camel_case_l(workflow_name) + ".cxx" # Compare the actual and expected file names. if expected_workflow_file_name != workflow_file_name: passed = False - print_error(path, i + 1, self.name, f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. (Matches {expected_workflow_file_name}.)") + print_error( + path, + i + 1, + self.name, + f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. (Matches {expected_workflow_file_name}.)", + ) return passed class TestNameTask(TestSpec): """Test explicit task names. Detect usage of TaskName, check whether it is needed and whether the task name matches the struct name.""" + name = "name/o2-task" message = "Specify task name only when it cannot be derived from the struct name. Only append to the default name." suffixes = [".cxx"] per_line = False - def test_file(self, path : str, content) -> bool: - is_inside_define = False # Are we inside defineDataProcessing? - is_inside_adapt = False # Are we inside adaptAnalysisTask? + def test_file(self, path: str, content) -> bool: + is_inside_define = False # Are we inside defineDataProcessing? + is_inside_adapt = False # Are we inside adaptAnalysisTask? struct_name = "" - struct_templated = False # Is the struct templated? - n_parens_opened = 0 # number of opened parentheses + struct_templated = False # Is the struct templated? + n_parens_opened = 0 # number of opened parentheses passed = True for i, line in enumerate(content): if not line.strip(): @@ -919,10 +1002,10 @@ def test_file(self, path : str, content) -> bool: continue # print(f"{i + 1}: Entering adapt.") is_inside_adapt = True - line = line[(index + len("adaptAnalysisTask<")):] + line = line[(index + len("adaptAnalysisTask<")) :] # Extract struct name. if not (match := re.match(r"([^>]+)", line)): - print_error(path, i + 1, self.name, f"Failed to extract struct name from \"{line}\".") + print_error(path, i + 1, self.name, f'Failed to extract struct name from "{line}".') return False struct_name = match.group(1) if (index := struct_name.find("<")) > -1: @@ -930,7 +1013,7 @@ def test_file(self, path : str, content) -> bool: # print(f"{i + 1}: Got templated struct name {struct_name}") struct_name = struct_name[:index] # print(f"{i + 1}: Got struct name {struct_name}") - line = line[(line.index(struct_name) + len(struct_name)):] + line = line[(line.index(struct_name) + len(struct_name)) :] if is_inside_adapt: n_parens_opened += line.count("(") - line.count(")") # print(f"{i + 1}: {n_parens_opened} opened parens") @@ -943,32 +1026,58 @@ def test_file(self, path : str, content) -> bool: passed = False # Extract explicit task name. if not (match := re.search(r"TaskName\{\"([^\}]+)\"\}", line)): - print_error(path, i + 1, self.name, f"Failed to extract explicit task name from \"{line}\".") + print_error(path, i + 1, self.name, f'Failed to extract explicit task name from "{line}".') return False task_name = match.group(1) # print(f"{i + 1}: Got struct \"{struct_name}\" with task name \"{task_name}\".") # Test explicit task name. - device_name_from_struct_name = camel_case_to_kebab_case(struct_name) # default device name, in absence of TaskName - device_name_from_task_name = camel_case_to_kebab_case(task_name) # actual device name, generated from TaskName - struct_name_from_device_name = kebab_case_to_camel_case_u(device_name_from_task_name) # struct name matching the TaskName + device_name_from_struct_name = camel_case_to_kebab_case( + struct_name + ) # default device name, in absence of TaskName + device_name_from_task_name = camel_case_to_kebab_case( + task_name + ) # actual device name, generated from TaskName + struct_name_from_device_name = kebab_case_to_camel_case_u( + device_name_from_task_name + ) # struct name matching the TaskName if device_name_from_struct_name == device_name_from_task_name: # If the task name results in the same device name as the struct name would, TaskName is redundant and should be removed. - print_error(path, i + 1, self.name, f"Specified task name {task_name} and the struct name {struct_name} produce the same device name {device_name_from_struct_name}. TaskName is redundant.") + print_error( + path, + i + 1, + self.name, + f"Specified task name {task_name} and the struct name {struct_name} produce the same device name {device_name_from_struct_name}. TaskName is redundant.", + ) passed = False elif device_name_from_struct_name.replace("-", "") == device_name_from_task_name.replace("-", ""): # If the device names generated from the task name and from the struct name differ in hyphenation, capitalisation of the struct name should be fixed and TaskName should be removed. (special cases: alice3-, -2prong) - print_error(path, i + 1, self.name, f"Device names {device_name_from_task_name} and {device_name_from_struct_name} generated from the specified task name {task_name} and from the struct name {struct_name}, respectively, differ in hyphenation. Consider fixing capitalisation of the struct name to {struct_name_from_device_name} and removing TaskName.") + print_error( + path, + i + 1, + self.name, + f"Device names {device_name_from_task_name} and {device_name_from_struct_name} generated from the specified task name {task_name} and from the struct name {struct_name}, respectively, differ in hyphenation. Consider fixing capitalisation of the struct name to {struct_name_from_device_name} and removing TaskName.", + ) passed = False elif device_name_from_task_name.startswith(device_name_from_struct_name): # If the device name generated from the task name is an extension of the device name generated from the struct name, accept it if the struct is templated. If the struct is not templated, extension is acceptable if adaptAnalysisTask is called multiple times for the same struct. if not struct_templated: - print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} but the struct is not templated. Is it adapted multiple times?") + print_error( + path, + i + 1, + self.name, + f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} but the struct is not templated. Is it adapted multiple times?", + ) passed = False # else: # print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} and the struct is templated. All good") else: # Other cases should be rejected. - print_error(path, i + 1, self.name, f"Specified task name {task_name} produces device name {device_name_from_task_name} which does not match the device name {device_name_from_struct_name} from the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})") + print_error( + path, + i + 1, + self.name, + f"Specified task name {task_name} produces device name {device_name_from_task_name} which does not match the device name {device_name_from_struct_name} from the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})", + ) passed = False return passed @@ -978,8 +1087,9 @@ def test_file(self, path : str, content) -> bool: class TestHfConstAuto(TestSpec): """PWGHF: Detect swapped const auto.""" + name = "pwghf/const-auto" - message = "Use \"const auto\" instead of \"auto const\"." + message = 'Use "const auto" instead of "auto const".' suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: @@ -994,8 +1104,9 @@ def test_line(self, line: str) -> bool: class TestHfNameStructClass(TestSpec): """PWGHF: Test names of structs and classes.""" + name = "pwghf/name/struct-class" - message = "Names of PWGHF structs and classes must start with \"Hf\"." + message = 'Names of PWGHF structs and classes must start with "Hf".' suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: @@ -1009,7 +1120,7 @@ def test_line(self, line: str) -> bool: line = remove_comment_cpp(line) # Extract struct/class name. words = line.split() - if not words[1].isalnum(): # "struct : ..." + if not words[1].isalnum(): # "struct : ..." return True struct_name = words[1] # The actual test comes here. @@ -1019,23 +1130,42 @@ def test_line(self, line: str) -> bool: class TestHfStructMembers(TestSpec): """PWGHF: Test order of struct members. Caveat: Does not see Configurables in ConfigurableGroup.""" + name = "pwghf/struct-member-order" message = "Declare struct members in the conventional order. See the PWGHF coding guidelines." suffixes = [".cxx"] per_line = False - member_order = ["Spawns<", "Builds<", "Produces<", "Configurable<", "HfHelper ", "SliceCache ", "Service<", "using ", "Filter ", "Preslice<", "Partition<", "ConfigurableAxis ", "AxisSpec ", "HistogramRegistry ", "OutputObj<", "void init(", "void process"] + member_order = [ + "Spawns<", + "Builds<", + "Produces<", + "Configurable<", + "HfHelper ", + "SliceCache ", + "Service<", + "using ", + "Filter ", + "Preslice<", + "Partition<", + "ConfigurableAxis ", + "AxisSpec ", + "HistogramRegistry ", + "OutputObj<", + "void init(", + "void process", + ] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: passed = True - dic_struct:dict[str, dict] = {} + dic_struct: dict[str, dict] = {} struct_name = "" for i, line in enumerate(content): if is_comment_cpp(line): continue - if line.startswith("struct "): # expecting no indentation + if line.startswith("struct "): # expecting no indentation line = remove_comment_cpp(line) struct_name = line.strip().split()[1] dic_struct[struct_name] = {} @@ -1044,24 +1174,31 @@ def test_file(self, path : str, content) -> bool: continue # Save line numbers of members of the current struct for each category. for member in self.member_order: - if line.startswith(f" {member}"): # expecting single-level indentation for direct members + if line.startswith(f" {member}"): # expecting single-level indentation for direct members if member not in dic_struct[struct_name]: dic_struct[struct_name][member] = [] - dic_struct[struct_name][member].append(i + 1) # save line number + dic_struct[struct_name][member].append(i + 1) # save line number break # print(dic_struct) # Detect members declared in a wrong order. - last_line_last_member = 0 # line number of the last member of the previous member category - index_last_member = 0 # index of the previous member category in the member_order list + last_line_last_member = 0 # line number of the last member of the previous member category + index_last_member = 0 # index of the previous member category in the member_order list for struct_name in dic_struct: for i_m, member in enumerate(self.member_order): if member not in dic_struct[struct_name]: continue - first_line = min(dic_struct[struct_name][member]) # line number of the first member of this category - last_line = max(dic_struct[struct_name][member]) # line number of the last member of this category - if first_line < last_line_last_member: # The current category starts before the end of the previous category. + first_line = min(dic_struct[struct_name][member]) # line number of the first member of this category + last_line = max(dic_struct[struct_name][member]) # line number of the last member of this category + if ( + first_line < last_line_last_member + ): # The current category starts before the end of the previous category. passed = False - print_error(path, first_line, self.name, f"{struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).") + print_error( + path, + first_line, + self.name, + f"{struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).", + ) last_line_last_member = last_line index_last_member = i_m return passed @@ -1069,6 +1206,7 @@ def test_file(self, path : str, content) -> bool: class TestHfNameFileWorkflow(TestSpec): """PWGHF: Test names of workflow files.""" + name = "pwghf/name/workflow-file" message = "Name of a workflow file must match the name of the main task in it. See the PWGHF O2 naming conventions for details." suffixes = [".cxx"] @@ -1077,19 +1215,19 @@ class TestHfNameFileWorkflow(TestSpec): def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path - def test_file(self, path : str, content) -> bool: + def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path).rstrip(".cxx") if file_name[0].isupper(): return False - base_struct_name = f"Hf{file_name[0].upper()}{file_name[1:]}" # expected base of struct names + base_struct_name = f"Hf{file_name[0].upper()}{file_name[1:]}" # expected base of struct names # print(f"For file {file_name} expecting to find {base_struct_name}.") - struct_names = [] # actual struct names in the file + struct_names = [] # actual struct names in the file for line in content: if not line.startswith("struct "): continue # Extract struct name. words = line.split() - if not words[1].isalnum(): # "struct : ..." + if not words[1].isalnum(): # "struct : ..." continue struct_name = words[1] struct_names.append(struct_name) @@ -1102,6 +1240,7 @@ def test_file(self, path : str, content) -> bool: class TestHfNameConfigurable(TestSpec): """PWGHF: Test names of configurables.""" + name = "pwghf/name/configurable" message = "Use lowerCamelCase for names of configurables and use the same name for the struct member as for the JSON string." suffixes = [".h", ".cxx"] @@ -1116,16 +1255,16 @@ def test_line(self, line: str) -> bool: return True # Extract Configurable name. words = line.split() - if words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", - name_cpp = words[1] # nameCpp - name_json = words[3][1:] # expecting "nameJson", + if words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", + name_cpp = words[1] # nameCpp + name_json = words[3][1:] # expecting "nameJson", else: - names = words[1].split("{") # expecting Configurable... nameCpp{"nameJson", - name_cpp = names[0] # nameCpp - name_json = names[1] # expecting "nameJson", - if name_json[0] != "\"": # JSON name is not a literal string. + names = words[1].split("{") # expecting Configurable... nameCpp{"nameJson", + name_cpp = names[0] # nameCpp + name_json = names[1] # expecting "nameJson", + if name_json[0] != '"': # JSON name is not a literal string. return True - name_json = name_json.strip("\",") # expecting nameJson + name_json = name_json.strip('",') # expecting nameJson # The actual test comes here. return is_lower_camel_case(name_cpp) and name_cpp == name_json @@ -1135,9 +1274,7 @@ def test_line(self, line: str) -> bool: def main(): """Main function""" - parser = argparse.ArgumentParser( - description="Run tests on files to find code issues." - ) + parser = argparse.ArgumentParser(description="Run tests on files to find code issues.") parser.add_argument("paths", type=str, nargs="+", help="File path(s)") parser.add_argument( "-g", @@ -1150,7 +1287,7 @@ def main(): global github_mode github_mode = True - tests = [] # list of activated tests + tests = [] # list of activated tests # Bad practice enable_bad_practice = True @@ -1202,9 +1339,9 @@ def main(): tests.append(TestHfNameFileWorkflow()) tests.append(TestHfNameConfigurable()) - test_names = [t.name for t in tests] # short names of activated tests - suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests - passed = True # global result of all tests + test_names = [t.name for t in tests] # short names of activated tests + suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests + passed = True # global result of all tests # Report overview before running. print(f"Testing {len(args.paths)} files.") @@ -1229,7 +1366,7 @@ def main(): passed = False # print(f"File \"{path}\" {'passed' if result else 'failed'} the test {test.name}.") except IOError: - print(f"Failed to open file \"{path}\".") + print(f'Failed to open file "{path}".') sys.exit(1) # Report global result. From 1c65770a2bee991e369c8f53d6378df0f7ac7317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 27 Jul 2024 03:35:42 +0200 Subject: [PATCH 60/96] Lint with ruff --- Scripts/o2_linter.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index d79307fa07c..7e7d3d640e5 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -29,7 +29,7 @@ def is_camel_case(name: str) -> bool: """forExample or ForExample""" - return not "_" in name and not "-" in name and not " " in name + return "_" not in name and "-" not in name and " " not in name def is_upper_camel_case(name: str) -> bool: @@ -50,21 +50,21 @@ def is_kebab_case(name: str) -> bool: """for-example""" if not name: return False - return name.islower() and not "_" in name and not " " in name + return name.islower() and "_" not in name and " " not in name def is_snake_case(name: str) -> bool: """for_example""" if not name: return False - return name.islower() and not "-" in name and not " " in name + return name.islower() and "-" not in name and " " not in name def is_screaming_snake_case(name: str) -> bool: """FOR_EXAMPLE""" if not name: return False - return name.isupper() and not "-" in name and not " " in name + return name.isupper() and "-" not in name and " " not in name def kebab_case_to_camel_case_u(line: str) -> str: @@ -170,7 +170,7 @@ def is_disabled(self, line: str, prefix_comment="//") -> bool: """Detect whether the test is explicitly disabled.""" prefix_disable = "o2-linter: disable=" for prefix in [prefix_comment, prefix_disable]: - if not prefix in line: + if prefix not in line: return False line = line[(line.index(prefix) + len(prefix)) :] # Strip away part before prefix. if self.name in line: @@ -289,7 +289,7 @@ def test_line(self, line: str) -> bool: matches = [(it.start(), it.group()) for it in iterators] if not matches: continue - if not '"' in line: # Found a match which cannot be inside a string. + if '"' not in line: # Found a match which cannot be inside a string. return False # Ignore matches inside strings. for match in matches: @@ -309,7 +309,7 @@ class TestROOT(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and not "Macros/" in path + return TestSpec.file_matches(self, path) and "Macros/" not in path def test_line(self, line: str) -> bool: pattern = r"TMath|(U?(Int|Char|Short)|Double(32)?|Float(16)?|U?Long(64)?|Bool)_t" @@ -327,7 +327,7 @@ class TestPi(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and not "Macros/" in path + return TestSpec.file_matches(self, path) and "Macros/" not in path def test_line(self, line: str) -> bool: pattern = r"M_PI|TMath::(Two)?Pi" @@ -379,13 +379,13 @@ class TestPdgDatabase(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and not "Macros/" in path + return TestSpec.file_matches(self, path) and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True line = remove_comment_cpp(line) - return not "TDatabasePDG" in line + return "TDatabasePDG" not in line class TestPdgCode(TestSpec): @@ -438,7 +438,7 @@ class TestLogging(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and not "Macros/" in path + return TestSpec.file_matches(self, path) and "Macros/" not in path def test_line(self, line: str) -> bool: pattern = r"^([Pp]rintf\(|(std::)?cout <)" @@ -734,7 +734,7 @@ def test_line(self, line: str) -> bool: return True line = remove_comment_cpp(line) words = line.split() - if not "constexpr" in words or not "=" in words: + if "constexpr" not in words or "=" not in words: return True # Extract constant name. words = words[: words.index("=")] # keep only words before "=" @@ -863,24 +863,24 @@ class TestNameEnum(TestNameUpperCamelCase): """Test names of enumerators.""" keyword = "enum" - name = f"name/enum" - message = f"Use UpperCamelCase for names of enumerators and their values." + name = "name/enum" + message = "Use UpperCamelCase for names of enumerators and their values." class TestNameClass(TestNameUpperCamelCase): """Test names of classes.""" keyword = "class" - name = f"name/class" - message = f"Use UpperCamelCase for names of classes." + name = "name/class" + message = "Use UpperCamelCase for names of classes." class TestNameStruct(TestNameUpperCamelCase): """Test names of structs.""" keyword = "struct" - name = f"name/struct" - message = f"Use UpperCamelCase for names of structs." + name = "name/struct" + message = "Use UpperCamelCase for names of structs." class TestNameFileCpp(TestSpec): @@ -1021,7 +1021,7 @@ def test_file(self, path: str, content) -> bool: # print(f"{i + 1}: Exiting adapt.") is_inside_adapt = False # Find explicit task name. - if not "TaskName{" in line: + if "TaskName{" not in line: continue passed = False # Extract explicit task name. @@ -1099,7 +1099,7 @@ def test_line(self, line: str) -> bool: if is_comment_cpp(line): return True line = remove_comment_cpp(line) - return not "auto const" in line + return "auto const" not in line class TestHfNameStructClass(TestSpec): @@ -1213,7 +1213,7 @@ class TestHfNameFileWorkflow(TestSpec): per_line = False def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path + return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path).rstrip(".cxx") @@ -1246,7 +1246,7 @@ class TestHfNameConfigurable(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path and not "Macros/" in path + return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): From 854dc91828e18596d4dd79df453b1761dc80b6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 27 Jul 2024 03:41:55 +0200 Subject: [PATCH 61/96] Break lines --- Scripts/o2_linter.py | 49 +++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 7e7d3d640e5..d8ee7e0e839 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -345,7 +345,8 @@ class TestTwoPiAddSubtract(TestSpec): suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: - pattern_two_pi = r"(2(\.0*f?)? \* (M_PI|TMath::Pi\(\)|(((o2::)?constants::)?math::)?PI)|(((o2::)?constants::)?math::)?TwoPI|TMath::TwoPi\(\))" + pattern_two_pi = r"(2(\.0*f?)? \* (M_PI|TMath::Pi\(\)|(((o2::)?constants::)?math::)?PI)|" \ + r"(((o2::)?constants::)?math::)?TwoPI|TMath::TwoPi\(\))" pattern = rf"[\+-]=? {pattern_two_pi}" if is_comment_cpp(line): return True @@ -375,7 +376,8 @@ class TestPdgDatabase(TestSpec): """Detect use of TDatabasePDG.""" name = "pdg/database" - message = "Direct use of TDatabasePDG is not allowed. Use o2::constants::physics::Mass... or Service." + message = "Direct use of TDatabasePDG is not allowed. " \ + "Use o2::constants::physics::Mass... or Service." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: @@ -522,7 +524,8 @@ class TestWorkflowOptions(TestSpec): """Detect usage of workflow options in defineDataProcessing. (Not supported on AliHyperloop.)""" name = "o2-workflow-options" - message = "Do not use workflow options to customise workflow topology composition in defineDataProcessing. Use process function switches or metadata instead." + message = "Do not use workflow options to customise workflow topology composition in defineDataProcessing. " \ + "Use process function switches or metadata instead." suffixes = [".cxx"] per_line = False @@ -957,7 +960,8 @@ def test_file(self, path: str, content) -> bool: path, i + 1, self.name, - f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. (Matches {expected_workflow_file_name}.)", + f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. " \ + f"(Matches {expected_workflow_file_name}.)", ) return passed @@ -1041,16 +1045,20 @@ def test_file(self, path: str, content) -> bool: device_name_from_task_name ) # struct name matching the TaskName if device_name_from_struct_name == device_name_from_task_name: - # If the task name results in the same device name as the struct name would, TaskName is redundant and should be removed. + # If the task name results in the same device name as the struct name would, + # TaskName is redundant and should be removed. print_error( path, i + 1, self.name, - f"Specified task name {task_name} and the struct name {struct_name} produce the same device name {device_name_from_struct_name}. TaskName is redundant.", + f"Specified task name {task_name} and the struct name {struct_name} produce " \ + f"the same device name {device_name_from_struct_name}. TaskName is redundant.", ) passed = False elif device_name_from_struct_name.replace("-", "") == device_name_from_task_name.replace("-", ""): - # If the device names generated from the task name and from the struct name differ in hyphenation, capitalisation of the struct name should be fixed and TaskName should be removed. (special cases: alice3-, -2prong) + # If the device names generated from the task name and from the struct name differ in hyphenation, + # capitalisation of the struct name should be fixed and TaskName should be removed. + # (special cases: alice3-, -2prong) print_error( path, i + 1, @@ -1059,24 +1067,34 @@ def test_file(self, path: str, content) -> bool: ) passed = False elif device_name_from_task_name.startswith(device_name_from_struct_name): - # If the device name generated from the task name is an extension of the device name generated from the struct name, accept it if the struct is templated. If the struct is not templated, extension is acceptable if adaptAnalysisTask is called multiple times for the same struct. + # If the device name generated from the task name is an extension of the device name generated + # from the struct name, accept it if the struct is templated. If the struct is not templated, + # extension is acceptable if adaptAnalysisTask is called multiple times for the same struct. if not struct_templated: print_error( path, i + 1, self.name, - f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} but the struct is not templated. Is it adapted multiple times?", + f"Device name {device_name_from_task_name} from the specified task name " \ + f"{task_name} is an extension of the device name {device_name_from_struct_name} " \ + f"from the struct name {struct_name} but the struct is not templated. " \ + "Is it adapted multiple times?", ) passed = False # else: - # print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from the specified task name {task_name} is an extension of the device name {device_name_from_struct_name} from the struct name {struct_name} and the struct is templated. All good") + # print_error(path, i + 1, self.name, f"Device name {device_name_from_task_name} from + # the specified task name {task_name} is an extension of the device name + # {device_name_from_struct_name} from the struct name {struct_name} and the struct is templated. + # All good") else: # Other cases should be rejected. print_error( path, i + 1, self.name, - f"Specified task name {task_name} produces device name {device_name_from_task_name} which does not match the device name {device_name_from_struct_name} from the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})", + f"Specified task name {task_name} produces device name {device_name_from_task_name} " \ + f"which does not match the device name {device_name_from_struct_name} from " \ + f"the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})", ) passed = False return passed @@ -1197,7 +1215,8 @@ def test_file(self, path: str, content) -> bool: path, first_line, self.name, - f"{struct_name}: {member.strip()} appears too early (before end of {self.member_order[index_last_member].strip()}).", + f"{struct_name}: {member.strip()} appears too early " \ + f"(before end of {self.member_order[index_last_member].strip()}).", ) last_line_last_member = last_line index_last_member = i_m @@ -1208,7 +1227,8 @@ class TestHfNameFileWorkflow(TestSpec): """PWGHF: Test names of workflow files.""" name = "pwghf/name/workflow-file" - message = "Name of a workflow file must match the name of the main task in it. See the PWGHF O2 naming conventions for details." + message = "Name of a workflow file must match the name of the main task in it. " \ + "See the PWGHF O2 naming conventions for details." suffixes = [".cxx"] per_line = False @@ -1242,7 +1262,8 @@ class TestHfNameConfigurable(TestSpec): """PWGHF: Test names of configurables.""" name = "pwghf/name/configurable" - message = "Use lowerCamelCase for names of configurables and use the same name for the struct member as for the JSON string." + message = "Use lowerCamelCase for names of configurables and use the same name " \ + "for the struct member as for the JSON string." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: From e4d582403e2facf536b4dc7e4e7b791882c8e951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 27 Jul 2024 03:44:31 +0200 Subject: [PATCH 62/96] Break lines + format --- Scripts/o2_linter.py | 51 +++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index d8ee7e0e839..820949ee70e 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -345,8 +345,10 @@ class TestTwoPiAddSubtract(TestSpec): suffixes = [".h", ".cxx"] def test_line(self, line: str) -> bool: - pattern_two_pi = r"(2(\.0*f?)? \* (M_PI|TMath::Pi\(\)|(((o2::)?constants::)?math::)?PI)|" \ + pattern_two_pi = ( + r"(2(\.0*f?)? \* (M_PI|TMath::Pi\(\)|(((o2::)?constants::)?math::)?PI)|" r"(((o2::)?constants::)?math::)?TwoPI|TMath::TwoPi\(\))" + ) pattern = rf"[\+-]=? {pattern_two_pi}" if is_comment_cpp(line): return True @@ -376,8 +378,10 @@ class TestPdgDatabase(TestSpec): """Detect use of TDatabasePDG.""" name = "pdg/database" - message = "Direct use of TDatabasePDG is not allowed. " \ + message = ( + "Direct use of TDatabasePDG is not allowed. " "Use o2::constants::physics::Mass... or Service." + ) suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: @@ -524,8 +528,10 @@ class TestWorkflowOptions(TestSpec): """Detect usage of workflow options in defineDataProcessing. (Not supported on AliHyperloop.)""" name = "o2-workflow-options" - message = "Do not use workflow options to customise workflow topology composition in defineDataProcessing. " \ + message = ( + "Do not use workflow options to customise workflow topology composition in defineDataProcessing. " "Use process function switches or metadata instead." + ) suffixes = [".cxx"] per_line = False @@ -960,8 +966,8 @@ def test_file(self, path: str, content) -> bool: path, i + 1, self.name, - f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. " \ - f"(Matches {expected_workflow_file_name}.)", + f"Workflow name {workflow_name} does not match its file name {workflow_file_name}. " + f"(Matches {expected_workflow_file_name}.)", ) return passed @@ -1051,8 +1057,8 @@ def test_file(self, path: str, content) -> bool: path, i + 1, self.name, - f"Specified task name {task_name} and the struct name {struct_name} produce " \ - f"the same device name {device_name_from_struct_name}. TaskName is redundant.", + f"Specified task name {task_name} and the struct name {struct_name} produce " + f"the same device name {device_name_from_struct_name}. TaskName is redundant.", ) passed = False elif device_name_from_struct_name.replace("-", "") == device_name_from_task_name.replace("-", ""): @@ -1063,7 +1069,10 @@ def test_file(self, path: str, content) -> bool: path, i + 1, self.name, - f"Device names {device_name_from_task_name} and {device_name_from_struct_name} generated from the specified task name {task_name} and from the struct name {struct_name}, respectively, differ in hyphenation. Consider fixing capitalisation of the struct name to {struct_name_from_device_name} and removing TaskName.", + f"Device names {device_name_from_task_name} and {device_name_from_struct_name} generated " + f"from the specified task name {task_name} and from the struct name {struct_name}, " + f"respectively, differ in hyphenation. Consider fixing capitalisation of the struct name " + f"to {struct_name_from_device_name} and removing TaskName.", ) passed = False elif device_name_from_task_name.startswith(device_name_from_struct_name): @@ -1075,10 +1084,10 @@ def test_file(self, path: str, content) -> bool: path, i + 1, self.name, - f"Device name {device_name_from_task_name} from the specified task name " \ - f"{task_name} is an extension of the device name {device_name_from_struct_name} " \ - f"from the struct name {struct_name} but the struct is not templated. " \ - "Is it adapted multiple times?", + f"Device name {device_name_from_task_name} from the specified task name " + f"{task_name} is an extension of the device name {device_name_from_struct_name} " + f"from the struct name {struct_name} but the struct is not templated. " + "Is it adapted multiple times?", ) passed = False # else: @@ -1092,9 +1101,9 @@ def test_file(self, path: str, content) -> bool: path, i + 1, self.name, - f"Specified task name {task_name} produces device name {device_name_from_task_name} " \ - f"which does not match the device name {device_name_from_struct_name} from " \ - f"the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})", + f"Specified task name {task_name} produces device name {device_name_from_task_name} " + f"which does not match the device name {device_name_from_struct_name} from " + f"the struct name {struct_name}. (Matching struct name {struct_name_from_device_name})", ) passed = False return passed @@ -1215,8 +1224,8 @@ def test_file(self, path: str, content) -> bool: path, first_line, self.name, - f"{struct_name}: {member.strip()} appears too early " \ - f"(before end of {self.member_order[index_last_member].strip()}).", + f"{struct_name}: {member.strip()} appears too early " + f"(before end of {self.member_order[index_last_member].strip()}).", ) last_line_last_member = last_line index_last_member = i_m @@ -1227,8 +1236,10 @@ class TestHfNameFileWorkflow(TestSpec): """PWGHF: Test names of workflow files.""" name = "pwghf/name/workflow-file" - message = "Name of a workflow file must match the name of the main task in it. " \ + message = ( + "Name of a workflow file must match the name of the main task in it. " "See the PWGHF O2 naming conventions for details." + ) suffixes = [".cxx"] per_line = False @@ -1262,8 +1273,10 @@ class TestHfNameConfigurable(TestSpec): """PWGHF: Test names of configurables.""" name = "pwghf/name/configurable" - message = "Use lowerCamelCase for names of configurables and use the same name " \ + message = ( + "Use lowerCamelCase for names of configurables and use the same name " "for the struct member as for the JSON string." + ) suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: From 3bad9395f28bbc4a0a01a4e003ed86136a92d3d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 27 Jul 2024 04:02:20 +0200 Subject: [PATCH 63/96] Update description --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 820949ee70e..423f7101ff6 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1308,7 +1308,7 @@ def test_line(self, line: str) -> bool: def main(): """Main function""" - parser = argparse.ArgumentParser(description="Run tests on files to find code issues.") + parser = argparse.ArgumentParser(description="O2 linter (Find O2-specific issues in O2 code)") parser.add_argument("paths", type=str, nargs="+", help="File path(s)") parser.add_argument( "-g", From 8e867ca6e64c7e36ca6d4fcca5ec0ffd03ca53f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 29 Jul 2024 10:46:30 +0200 Subject: [PATCH 64/96] Improve result report --- Scripts/o2_linter.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 423f7101ff6..8d6041f4563 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -25,6 +25,7 @@ from typing import Union github_mode = False # GitHub mode +prefix_disable = "o2-linter: disable=" def is_camel_case(name: str) -> bool: @@ -168,7 +169,6 @@ def file_matches(self, path: str) -> bool: def is_disabled(self, line: str, prefix_comment="//") -> bool: """Detect whether the test is explicitly disabled.""" - prefix_disable = "o2-linter: disable=" for prefix in [prefix_comment, prefix_disable]: if prefix not in line: return False @@ -1404,10 +1404,23 @@ def main(): sys.exit(1) # Report global result. + title_result = "O2 linter result" if passed: - print("All tests passed.") + msg_result = "All tests passed." + if github_mode: + print(f"::notice title={title_result}::{msg_result}") + else: + print(f"{title_result}: {msg_result}") else: - print("Issues have been found.") + msg_result = "Issues have been found." + msg_disable = f"You can disable a test for a line by adding a comment with \"{prefix_disable}\"" \ + " followed by the name of the test." + if github_mode: + print(f"::error title={title_result}::{msg_result}") + print(f"::notice::{msg_disable}") + else: + print(f"{title_result}: {msg_result}") + print(msg_disable) sys.exit(1) From 7a4e57ab5155b883a88a3f734d522db43a718609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 29 Jul 2024 17:34:57 +0200 Subject: [PATCH 65/96] TestNameTask: Check kebab-case of device name --- Scripts/o2_linter.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 8d6041f4563..f266e1facc0 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1050,7 +1050,16 @@ def test_file(self, path: str, content) -> bool: struct_name_from_device_name = kebab_case_to_camel_case_u( device_name_from_task_name ) # struct name matching the TaskName - if device_name_from_struct_name == device_name_from_task_name: + if not is_kebab_case(device_name_from_task_name): + print_error( + path, + i + 1, + self.name, + f"Specified task name {task_name} produces an invalid device name " + f"{device_name_from_task_name}.", + ) + passed = False + elif device_name_from_struct_name == device_name_from_task_name: # If the task name results in the same device name as the struct name would, # TaskName is redundant and should be removed. print_error( From ce46edc52b1e82fdf2c65a0b255f35bb2e6516d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 29 Jul 2024 18:19:59 +0200 Subject: [PATCH 66/96] Add TestNameType. TestNameTask: Check kebab-case. --- Scripts/o2_linter.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index f266e1facc0..db8062bf38c 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -547,7 +547,7 @@ def test_file(self, path: str, content) -> bool: line = remove_comment_cpp(line) # Wait for defineDataProcessing. if not is_inside_define: - if not re.match(r"(o2::)?(framework::)?WorkflowSpec defineDataProcessing\(", line): + if not re.match(r"((o2::)?framework::)?WorkflowSpec defineDataProcessing\(", line): continue # print(f"{i + 1}: Entering define.") is_inside_define = True @@ -844,6 +844,24 @@ def test_line(self, line: str) -> bool: return is_snake_case(namespace_name) +class TestNameType(TestSpec): + """Test names of defined types.""" + + name = "name/type" + message = "Use UpperCamelCase for names of defined types." + suffixes = [".h", ".cxx", ".C"] + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + if not (match := re.match(r"using (\w+) = ", line)): + return True + # Extract type name. + type_name = match.group(1) + # The actual test comes here. + return is_upper_camel_case(type_name) + + class TestNameUpperCamelCase(TestSpec): """Base class for a test of UpperCamelCase names.""" @@ -998,7 +1016,7 @@ def test_file(self, path: str, content) -> bool: line = remove_comment_cpp(line) # Wait for defineDataProcessing. if not is_inside_define: - if not re.match(r"(o2::)?(framework::)?WorkflowSpec defineDataProcessing\(", line): + if not re.match(r"((o2::)?framework::)?WorkflowSpec defineDataProcessing\(", line): continue # print(f"{i + 1}: Entering define.") is_inside_define = True @@ -1365,6 +1383,7 @@ def main(): tests.append(TestNameColumn()) tests.append(TestNameTable()) tests.append(TestNameNamespace()) + tests.append(TestNameType()) tests.append(TestNameEnum()) tests.append(TestNameClass()) tests.append(TestNameStruct()) From 88a9f46958c0cfd2f2118fce715f579c14381353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 30 Jul 2024 00:21:57 +0200 Subject: [PATCH 67/96] Format --- Scripts/o2_linter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index db8062bf38c..ee9c63e7e72 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1441,8 +1441,10 @@ def main(): print(f"{title_result}: {msg_result}") else: msg_result = "Issues have been found." - msg_disable = f"You can disable a test for a line by adding a comment with \"{prefix_disable}\"" \ + msg_disable = ( + f'You can disable a test for a line by adding a comment with "{prefix_disable}"' " followed by the name of the test." + ) if github_mode: print(f"::error title={title_result}::{msg_result}") print(f"::notice::{msg_disable}") From 8fb37a7380d98bb7b8866b3962d0713206fda6ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 30 Jul 2024 13:16:19 +0200 Subject: [PATCH 68/96] TestNameNamespace, TestNameMacro: Check underscores --- Scripts/o2_linter.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index ee9c63e7e72..70ffc120cbd 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -713,7 +713,7 @@ class TestNameMacro(TestSpec): """Test macro names.""" name = "name/macro" - message = "Use SCREAMING_SNAKE_CASE for names of macros." + message = "Use SCREAMING_SNAKE_CASE for names of macros. Leading and double underscores are not allowed." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -726,6 +726,10 @@ def test_line(self, line: str) -> bool: if "(" in macro_name: macro_name = macro_name.split("(")[0] # The actual test comes here. + if macro_name.startswith("_"): + return False + if "__" in macro_name: + return False return is_screaming_snake_case(macro_name) @@ -828,7 +832,7 @@ class TestNameNamespace(TestSpec): """Test names of namespaces.""" name = "name/namespace" - message = "Use snake_case for names of namespaces." + message = "Use snake_case for names of namespaces. Double underscores are not allowed." suffixes = [".h", ".cxx", ".C"] def test_line(self, line: str) -> bool: @@ -841,6 +845,8 @@ def test_line(self, line: str) -> bool: if namespace_name == "{": # ignore anonymous namespaces return True # The actual test comes here. + if "__" in namespace_name: + return False return is_snake_case(namespace_name) From e0b7d51caf61938159b599f4ee71d3bc9b5264d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Tue, 30 Jul 2024 14:33:59 +0200 Subject: [PATCH 69/96] Fix documentation --- Scripts/o2_linter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 70ffc120cbd..5016e669d6d 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -302,7 +302,7 @@ def test_line(self, line: str) -> bool: class TestROOT(TestSpec): - """Detect use of unnecessary ROOT entities.""" + """Detect unnecessary use of ROOT entities.""" name = "root-entity" message = "Consider replacing ROOT entities with STD C++ or O2 entities." @@ -357,7 +357,7 @@ def test_line(self, line: str) -> bool: class TestPiMultipleFraction(TestSpec): - """Detect multiplying/dividing of pi for existing equivalent constants.""" + """Detect multiples/fractions of pi for existing equivalent constants.""" name = "pi-multiple-fraction" message = "Consider using multiples/fractions of PI defined in o2::constants::math." From 80cb8eaca3b591f9341aa36d9c96512e4ce4214a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Wed, 7 Aug 2024 09:37:57 +0200 Subject: [PATCH 70/96] TestHfStructMembers: Add PresliceUnsorted --- Scripts/o2_linter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 5016e669d6d..d375065489b 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -95,7 +95,7 @@ def camel_case_to_kebab_case(line: str) -> str: def print_error(path: str, line: Union[int, None], title: str, message: str): """Format and print error message.""" - # return # Use to suppress error message when counting speed. + # return # Use to suppress error messages. str_line = "" if line is None else f"{line}:" print(f"{path}:{str_line} {message} [{title}]") # terminal format if github_mode: @@ -1206,6 +1206,7 @@ class TestHfStructMembers(TestSpec): "using ", "Filter ", "Preslice<", + "PresliceUnsorted<", "Partition<", "ConfigurableAxis ", "AxisSpec ", From 6f40554c34d0c882404830e29ea5eba15661c9bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Wed, 14 Aug 2024 09:00:59 +0200 Subject: [PATCH 71/96] TestHfNameStructClass: Exclude Macros. TestHfNameFileWorkflow: Check Tasks. --- Scripts/o2_linter.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index d375065489b..6a30663deb8 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -25,7 +25,7 @@ from typing import Union github_mode = False # GitHub mode -prefix_disable = "o2-linter: disable=" +prefix_disable = "o2-linter: disable=" # prefix for disabling tests def is_camel_case(name: str) -> bool: @@ -120,7 +120,7 @@ def block_ranges(line: str, char_open: str, char_close: str) -> "list[list[int]] """Get list of index ranges of longest blocks opened with char_open and closed with char_close.""" # print(f"Looking for {char_open}{char_close} blocks in \"{line}\".") # print(line) - list_ranges: list[list[int]] = [] + list_ranges: "list[list[int]]" = [] if not all((line, len(char_open) == 1, len(char_close) == 1)): return list_ranges @@ -537,7 +537,7 @@ class TestWorkflowOptions(TestSpec): def test_file(self, path: str, content) -> bool: is_inside_define = False # Are we inside defineDataProcessing? - for i, line in enumerate(content): + for i, line in enumerate(content): # pylint: disable=unused-variable if not line.strip(): continue if self.is_disabled(line): @@ -1170,7 +1170,7 @@ class TestHfNameStructClass(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path + return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): @@ -1284,6 +1284,8 @@ def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path).rstrip(".cxx") if file_name[0].isupper(): return False + if "/Tasks/" in path and not file_name.startswith("task"): + return False base_struct_name = f"Hf{file_name[0].upper()}{file_name[1:]}" # expected base of struct names # print(f"For file {file_name} expecting to find {base_struct_name}.") struct_names = [] # actual struct names in the file @@ -1352,7 +1354,7 @@ def main(): ) args = parser.parse_args() if args.github: - global github_mode + global github_mode # pylint: disable=global-statement github_mode = True tests = [] # list of activated tests @@ -1427,7 +1429,7 @@ def main(): # print(f"Skipping path \"{path}\".") continue try: - with open(path, "r") as file: + with open(path, "r", encoding="utf-8") as file: content = file.readlines() for test in tests: result = test.run(path, content) From d73e7c049e0658f8b1be2ec270b42e1a68a3e25b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 23 Nov 2024 15:36:45 +0100 Subject: [PATCH 72/96] Use set comprehension --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 6a30663deb8..b0bd050c0c1 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1411,7 +1411,7 @@ def main(): tests.append(TestHfNameConfigurable()) test_names = [t.name for t in tests] # short names of activated tests - suffixes = tuple(set([s for test in tests for s in test.suffixes])) # all suffixes from all enabled tests + suffixes = tuple({s for test in tests for s in test.suffixes}) # all suffixes from all enabled tests passed = True # global result of all tests # Report overview before running. From af8f7dc95a7ec3005bfd786b75eff28cc12d83bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Sat, 23 Nov 2024 16:54:43 +0100 Subject: [PATCH 73/96] Add counters and result summary --- Scripts/o2_linter.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index b0bd050c0c1..1410932c79e 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -162,6 +162,7 @@ class TestSpec(ABC): message = "Test failed" # error message suffixes: "list[str]" = [] # suffixes of files to test per_line = True # Test lines separately one by one. + n_issues = 0 # issue counter def file_matches(self, path: str) -> bool: """Test whether the path matches the pattern for files to test.""" @@ -202,10 +203,12 @@ def run(self, path: str, content) -> bool: continue if not self.test_line(line): passed = False + self.n_issues += 1 print_error(path, i + 1, self.name, self.message) else: passed = self.test_file(path, content) if not passed: + self.n_issues += 1 print_error(path, None, self.name, self.message) return passed @@ -1413,6 +1416,7 @@ def main(): test_names = [t.name for t in tests] # short names of activated tests suffixes = tuple({s for test in tests for s in test.suffixes}) # all suffixes from all enabled tests passed = True # global result of all tests + n_files_bad = {name: 0 for name in test_names} # counter of files with issues # Report overview before running. print(f"Testing {len(args.paths)} files.") @@ -1434,12 +1438,19 @@ def main(): for test in tests: result = test.run(path, content) if not result: + n_files_bad[test.name] += 1 passed = False # print(f"File \"{path}\" {'passed' if result else 'failed'} the test {test.name}.") except IOError: print(f'Failed to open file "{path}".') sys.exit(1) + # Report results per test. + print("\nResults per test") + print("test\tissues\tbad files") + for test in tests: + print(f"{test.name}\t{test.n_issues}\t{n_files_bad[test.name]}") + # Report global result. title_result = "O2 linter result" if passed: @@ -1458,7 +1469,7 @@ def main(): print(f"::error title={title_result}::{msg_result}") print(f"::notice::{msg_disable}") else: - print(f"{title_result}: {msg_result}") + print(f"\n{title_result}: {msg_result}") print(msg_disable) sys.exit(1) From 11a617466c564488dab16e0ccfe8e22a25b8ee05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 25 Nov 2024 17:03:08 +0100 Subject: [PATCH 74/96] TestNameTable: Support versioned tables --- Scripts/o2_linter.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 1410932c79e..698bd055924 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -816,7 +816,7 @@ def test_line(self, line: str) -> bool: return True if not (match := re.match(r"DECLARE(_[A-Z]+)*_TABLES?(_[A-Z]+)*\(", line)): return True - # Extract names of the column type and getter. + # Extract names of the table type. line = remove_comment_cpp(line) line = line[len(match.group()) :].strip() # Extract part after "(". if not (match := re.match(r"([^,\) ]+)", line)): @@ -825,6 +825,11 @@ def test_line(self, line: str) -> bool: table_type_name = match.group(1) # print(f"Got \"{table_type_name}\"") # return True + # Check for a version suffix. + if match := re.match(r"(.*)_([0-9]{3})", line): + table_type_name = match.group(1) + # table_version = match.group(2) + # print(f"Got versioned table \"{table_type_name}\", version {table_version}") if table_type_name[0] == "_": # probably a macro variable return True # The actual test comes here. From b3c7ceeb6bb40f12109c9810e7e16bb5c7493ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 25 Nov 2024 17:28:17 +0100 Subject: [PATCH 75/96] Drop TestHfConstAuto --- Scripts/o2_linter.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 698bd055924..ed5e34fb21e 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1153,23 +1153,6 @@ def test_file(self, path: str, content) -> bool: # PWG-HF -class TestHfConstAuto(TestSpec): - """PWGHF: Detect swapped const auto.""" - - name = "pwghf/const-auto" - message = 'Use "const auto" instead of "auto const".' - suffixes = [".h", ".cxx"] - - def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path - - def test_line(self, line: str) -> bool: - if is_comment_cpp(line): - return True - line = remove_comment_cpp(line) - return "auto const" not in line - - class TestHfNameStructClass(TestSpec): """PWGHF: Test names of structs and classes.""" @@ -1412,7 +1395,6 @@ def main(): # PWG-HF enable_pwghf = True if enable_pwghf: - tests.append(TestHfConstAuto()) tests.append(TestHfNameStructClass()) tests.append(TestHfStructMembers()) tests.append(TestHfNameFileWorkflow()) From e7ec0449e00ad39ce28f5105d8aaabceb8c11f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 25 Nov 2024 17:48:10 +0100 Subject: [PATCH 76/96] TestHfStructMembers: Use items to iterate --- Scripts/o2_linter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index ed5e34fb21e..86b4a1e6969 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1212,7 +1212,7 @@ def file_matches(self, path: str) -> bool: def test_file(self, path: str, content) -> bool: passed = True - dic_struct: dict[str, dict] = {} + dic_structs: dict[str, dict] = {} struct_name = "" for i, line in enumerate(content): if is_comment_cpp(line): @@ -1220,27 +1220,27 @@ def test_file(self, path: str, content) -> bool: if line.startswith("struct "): # expecting no indentation line = remove_comment_cpp(line) struct_name = line.strip().split()[1] - dic_struct[struct_name] = {} + dic_structs[struct_name] = {} continue if not struct_name: continue # Save line numbers of members of the current struct for each category. for member in self.member_order: if line.startswith(f" {member}"): # expecting single-level indentation for direct members - if member not in dic_struct[struct_name]: - dic_struct[struct_name][member] = [] - dic_struct[struct_name][member].append(i + 1) # save line number + if member not in dic_structs[struct_name]: + dic_structs[struct_name][member] = [] + dic_structs[struct_name][member].append(i + 1) # save line number break # print(dic_struct) # Detect members declared in a wrong order. last_line_last_member = 0 # line number of the last member of the previous member category index_last_member = 0 # index of the previous member category in the member_order list - for struct_name in dic_struct: + for struct_name, dic_struct in dic_structs.items(): for i_m, member in enumerate(self.member_order): - if member not in dic_struct[struct_name]: + if member not in dic_struct: continue - first_line = min(dic_struct[struct_name][member]) # line number of the first member of this category - last_line = max(dic_struct[struct_name][member]) # line number of the last member of this category + first_line = min(dic_struct[member]) # line number of the first member of this category + last_line = max(dic_struct[member]) # line number of the last member of this category if ( first_line < last_line_last_member ): # The current category starts before the end of the previous category. From f862316e3c113aeb57c2398b19debb56ff8c53b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 25 Nov 2024 18:27:14 +0100 Subject: [PATCH 77/96] Add tips. Pin Ubuntu version. Call python3. --- .github/workflows/o2-linter.yml | 5 +++-- Scripts/o2_linter.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/o2-linter.yml b/.github/workflows/o2-linter.yml index f6127db25cc..3f124b56750 100644 --- a/.github/workflows/o2-linter.yml +++ b/.github/workflows/o2-linter.yml @@ -14,7 +14,7 @@ concurrency: jobs: o2-linter: name: O2 linter - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout Code uses: actions/checkout@v4 @@ -26,4 +26,5 @@ jobs: readarray -t files < <(git diff --diff-filter d --name-only origin/${{ env.MAIN_BRANCH }}...) [ ${{ github.event_name }} == 'pull_request' ] && options="-g" # shellcheck disable=SC2086 # Ignore unquoted options. - python Scripts/o2_linter.py $options "${files[@]}" + python3 Scripts/o2_linter.py $options "${files[@]}" + echo "Tip: If you allow actions in your fork repository, O2 linter will run when you push commits." diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 86b4a1e6969..3c4d1cb29f8 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1458,6 +1458,9 @@ def main(): else: print(f"\n{title_result}: {msg_result}") print(msg_disable) + # Print tips. + print("\nTip: You can run the O2 linter locally with: python3 Scripts/o2_linter.py ") + if not passed: sys.exit(1) From d24ed0f61f1be2e8c684a2cccb87e5f784540c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 25 Nov 2024 21:27:37 +0100 Subject: [PATCH 78/96] Improve report spacing --- Scripts/o2_linter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 3c4d1cb29f8..73eda0dae55 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1434,9 +1434,10 @@ def main(): # Report results per test. print("\nResults per test") - print("test\tissues\tbad files") + len_max = max(len(name) for name in test_names) + print(f"test{' ' * (len_max - len('test'))}\tissues\tbad files") for test in tests: - print(f"{test.name}\t{test.n_issues}\t{n_files_bad[test.name]}") + print(f"{test.name}{' ' * (len_max - len(test.name))}\t{test.n_issues}\t{n_files_bad[test.name]}") # Report global result. title_result = "O2 linter result" From 2b60e774d7819635d6151b670ea2977c46272a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Mon, 25 Nov 2024 21:36:37 +0100 Subject: [PATCH 79/96] Use warnings instead of errors --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 73eda0dae55..8afde3f9c68 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -100,7 +100,7 @@ def print_error(path: str, line: Union[int, None], title: str, message: str): print(f"{path}:{str_line} {message} [{title}]") # terminal format if github_mode: str_line = "" if line is None else f",line={line}" - print(f"::error file={path}{str_line},title=[{title}]::{message}") # GitHub annotation format + print(f"::warning file={path}{str_line},title=[{title}]::{message}") # GitHub annotation format def is_comment_cpp(line: str) -> bool: From 27c9c112da400dd8566ba0ba8fe448b0deee5f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Wed, 27 Nov 2024 14:22:12 +0100 Subject: [PATCH 80/96] Split TestHfNameFileWorkflow into TestNameFileWorkflow and TestHfNameFileWorkflowTask --- Scripts/o2_linter.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 8afde3f9c68..ffd7783a316 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1257,13 +1257,12 @@ def test_file(self, path: str, content) -> bool: return passed -class TestHfNameFileWorkflow(TestSpec): - """PWGHF: Test names of workflow files.""" +class TestHfNameFileWorkflowTask(TestSpec): + """PWGHF: Test names of task workflow files.""" - name = "pwghf/name/workflow-file" + name = "pwghf/name/workflow-file-task" message = ( - "Name of a workflow file must match the name of the main task in it. " - "See the PWGHF O2 naming conventions for details." + 'Name of a PWGHF task workflow file must start with "task". ' ) suffixes = [".cxx"] per_line = False @@ -1272,12 +1271,31 @@ def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path def test_file(self, path: str, content) -> bool: - file_name = os.path.basename(path).rstrip(".cxx") - if file_name[0].isupper(): - return False + file_name = os.path.basename(path) if "/Tasks/" in path and not file_name.startswith("task"): return False - base_struct_name = f"Hf{file_name[0].upper()}{file_name[1:]}" # expected base of struct names + return True + + +class TestNameFileWorkflow(TestSpec): + """Test names of workflow files.""" + + name = "name/workflow-file" + message = ( + "Name of a workflow file must match the name of the main task in it (without the PWG prefix). " + '(Class implementation files should be in "Core" directories.)' + ) + suffixes = [".cxx"] + per_line = False + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "/Core/" not in path + + def test_file(self, path: str, content) -> bool: + file_name = os.path.basename(path).rstrip(".cxx") + base_struct_name = f"{file_name[0].upper()}{file_name[1:]}" # expected base of struct names + if "PWGHF/" in path: + base_struct_name = "Hf" + base_struct_name # print(f"For file {file_name} expecting to find {base_struct_name}.") struct_names = [] # actual struct names in the file for line in content: @@ -1397,7 +1415,8 @@ def main(): if enable_pwghf: tests.append(TestHfNameStructClass()) tests.append(TestHfStructMembers()) - tests.append(TestHfNameFileWorkflow()) + tests.append(TestHfNameFileWorkflowTask()) + tests.append(TestNameFileWorkflow()) tests.append(TestHfNameConfigurable()) test_names = [t.name for t in tests] # short names of activated tests From 4e584d6f8260d238b047a8a81ddb3220af4314eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 01:42:46 +0100 Subject: [PATCH 81/96] Sort suffixes --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index ffd7783a316..486930f5356 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1428,7 +1428,7 @@ def main(): print(f"Testing {len(args.paths)} files.") # print(args.paths) print("Enabled tests:", test_names) - print("Suffixes of tested files:", suffixes) + print("Suffixes of tested files:", sorted(suffixes)) # print(f"Github annotations: {github_mode}.") # Test files. From e55e84a29dff7e206be115cb40d1b2b826961a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 01:44:13 +0100 Subject: [PATCH 82/96] Replace task with struct --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 486930f5356..e9d7fec2833 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1282,7 +1282,7 @@ class TestNameFileWorkflow(TestSpec): name = "name/workflow-file" message = ( - "Name of a workflow file must match the name of the main task in it (without the PWG prefix). " + "Name of a workflow file must match the name of the main struct in it (without the PWG prefix). " '(Class implementation files should be in "Core" directories.)' ) suffixes = [".cxx"] From 0a9165e6d28bc1693e9a019391e5051eb69cc3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 03:13:07 +0100 Subject: [PATCH 83/96] TestROOT: Improve pattern and message --- Scripts/o2_linter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index e9d7fec2833..18ca539fb31 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -308,14 +308,17 @@ class TestROOT(TestSpec): """Detect unnecessary use of ROOT entities.""" name = "root-entity" - message = "Consider replacing ROOT entities with STD C++ or O2 entities." + message = "Consider replacing ROOT entities with equivalents from standard C++ or from O2." suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: return TestSpec.file_matches(self, path) and "Macros/" not in path def test_line(self, line: str) -> bool: - pattern = r"TMath|(U?(Int|Char|Short)|Double(32)?|Float(16)?|U?Long(64)?|Bool)_t" + pattern = ( + r"TMath::(Abs|Sqrt|Power|Min|Max|Log(2|10)?|Exp|A?(Sin|Cos|Tan)H?|ATan2|Erfc?|Hypot)\(|" + r"(U?(Int|Char|Short)|Double(32)?|Float(16)?|U?Long(64)?|Bool)_t" + ) if is_comment_cpp(line): return True line = remove_comment_cpp(line) From 25b33aedba56dc30e7ce088fdd2046a1a21868e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 03:39:27 +0100 Subject: [PATCH 84/96] TestStdPrefix: Improve patterns --- Scripts/o2_linter.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 18ca539fb31..b79bc4d87c3 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -274,13 +274,12 @@ class TestStdPrefix(TestSpec): r"pow\(", r"min\(", r"max\(", - r"log\(", + r"log(2|10)?\(", r"exp\(", - r"sin\(", - r"cos\(", - r"tan\(", - r"atan\(", + r"a?(sin|cos|tan)h?\(", r"atan2\(", + r"erfc?\(", + r"hypot\(", ] def test_line(self, line: str) -> bool: From c5a024fdcdac7bf2e43cf44548b71004e46c3e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 04:02:29 +0100 Subject: [PATCH 85/96] TestNameFileWorkflow: Allow disabling --- Scripts/o2_linter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index b79bc4d87c3..6df784929e8 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1264,7 +1264,7 @@ class TestHfNameFileWorkflowTask(TestSpec): name = "pwghf/name/workflow-file-task" message = ( - 'Name of a PWGHF task workflow file must start with "task". ' + 'Name of a PWGHF task workflow file must start with "task".' ) suffixes = [".cxx"] per_line = False @@ -1301,6 +1301,8 @@ def test_file(self, path: str, content) -> bool: # print(f"For file {file_name} expecting to find {base_struct_name}.") struct_names = [] # actual struct names in the file for line in content: + if self.is_disabled(line): + return True if not line.startswith("struct "): continue # Extract struct name. From bee0d13feb255acdb4fac7213fc968e48474f656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 04:04:45 +0100 Subject: [PATCH 86/96] Rename TestHfNameFileWorkflowTask to TestHfNameFileTask --- Scripts/o2_linter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 6df784929e8..2bfdacf2d2d 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1259,10 +1259,10 @@ def test_file(self, path: str, content) -> bool: return passed -class TestHfNameFileWorkflowTask(TestSpec): +class TestHfNameFileTask(TestSpec): """PWGHF: Test names of task workflow files.""" - name = "pwghf/name/workflow-file-task" + name = "pwghf/name/task-file" message = ( 'Name of a PWGHF task workflow file must start with "task".' ) @@ -1419,7 +1419,7 @@ def main(): if enable_pwghf: tests.append(TestHfNameStructClass()) tests.append(TestHfStructMembers()) - tests.append(TestHfNameFileWorkflowTask()) + tests.append(TestHfNameFileTask()) tests.append(TestNameFileWorkflow()) tests.append(TestHfNameConfigurable()) From b358b4b4086c6d48f4ac0ced7c435322cb62d548 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 04:36:32 +0100 Subject: [PATCH 87/96] TestHfNameConfigurable: Generalise to TestNameConfigurable --- Scripts/o2_linter.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 2bfdacf2d2d..9a2e1efd57a 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1318,18 +1318,18 @@ def test_file(self, path: str, content) -> bool: return False -class TestHfNameConfigurable(TestSpec): - """PWGHF: Test names of configurables.""" +class TestNameConfigurable(TestSpec): + """Test names of configurables.""" - name = "pwghf/name/configurable" + name = "name/configurable" message = ( "Use lowerCamelCase for names of configurables and use the same name " - "for the struct member as for the JSON string." + "for the struct member as for the JSON string. (Declare the type and names on the same line.)" ) suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path + return TestSpec.file_matches(self, path) and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): @@ -1338,13 +1338,19 @@ def test_line(self, line: str) -> bool: return True # Extract Configurable name. words = line.split() - if words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", + if len(words) < 2: + return False + if len(words) > 2 and words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", name_cpp = words[1] # nameCpp name_json = words[3][1:] # expecting "nameJson", else: names = words[1].split("{") # expecting Configurable... nameCpp{"nameJson", + if len(names) < 2: + return False name_cpp = names[0] # nameCpp name_json = names[1] # expecting "nameJson", + if not name_json: + return False if name_json[0] != '"': # JSON name is not a literal string. return True name_json = name_json.strip('",') # expecting nameJson @@ -1421,7 +1427,7 @@ def main(): tests.append(TestHfStructMembers()) tests.append(TestHfNameFileTask()) tests.append(TestNameFileWorkflow()) - tests.append(TestHfNameConfigurable()) + tests.append(TestNameConfigurable()) test_names = [t.name for t in tests] # short names of activated tests suffixes = tuple({s for test in tests for s in test.suffixes}) # all suffixes from all enabled tests From b87d09f1f28f8151539e1250dec36a2bcc75d0ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 04:44:48 +0100 Subject: [PATCH 88/96] Adjust test order --- Scripts/o2_linter.py | 204 +++++++++++++++++++++---------------------- 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 9a2e1efd57a..09aa9d59487 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1152,6 +1152,85 @@ def test_file(self, path: str, content) -> bool: return passed +class TestNameFileWorkflow(TestSpec): + """Test names of workflow files.""" + + name = "name/workflow-file" + message = ( + "Name of a workflow file must match the name of the main struct in it (without the PWG prefix). " + '(Class implementation files should be in "Core" directories.)' + ) + suffixes = [".cxx"] + per_line = False + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "/Core/" not in path + + def test_file(self, path: str, content) -> bool: + file_name = os.path.basename(path).rstrip(".cxx") + base_struct_name = f"{file_name[0].upper()}{file_name[1:]}" # expected base of struct names + if "PWGHF/" in path: + base_struct_name = "Hf" + base_struct_name + # print(f"For file {file_name} expecting to find {base_struct_name}.") + struct_names = [] # actual struct names in the file + for line in content: + if self.is_disabled(line): + return True + if not line.startswith("struct "): + continue + # Extract struct name. + words = line.split() + if not words[1].isalnum(): # "struct : ..." + continue + struct_name = words[1] + struct_names.append(struct_name) + # print(f"Found structs: {struct_names}.") + for struct_name in struct_names: + if struct_name.startswith(base_struct_name): + return True + return False + + +class TestNameConfigurable(TestSpec): + """Test names of configurables.""" + + name = "name/configurable" + message = ( + "Use lowerCamelCase for names of configurables and use the same name " + "for the struct member as for the JSON string. (Declare the type and names on the same line.)" + ) + suffixes = [".h", ".cxx"] + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "Macros/" not in path + + def test_line(self, line: str) -> bool: + if is_comment_cpp(line): + return True + if not line.startswith("Configurable"): + return True + # Extract Configurable name. + words = line.split() + if len(words) < 2: + return False + if len(words) > 2 and words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", + name_cpp = words[1] # nameCpp + name_json = words[3][1:] # expecting "nameJson", + else: + names = words[1].split("{") # expecting Configurable... nameCpp{"nameJson", + if len(names) < 2: + return False + name_cpp = names[0] # nameCpp + name_json = names[1] # expecting "nameJson", + if not name_json: + return False + if name_json[0] != '"': # JSON name is not a literal string. + return True + name_json = name_json.strip('",') # expecting nameJson + # The actual test comes here. + return is_lower_camel_case(name_cpp) and name_cpp == name_json + + # PWG-HF @@ -1180,6 +1259,26 @@ def test_line(self, line: str) -> bool: return struct_name.startswith("Hf") +class TestHfNameFileTask(TestSpec): + """PWGHF: Test names of task workflow files.""" + + name = "pwghf/name/task-file" + message = ( + 'Name of a PWGHF task workflow file must start with "task".' + ) + suffixes = [".cxx"] + per_line = False + + def file_matches(self, path: str) -> bool: + return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path + + def test_file(self, path: str, content) -> bool: + file_name = os.path.basename(path) + if "/Tasks/" in path and not file_name.startswith("task"): + return False + return True + + class TestHfStructMembers(TestSpec): """PWGHF: Test order of struct members. Caveat: Does not see Configurables in ConfigurableGroup.""" @@ -1259,105 +1358,6 @@ def test_file(self, path: str, content) -> bool: return passed -class TestHfNameFileTask(TestSpec): - """PWGHF: Test names of task workflow files.""" - - name = "pwghf/name/task-file" - message = ( - 'Name of a PWGHF task workflow file must start with "task".' - ) - suffixes = [".cxx"] - per_line = False - - def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path - - def test_file(self, path: str, content) -> bool: - file_name = os.path.basename(path) - if "/Tasks/" in path and not file_name.startswith("task"): - return False - return True - - -class TestNameFileWorkflow(TestSpec): - """Test names of workflow files.""" - - name = "name/workflow-file" - message = ( - "Name of a workflow file must match the name of the main struct in it (without the PWG prefix). " - '(Class implementation files should be in "Core" directories.)' - ) - suffixes = [".cxx"] - per_line = False - - def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "/Core/" not in path - - def test_file(self, path: str, content) -> bool: - file_name = os.path.basename(path).rstrip(".cxx") - base_struct_name = f"{file_name[0].upper()}{file_name[1:]}" # expected base of struct names - if "PWGHF/" in path: - base_struct_name = "Hf" + base_struct_name - # print(f"For file {file_name} expecting to find {base_struct_name}.") - struct_names = [] # actual struct names in the file - for line in content: - if self.is_disabled(line): - return True - if not line.startswith("struct "): - continue - # Extract struct name. - words = line.split() - if not words[1].isalnum(): # "struct : ..." - continue - struct_name = words[1] - struct_names.append(struct_name) - # print(f"Found structs: {struct_names}.") - for struct_name in struct_names: - if struct_name.startswith(base_struct_name): - return True - return False - - -class TestNameConfigurable(TestSpec): - """Test names of configurables.""" - - name = "name/configurable" - message = ( - "Use lowerCamelCase for names of configurables and use the same name " - "for the struct member as for the JSON string. (Declare the type and names on the same line.)" - ) - suffixes = [".h", ".cxx"] - - def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "Macros/" not in path - - def test_line(self, line: str) -> bool: - if is_comment_cpp(line): - return True - if not line.startswith("Configurable"): - return True - # Extract Configurable name. - words = line.split() - if len(words) < 2: - return False - if len(words) > 2 and words[2] == "=": # expecting Configurable... nameCpp = {"nameJson", - name_cpp = words[1] # nameCpp - name_json = words[3][1:] # expecting "nameJson", - else: - names = words[1].split("{") # expecting Configurable... nameCpp{"nameJson", - if len(names) < 2: - return False - name_cpp = names[0] # nameCpp - name_json = names[1] # expecting "nameJson", - if not name_json: - return False - if name_json[0] != '"': # JSON name is not a literal string. - return True - name_json = name_json.strip('",') # expecting nameJson - # The actual test comes here. - return is_lower_camel_case(name_cpp) and name_cpp == name_json - - # End of test implementations @@ -1419,15 +1419,15 @@ def main(): tests.append(TestNameFilePython()) tests.append(TestNameWorkflow()) tests.append(TestNameTask()) + tests.append(TestNameFileWorkflow()) + tests.append(TestNameConfigurable()) # PWG-HF enable_pwghf = True if enable_pwghf: tests.append(TestHfNameStructClass()) - tests.append(TestHfStructMembers()) tests.append(TestHfNameFileTask()) - tests.append(TestNameFileWorkflow()) - tests.append(TestNameConfigurable()) + tests.append(TestHfStructMembers()) test_names = [t.name for t in tests] # short names of activated tests suffixes = tuple({s for test in tests for s in test.suffixes}) # all suffixes from all enabled tests From 493799ea0862afcab40f3819b6b62f1d4a88dc27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 04:51:11 +0100 Subject: [PATCH 89/96] Use super --- Scripts/o2_linter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 09aa9d59487..7e9aa64a0cf 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -311,7 +311,7 @@ class TestROOT(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "Macros/" not in path + return super().file_matches(path) and "Macros/" not in path def test_line(self, line: str) -> bool: pattern = ( @@ -332,7 +332,7 @@ class TestPi(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "Macros/" not in path + return super().file_matches(path) and "Macros/" not in path def test_line(self, line: str) -> bool: pattern = r"M_PI|TMath::(Two)?Pi" @@ -390,7 +390,7 @@ class TestPdgDatabase(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "Macros/" not in path + return super().file_matches(path) and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): @@ -449,7 +449,7 @@ class TestLogging(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "Macros/" not in path + return super().file_matches(path) and "Macros/" not in path def test_line(self, line: str) -> bool: pattern = r"^([Pp]rintf\(|(std::)?cout <)" @@ -1164,7 +1164,7 @@ class TestNameFileWorkflow(TestSpec): per_line = False def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "/Core/" not in path + return super().file_matches(path) and "/Core/" not in path def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path).rstrip(".cxx") @@ -1202,7 +1202,7 @@ class TestNameConfigurable(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "Macros/" not in path + return super().file_matches(path) and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): @@ -1242,7 +1242,7 @@ class TestHfNameStructClass(TestSpec): suffixes = [".h", ".cxx"] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path + return super().file_matches(path) and "PWGHF/" in path and "Macros/" not in path def test_line(self, line: str) -> bool: if is_comment_cpp(line): @@ -1270,7 +1270,7 @@ class TestHfNameFileTask(TestSpec): per_line = False def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path and "Macros/" not in path + return super().file_matches(path) and "PWGHF/" in path and "Macros/" not in path def test_file(self, path: str, content) -> bool: file_name = os.path.basename(path) @@ -1309,7 +1309,7 @@ class TestHfStructMembers(TestSpec): ] def file_matches(self, path: str) -> bool: - return TestSpec.file_matches(self, path) and "PWGHF/" in path + return super().file_matches(path) and "PWGHF/" in path def test_file(self, path: str, content) -> bool: passed = True From a2fb4d78ffa6143b6c2f5f91f20baaa765995926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 05:05:32 +0100 Subject: [PATCH 90/96] Format --- Scripts/o2_linter.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 7e9aa64a0cf..e53963dad32 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1263,9 +1263,7 @@ class TestHfNameFileTask(TestSpec): """PWGHF: Test names of task workflow files.""" name = "pwghf/name/task-file" - message = ( - 'Name of a PWGHF task workflow file must start with "task".' - ) + message = ('Name of a PWGHF task workflow file must start with "task".') suffixes = [".cxx"] per_line = False From e065351c256d43b8dce06dc7bc2f0c3d352f708f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 05:10:21 +0100 Subject: [PATCH 91/96] Format --- Scripts/o2_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index e53963dad32..b040fa5d7b8 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1263,7 +1263,7 @@ class TestHfNameFileTask(TestSpec): """PWGHF: Test names of task workflow files.""" name = "pwghf/name/task-file" - message = ('Name of a PWGHF task workflow file must start with "task".') + message = 'Name of a PWGHF task workflow file must start with "task".' suffixes = [".cxx"] per_line = False From 406d3aebe2b4f542b92eb475340c1c7596992c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 13:18:56 +0100 Subject: [PATCH 92/96] TestUsingDirectives: Allow scoped using directives --- Scripts/o2_linter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index b040fa5d7b8..1565d0a2e51 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -195,7 +195,8 @@ def run(self, path: str, content) -> bool: # print(f"Running test {self.name} for {path} with {len(content)} lines") if self.per_line: for i, line in enumerate(content): - line = line.strip() + if not isinstance(self, TestUsingDirectives): # Keep the indentation if needed. + line = line.strip() if not line: continue # print(i + 1, line) From 8657ea7d87a22883a01ac52b6793c1261ea794d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 14:11:40 +0100 Subject: [PATCH 93/96] TestConstRefInSubscription: Test only top-level process functions --- Scripts/o2_linter.py | 57 +++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 1565d0a2e51..27b6e3ce364 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -478,7 +478,9 @@ def test_line(self, line: str) -> bool: class TestConstRefInSubscription(TestSpec): - """Test const refs in process function subscriptions.""" + """Test const refs in process function subscriptions. + Test only top-level process functions (called process() or has PROCESS_SWITCH). + """ name = "const-ref-in-process" message = "Use constant references for table subscriptions in process functions." @@ -490,6 +492,23 @@ def test_file(self, path: str, content) -> bool: n_parens_opened = 0 # number of opened parentheses arguments = "" # process function arguments line_process = 0 # line number of the process function + # Find names of all top-level process functions with switches. + names_functions = ["process"] + for i, line in enumerate(content): + line = line.strip() + if is_comment_cpp(line): + continue + if not line.startswith("PROCESS_SWITCH"): + continue + words = line.split() + if len(words) < 2: + passed = False + print_error(path, i + 1, self.name, + "Failed to get the process function name. Keep it on the same line as the switch.") + continue + names_functions.append(words[1][:-1]) + # print_error(path, i + 1, self.name, f"Got process function name {words[1][:-1]}.") + # Test process functions. for i, line in enumerate(content): line = line.strip() if is_comment_cpp(line): @@ -498,7 +517,7 @@ def test_file(self, path: str, content) -> bool: continue if "//" in line: # Remove comment. (Ignore /* to avoid truncating at /*parameter*/.) line = line[: line.index("//")] - if re.search(r"^void process[\w]*\(", line): + if (match := re.match(r"void (process[\w]*)\(", line)) and match.group(1) in names_functions: line_process = i + 1 i_closing = line.rfind(")") i_start = line.index("(") + 1 @@ -1380,29 +1399,29 @@ def main(): # Bad practice enable_bad_practice = True if enable_bad_practice: - tests.append(TestIOStream()) - tests.append(TestUsingStd()) - tests.append(TestUsingDirectives()) - tests.append(TestStdPrefix()) - tests.append(TestROOT()) - tests.append(TestPi()) - tests.append(TestTwoPiAddSubtract()) - tests.append(TestPiMultipleFraction()) - tests.append(TestPdgDatabase()) - tests.append(TestPdgCode()) - tests.append(TestPdgMass()) - tests.append(TestLogging()) - tests.append(TestConstRefInForLoop()) + # tests.append(TestIOStream()) + # tests.append(TestUsingStd()) + # tests.append(TestUsingDirectives()) + # tests.append(TestStdPrefix()) + # tests.append(TestROOT()) + # tests.append(TestPi()) + # tests.append(TestTwoPiAddSubtract()) + # tests.append(TestPiMultipleFraction()) + # tests.append(TestPdgDatabase()) + # tests.append(TestPdgCode()) + # tests.append(TestPdgMass()) + # tests.append(TestLogging()) + # tests.append(TestConstRefInForLoop()) tests.append(TestConstRefInSubscription()) - tests.append(TestWorkflowOptions()) + # tests.append(TestWorkflowOptions()) # Documentation - enable_documentation = True + enable_documentation = False if enable_documentation: tests.append(TestDocumentationFile()) # Naming conventions - enable_naming = True + enable_naming = False if enable_naming: tests.append(TestNameFunctionVariable()) tests.append(TestNameMacro()) @@ -1422,7 +1441,7 @@ def main(): tests.append(TestNameConfigurable()) # PWG-HF - enable_pwghf = True + enable_pwghf = False if enable_pwghf: tests.append(TestHfNameStructClass()) tests.append(TestHfNameFileTask()) From 984c52f068684e2cd439389d10b72640a3c3be03 Mon Sep 17 00:00:00 2001 From: ALICE Builder Date: Thu, 28 Nov 2024 14:31:47 +0100 Subject: [PATCH 94/96] [MegaLinter] Apply linters automatic fixes to #7066 (#115) --- Scripts/o2_linter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 27b6e3ce364..4daf752a2ab 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -503,8 +503,12 @@ def test_file(self, path: str, content) -> bool: words = line.split() if len(words) < 2: passed = False - print_error(path, i + 1, self.name, - "Failed to get the process function name. Keep it on the same line as the switch.") + print_error( + path, + i + 1, + self.name, + "Failed to get the process function name. Keep it on the same line as the switch.", + ) continue names_functions.append(words[1][:-1]) # print_error(path, i + 1, self.name, f"Got process function name {words[1][:-1]}.") From 48526a14a86e9712d3e51b1c9a36dad14129e09a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 14:39:15 +0100 Subject: [PATCH 95/96] Improve comments --- Scripts/o2_linter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 4daf752a2ab..8d0bb8840e7 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -492,8 +492,8 @@ def test_file(self, path: str, content) -> bool: n_parens_opened = 0 # number of opened parentheses arguments = "" # process function arguments line_process = 0 # line number of the process function - # Find names of all top-level process functions with switches. - names_functions = ["process"] + # Find names of all top-level process functions. + names_functions = ["process"] # names of allowed process functions to test for i, line in enumerate(content): line = line.strip() if is_comment_cpp(line): @@ -510,7 +510,7 @@ def test_file(self, path: str, content) -> bool: "Failed to get the process function name. Keep it on the same line as the switch.", ) continue - names_functions.append(words[1][:-1]) + names_functions.append(words[1][:-1]) # Remove the trailing comma. # print_error(path, i + 1, self.name, f"Got process function name {words[1][:-1]}.") # Test process functions. for i, line in enumerate(content): From 8fcbe934ae83d98c07da6039735dcf902436d1a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADt=20Ku=C4=8Dera?= Date: Thu, 28 Nov 2024 20:32:07 +0100 Subject: [PATCH 96/96] Enable all tests back --- Scripts/o2_linter.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/Scripts/o2_linter.py b/Scripts/o2_linter.py index 8d0bb8840e7..f88911d1ee9 100644 --- a/Scripts/o2_linter.py +++ b/Scripts/o2_linter.py @@ -1403,29 +1403,29 @@ def main(): # Bad practice enable_bad_practice = True if enable_bad_practice: - # tests.append(TestIOStream()) - # tests.append(TestUsingStd()) - # tests.append(TestUsingDirectives()) - # tests.append(TestStdPrefix()) - # tests.append(TestROOT()) - # tests.append(TestPi()) - # tests.append(TestTwoPiAddSubtract()) - # tests.append(TestPiMultipleFraction()) - # tests.append(TestPdgDatabase()) - # tests.append(TestPdgCode()) - # tests.append(TestPdgMass()) - # tests.append(TestLogging()) - # tests.append(TestConstRefInForLoop()) + tests.append(TestIOStream()) + tests.append(TestUsingStd()) + tests.append(TestUsingDirectives()) + tests.append(TestStdPrefix()) + tests.append(TestROOT()) + tests.append(TestPi()) + tests.append(TestTwoPiAddSubtract()) + tests.append(TestPiMultipleFraction()) + tests.append(TestPdgDatabase()) + tests.append(TestPdgCode()) + tests.append(TestPdgMass()) + tests.append(TestLogging()) + tests.append(TestConstRefInForLoop()) tests.append(TestConstRefInSubscription()) - # tests.append(TestWorkflowOptions()) + tests.append(TestWorkflowOptions()) # Documentation - enable_documentation = False + enable_documentation = True if enable_documentation: tests.append(TestDocumentationFile()) # Naming conventions - enable_naming = False + enable_naming = True if enable_naming: tests.append(TestNameFunctionVariable()) tests.append(TestNameMacro()) @@ -1445,7 +1445,7 @@ def main(): tests.append(TestNameConfigurable()) # PWG-HF - enable_pwghf = False + enable_pwghf = True if enable_pwghf: tests.append(TestHfNameStructClass()) tests.append(TestHfNameFileTask())