Merge pull request #176 from mvdebolskiy/fix-fates-tag-fleximod

Fix fates tag fleximod

Author: mvdebolskiy

.gitmodules

@@ -28,7 +28,7 @@
 [submodule "fates"]
 path = src/fates
 url = https://github.com/NorESMhub/fates
-fxtag = sci.1.88.6_api.42.0.0_14pft_nor_sci1_api1
+fxtag = sci.1.88.6_api.42.0.0_nor_sci1_api1
 fxrequired = AlwaysRequired
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NorESMhub/fates

.lib/git-fleximod/.github/workflows/pytest.yaml

@@ -73,8 +73,7 @@ jobs:
       - run: |
           git config --global user.name "${GITHUB_ACTOR}"
           git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com"
-          poetry run pytest
-      - name: Setup tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
-
+          poetry run pytest --doctest-modules
+#      - name: Setup tmate session
+#        if: ${{ failure() }}
+#        uses: mxschmitt/action-tmate@v3

.lib/git-fleximod/.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: ^utils/.*$
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v6.0.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace

.lib/git-fleximod/CODE_OF_CONDUCT.md

@@ -17,7 +17,7 @@ Examples of behaviors that contribute to a positive environment include:
 * Communicate openly with respect for others, critiquing ideas rather than individuals and gracefully accepting criticism
 * Acknowledging the contributions of others
 * Avoid personal attacks directed toward other participants
-* Be mindful of your surroundings and of your fellow participants 
+* Be mindful of your surroundings and of your fellow participants
 * Alert UCAR staff and suppliers/vendors if you notice a dangerous situation or someone in distress
 * Respect the rules and policies of the project and venue
 
@@ -29,7 +29,7 @@ Examples of unacceptable behavior include, but are not limited to:
 * Personal attacks directed at other guests, members, participants, etc.
 * Publishing others' private information, such as a physical or electronic address, without explicit permission
 * Alarming, intimidating, threatening, or hostile comments or conduct
-* Inappropriate use of nudity and/or sexual images 
+* Inappropriate use of nudity and/or sexual images
 * Threatening or stalking anyone, including a participant
 * Other conduct which could reasonably be considered inappropriate in a professional setting
 
@@ -60,7 +60,7 @@ the [Attribution](#attribution) section.
 ## Reporting
 Instances of unacceptable behavior can be brought to the attention of the project administrator(s) who may take any action as
 outlined in the [Consequences](#consequences) section below.
-However, making a report to a project administrator is not considered an 'official report' to UCAR. 
+However, making a report to a project administrator is not considered an 'official report' to UCAR.
 
 Instances of unacceptable behavior may also be reported directly to UCAR pursuant to [UCAR's Harassment Reporting and Complaint
 Procedure](https://www2.fin.ucar.edu/procedures/hr/harassment-reporting-and-complaint-procedure), or anonymously through [UCAR's

.lib/git-fleximod/README.md

@@ -25,7 +25,7 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
 
 ## Supported .gitmodules Variables
 
-    fxtag: Specify a specific tag or branch to checkout for a submodule.
+    fxtag: Specify a specific tag or hash to checkout for a submodule.  Branches are not acceptable.
     fxrequired: Mark a submodule's checkout behavior, with allowed values:
       - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module).
       - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module).

.lib/git-fleximod/git_fleximod/cli.py

@@ -1,8 +1,8 @@
 from pathlib import Path
 import argparse, os, sys
-from git_fleximod import utils
 
-__version__ = "1.0.2"
+__version__ = "1.1.1"
+
 
 class CustomArgumentParser(argparse.ArgumentParser):
     def print_help(self, file=None):
@@ -17,13 +17,28 @@ def print_help(self, file=None):
         for path in candidate_paths:
             if os.path.exists(path):
                 with open(path) as f:
-                    print( f.read(), file=file)
+                    print(f.read(), file=file)
                     return
-        print( "README.md not found.", file=file)
+        print("README.md not found.", file=file)
+
 
 def find_root_dir(filename=".gitmodules"):
-    """ finds the highest directory in tree
-    which contains a file called filename """
+    """
+    Finds the highest directory in tree which contains a file called filename.
+
+    >>> import tempfile, os
+    >>> cwd = os.getcwd()
+    >>> with tempfile.TemporaryDirectory() as tmp:
+    ...     subdir = Path(tmp) / 'subdir'
+    ...     subdir.mkdir()
+    ...     f = Path(tmp) / '.gitmodules'
+    ...     _ = f.write_text('')
+    ...     os.chdir(subdir)
+    ...     result = find_root_dir('.gitmodules') == str(tmp)
+    ...     os.chdir(cwd)
+    ...     result
+    True
+    """
     d = Path.cwd()
     root = Path(d.root)
     dirlist = []
@@ -40,6 +55,7 @@ def find_root_dir(filename=".gitmodules"):
             return str(dl)
     return None
 
+
 def get_parser():
     description = """
     %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models
@@ -116,6 +132,13 @@ def get_parser():
         "verbosity level each time.",
     )
 
+    parser.add_argument(
+        "--no-mods-details",
+        action="store_true",
+        default=False,
+        help="Suppress details on local mods in status output.",
+    )
+
     parser.add_argument(
         "-V",
         "--version",

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -7,21 +7,27 @@
 
 import os
 import shutil
-import logging
-import textwrap
 import asyncio
+import logging
+from configparser import NoOptionError
 from git_fleximod import utils
 from git_fleximod import cli
 from git_fleximod.gitinterface import GitInterface
 from git_fleximod.gitmodules import GitModules
 from git_fleximod.submodule import Submodule
 
-# logger variable is global
-logger = None
+logger = logging.getLogger(__name__)
 
 
 def fxrequired_allowed_values():
-    return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional", "TopLevelRequired", "TopLevelOptional"]
+    return [
+        "ToplevelRequired",
+        "ToplevelOptional",
+        "AlwaysRequired",
+        "AlwaysOptional",
+        "TopLevelRequired",
+        "TopLevelOptional",
+    ]
 
 
 def commandline_arguments(args=None):
@@ -69,6 +75,7 @@ def commandline_arguments(args=None):
         options.components,
         options.exclude,
         options.force,
+        options.no_mods_details,
         action,
     )
 
@@ -151,8 +158,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
 
     if os.path.isdir(os.path.join(root_dir, path, ".git")):
         with utils.pushd(sprep_repo):
-            if os.path.isdir(os.path.join(topgit,".git")):
-                shutil.rmtree(os.path.join(topgit,".git"))
+            if os.path.isdir(os.path.join(topgit, ".git")):
+                shutil.rmtree(os.path.join(topgit, ".git"))
             shutil.move(".git", topgit)
             with open(".git", "w") as f:
                 f.write("gitdir: " + os.path.relpath(topgit))
@@ -167,7 +174,6 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
         with utils.pushd(sprep_repo):
             if os.path.isfile(sparsefile):
                 shutil.copy(sparsefile, gitsparse)
-                
 
     # Finally checkout the repo
     sprepo_git.git_operation("fetch", "origin", "--tags")
@@ -177,7 +183,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     rgit.config_set_value(f'submodule "{name}"', "active", "true")
     rgit.config_set_value(f'submodule "{name}"', "url", url)
 
-def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
+
+def init_submodule_from_gitmodules(gitmodules, name, root_dir, llogger):
     path = gitmodules.get(name, "path")
     url = gitmodules.get(name, "url")
     assert path and url, f"Malformed .gitmodules file {path} {url}"
@@ -187,60 +194,73 @@ def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger):
     fxurl = gitmodules.get(name, "fxDONOTUSEurl")
     fxsparse = gitmodules.get(name, "fxsparse")
     fxrequired = gitmodules.get(name, "fxrequired")
-    return Submodule(root_dir, name, path, url, fxtag=tag, fxurl=fxurl, fxsparse=fxsparse, fxrequired=fxrequired, logger=logger)
+    return Submodule(
+        root_dir,
+        name,
+        path,
+        url,
+        fxtag=tag,
+        fxurl=fxurl,
+        fxsparse=fxsparse,
+        fxrequired=fxrequired,
+        logger=llogger,
+    )
 
-def submodules_status(gitmodules, root_dir, toplevel=False, depth=0):
+
+def submodules_status(
+    gitmodules, root_dir, toplevel=False, depth=0, no_mods_details=False
+):
     testfails = 0
     localmods = 0
     needsupdate = 0
-    wrapper = textwrap.TextWrapper(initial_indent=' '*(depth*10), width=120,subsequent_indent=' '*(depth*20))
     for name in gitmodules.sections():
         submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger)
-            
-        result,n,l,t = submod.status()
+
+        result, n, l, t = submod.status(depth=depth, no_mods_details=no_mods_details)
         if toplevel or not submod.toplevel():
-            print(wrapper.fill(result))
+            print(result)
             testfails += t
             localmods += l
             needsupdate += n
         subdir = os.path.join(root_dir, submod.path)
         if os.path.exists(os.path.join(subdir, ".gitmodules")):
             gsubmod = GitModules(logger, confpath=subdir)
-            t,l,n = submodules_status(gsubmod, subdir, depth=depth+1)
+            t, l, n = submodules_status(
+                gsubmod, subdir, depth=depth + 1, no_mods_details=no_mods_details
+            )
             if toplevel or not submod.toplevel():
                 testfails += t
                 localmods += l
                 needsupdate += n
-            
+
     return testfails, localmods, needsupdate
 
-def git_toplevelroot(root_dir, logger):
-    rgit = GitInterface(root_dir, logger)
+
+def git_toplevelroot(root_dir, llogger):
+    rgit = GitInterface(root_dir, llogger)
     _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
+
 async def submodules_update(gitmodules, root_dir, requiredlist, force):
     async def update_submodule(name, requiredlist, force):
         submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger)
 
-        _, needsupdate, localmods, testfails = submod.status()
         if not submod.fxrequired:
             submod.fxrequired = "AlwaysRequired"
-        fxrequired = submod.fxrequired    
+        fxrequired = submod.fxrequired
         allowedvalues = fxrequired_allowed_values()
         assert fxrequired in allowedvalues
 
         superroot = git_toplevelroot(root_dir, logger)
-                                     
-        if (
-            fxrequired
-            and ((superroot and "Toplevel" in fxrequired)
-            or fxrequired not in requiredlist)
+
+        if fxrequired and (
+            (superroot and "Toplevel" in fxrequired) or fxrequired not in requiredlist
         ):
             if "Optional" in fxrequired and "Optional" not in requiredlist:
                 if fxrequired.startswith("Always"):
                     print(f"Skipping optional component {name:>20}")
-                return # continue to next submodule
+                return  # continue to next submodule
         optional = "AlwaysOptional" in requiredlist
 
         if fxrequired in requiredlist:
@@ -253,11 +273,16 @@ async def update_submodule(name, requiredlist, force):
                 newrequiredlist = ["AlwaysRequired"]
                 if optional:
                     newrequiredlist.append("AlwaysOptional")
-                await submodules_update(gitsubmodules, repodir, newrequiredlist, force=force)
+                await submodules_update(
+                    gitsubmodules, repodir, newrequiredlist, force=force
+                )
 
-    tasks = [update_submodule(name, requiredlist, force) for name in gitmodules.sections()]
+    tasks = [
+        update_submodule(name, requiredlist, force) for name in gitmodules.sections()
+    ]
     await asyncio.gather(*tasks)
 
+
 def local_mods_output():
     text = """\
     The submodules labeled with 'M' above are not in a clean state.
@@ -271,7 +296,8 @@ def local_mods_output():
 """
     print(text)
 
-def submodules_test(gitmodules, root_dir):
+
+def submodules_test(gitmodules, root_dir, no_mods_details=False):
     """
     This function tests the git submodules based on the provided parameters.
 
@@ -282,12 +308,15 @@ def submodules_test(gitmodules, root_dir):
     Parameters:
     gitmodules (ConfigParser): The gitmodules configuration.
     root_dir (str): The root directory for the git operation.
+    no_mods_details (bool, optional): If True, suppress details on local mods in status output
 
     Returns:
     int: The number of test failures.
     """
     # First check that fxtags are present and in sync with submodule hashes
-    testfails, localmods, needsupdate = submodules_status(gitmodules, root_dir)
+    testfails, localmods, needsupdate = submodules_status(
+        gitmodules, root_dir, no_mods_details=no_mods_details
+    )
     print("")
     # Then make sure that urls are consistant with fxurls (not forks and not ssh)
     # and that sparse checkout files exist
@@ -315,14 +344,17 @@ def main():
         includelist,
         excludelist,
         force,
+        no_mods_details,
         action,
     ) = commandline_arguments()
     # Get a logger for the package
     global logger
     logger = logging.getLogger(__name__)
 
-    logger.info("action is {} root_dir={} file_name={}".format(action, root_dir, file_name))
-    
+    logger.info(
+        "action is {} root_dir={} file_name={}".format(action, root_dir, file_name)
+    )
+
     if not root_dir or not os.path.isfile(os.path.join(root_dir, file_name)):
         if root_dir:
             file_path = utils.find_upwards(root_dir, file_name)
@@ -352,15 +384,17 @@ def main():
     if action == "update":
         asyncio.run(submodules_update(gitmodules, root_dir, fxrequired, force))
     elif action == "status":
-        tfails, lmods, updates = submodules_status(gitmodules, root_dir, toplevel=True)
+        tfails, lmods, updates = submodules_status(
+            gitmodules, root_dir, toplevel=True, no_mods_details=no_mods_details
+        )
         if tfails + lmods + updates > 0:
             print(
                 f"    testfails = {tfails}, local mods = {lmods}, needs updates {updates}\n"
             )
             if lmods > 0:
                 local_mods_output()
     elif action == "test":
-        retval = submodules_test(gitmodules, root_dir)
+        retval = submodules_test(gitmodules, root_dir, no_mods_details=no_mods_details)
     else:
         utils.fatal_error(f"unrecognized action request {action}")
     return retval