failed on setup with "RuntimeError: Failed to submit payu run. Error: Command '['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1']' returned non-zero exit status 1."
self = <model_config_tests.exp_test_helper.ExpTestHelper object at 0x7f9a8ac6d1b0>
n_runs = 1

    def submit_payu_run(self, n_runs: int = None) -> str:
        """
        Submit a payu run job.
    
        Parameters
        ----------
        n_runs: int
            The number of runs to submit with --nruns.
    
        Returns
        ----------
        str
            The job ID of the submitted payu run job
        """
        if self.disable_payu_run:
            return
    
        owd = Path.cwd()
        try:
            # Change to experiment directory and run.
            os.chdir(self.control_path)
    
            print("Running payu setup")
            result = sp.run(
                ["payu", "setup", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                # Add additional error messaging for debugging
                error_msg = (
                    "Failed to run payu setup:\n"
                    f"Return code: {result.returncode}\n"
                    f"--- stdout ---\n{result.stdout}\n"
                    f"--- stderr ---\n{result.stderr}"
                )
                print(error_msg)
                raise RuntimeError(error_msg)
    
            print("Running payu sweep")
            sp.run(
                ["payu", "sweep", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
                check=True,
            )
    
            run_command = ["payu", "run", "--lab", str(self.lab_path)]
            if n_runs:
                run_command.extend(["--nruns", str(n_runs)])
            print(f"Running payu run command: {' '.join(run_command)}")
>           result = sp.run(run_command, capture_output=True, text=True, check=True)

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:168: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

input = None, capture_output = True, timeout = None, check = True
popenargs = (['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1'],)
kwargs = {'stderr': -1, 'stdout': -1, 'text': True}
process = <Popen: returncode: 1 args: ['payu', 'run', '--lab', '/scratch/tm70/repro-ci...>
stdout = 'laboratory path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/l...32586456d391f4c8bcdd0f36e3367c55d204/lab/archive\nExperiment name is configured in config.yaml:  exp_default_runtime\n'
stderr = 'Traceback (most recent call last):\n  File "/g/data/vk83/prerelease/apps/base_conda/envs/payu-dev-20260213T043728Z-03...des_json\n    raise RuntimeError(\nRuntimeError: Unable to collect pbs node info: command timed out after 60 seconds\n'
retcode = 1

    def run(*popenargs,
            input=None, capture_output=False, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.
    
        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
    
        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.
    
        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.
    
        There is an optional argument "input", allowing you to
        pass bytes or a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.
    
        By default, all communication is in bytes, and therefore any "input" should
        be bytes, and the stdout and stderr will be bytes. If in text mode, any
        "input" should be a string, and stdout and stderr will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode is
        triggered by setting any of text, encoding, errors or universal_newlines.
    
        The other arguments are the same as for the Popen constructor.
        """
        if input is not None:
            if kwargs.get('stdin') is not None:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE
    
        if capture_output:
            if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
                raise ValueError('stdout and stderr arguments may not be used '
                                 'with capture_output.')
            kwargs['stdout'] = PIPE
            kwargs['stderr'] = PIPE
    
        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired as exc:
                process.kill()
                if _mswindows:
                    # Windows accumulates the output in a single blocking
                    # read() call run on child threads, with the timeout
                    # being done in a join() on those threads.  communicate()
                    # _after_ kill() is required to collect that and add it
                    # to the exception.
                    exc.stdout, exc.stderr = process.communicate()
                else:
                    # POSIX _communicate already populated the output so
                    # far into the TimeoutExpired exception.
                    process.wait()
                raise
            except:  # Including KeyboardInterrupt, communicate handled that.
                process.kill()
                # We don't call process.wait() as .__exit__ does that for us.
                raise
            retcode = process.poll()
            if check and retcode:
>               raise CalledProcessError(retcode, process.args,
                                         output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command '['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1']' returned non-zero exit status 1.

/g/data/vk83/prerelease/apps/base_conda/envs/payu-dev-20260213T043728Z-0359e0d/lib/python3.10/subprocess.py:524: CalledProcessError

During handling of the above exception, another exception occurred:

request = <SubRequest 'experiments' for <Function test_repro_historical>>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/base-experiment')
keep_archive = False

    @pytest.fixture(scope="class")
    def experiments(
        request, output_path: Path, control_path: Path, keep_archive: Optional[bool]
    ):
        """
        Parse the experiments markers from the requested tests and
        submit all necessary experiments at the same time.
    
        The scope is class so the experiments are only run once before all repro
        tests.
        """
    
        # Parse the experiments markers from the requested tests
        experiments_markers = []
        for item in request.session.items:
            if item.parent == request.node:
                marker = item.get_closest_marker("experiments")
                if marker:
                    experiments_markers.append(marker.args[0])
    
>       return _experiments(experiments_markers, output_path, control_path, keep_archive)

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:147: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:113: in _experiments
    experiments.setup_and_submit(
../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:279: in setup_and_submit
    exp.submit_payu_run(n_runs=n_runs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <model_config_tests.exp_test_helper.ExpTestHelper object at 0x7f9a8ac6d1b0>
n_runs = 1

    def submit_payu_run(self, n_runs: int = None) -> str:
        """
        Submit a payu run job.
    
        Parameters
        ----------
        n_runs: int
            The number of runs to submit with --nruns.
    
        Returns
        ----------
        str
            The job ID of the submitted payu run job
        """
        if self.disable_payu_run:
            return
    
        owd = Path.cwd()
        try:
            # Change to experiment directory and run.
            os.chdir(self.control_path)
    
            print("Running payu setup")
            result = sp.run(
                ["payu", "setup", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                # Add additional error messaging for debugging
                error_msg = (
                    "Failed to run payu setup:\n"
                    f"Return code: {result.returncode}\n"
                    f"--- stdout ---\n{result.stdout}\n"
                    f"--- stderr ---\n{result.stderr}"
                )
                print(error_msg)
                raise RuntimeError(error_msg)
    
            print("Running payu sweep")
            sp.run(
                ["payu", "sweep", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
                check=True,
            )
    
            run_command = ["payu", "run", "--lab", str(self.lab_path)]
            if n_runs:
                run_command.extend(["--nruns", str(n_runs)])
            print(f"Running payu run command: {' '.join(run_command)}")
            result = sp.run(run_command, capture_output=True, text=True, check=True)
            self.run_id = parse_run_id(result.stdout)
            print(f"Run Job ID: {self.run_id}")
        except sp.CalledProcessError as e:
>           raise RuntimeError(f"Failed to submit payu run. Error: {e}")
E           RuntimeError: Failed to submit payu run. Error: Command '['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1']' returned non-zero exit status 1.

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:172: RuntimeError

failed on setup with "RuntimeError: Failed to submit payu run. Error: Command '['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1']' returned non-zero exit status 1."
self = <model_config_tests.exp_test_helper.ExpTestHelper object at 0x7f9a8ac6d1b0>
n_runs = 1

    def submit_payu_run(self, n_runs: int = None) -> str:
        """
        Submit a payu run job.
    
        Parameters
        ----------
        n_runs: int
            The number of runs to submit with --nruns.
    
        Returns
        ----------
        str
            The job ID of the submitted payu run job
        """
        if self.disable_payu_run:
            return
    
        owd = Path.cwd()
        try:
            # Change to experiment directory and run.
            os.chdir(self.control_path)
    
            print("Running payu setup")
            result = sp.run(
                ["payu", "setup", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                # Add additional error messaging for debugging
                error_msg = (
                    "Failed to run payu setup:\n"
                    f"Return code: {result.returncode}\n"
                    f"--- stdout ---\n{result.stdout}\n"
                    f"--- stderr ---\n{result.stderr}"
                )
                print(error_msg)
                raise RuntimeError(error_msg)
    
            print("Running payu sweep")
            sp.run(
                ["payu", "sweep", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
                check=True,
            )
    
            run_command = ["payu", "run", "--lab", str(self.lab_path)]
            if n_runs:
                run_command.extend(["--nruns", str(n_runs)])
            print(f"Running payu run command: {' '.join(run_command)}")
>           result = sp.run(run_command, capture_output=True, text=True, check=True)

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:168: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

input = None, capture_output = True, timeout = None, check = True
popenargs = (['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1'],)
kwargs = {'stderr': -1, 'stdout': -1, 'text': True}
process = <Popen: returncode: 1 args: ['payu', 'run', '--lab', '/scratch/tm70/repro-ci...>
stdout = 'laboratory path:  /scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/l...32586456d391f4c8bcdd0f36e3367c55d204/lab/archive\nExperiment name is configured in config.yaml:  exp_default_runtime\n'
stderr = 'Traceback (most recent call last):\n  File "/g/data/vk83/prerelease/apps/base_conda/envs/payu-dev-20260213T043728Z-03...des_json\n    raise RuntimeError(\nRuntimeError: Unable to collect pbs node info: command timed out after 60 seconds\n'
retcode = 1

    def run(*popenargs,
            input=None, capture_output=False, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.
    
        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
    
        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.
    
        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.
    
        There is an optional argument "input", allowing you to
        pass bytes or a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.
    
        By default, all communication is in bytes, and therefore any "input" should
        be bytes, and the stdout and stderr will be bytes. If in text mode, any
        "input" should be a string, and stdout and stderr will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode is
        triggered by setting any of text, encoding, errors or universal_newlines.
    
        The other arguments are the same as for the Popen constructor.
        """
        if input is not None:
            if kwargs.get('stdin') is not None:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE
    
        if capture_output:
            if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
                raise ValueError('stdout and stderr arguments may not be used '
                                 'with capture_output.')
            kwargs['stdout'] = PIPE
            kwargs['stderr'] = PIPE
    
        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired as exc:
                process.kill()
                if _mswindows:
                    # Windows accumulates the output in a single blocking
                    # read() call run on child threads, with the timeout
                    # being done in a join() on those threads.  communicate()
                    # _after_ kill() is required to collect that and add it
                    # to the exception.
                    exc.stdout, exc.stderr = process.communicate()
                else:
                    # POSIX _communicate already populated the output so
                    # far into the TimeoutExpired exception.
                    process.wait()
                raise
            except:  # Including KeyboardInterrupt, communicate handled that.
                process.kill()
                # We don't call process.wait() as .__exit__ does that for us.
                raise
            retcode = process.poll()
            if check and retcode:
>               raise CalledProcessError(retcode, process.args,
                                         output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command '['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1']' returned non-zero exit status 1.

/g/data/vk83/prerelease/apps/base_conda/envs/payu-dev-20260213T043728Z-0359e0d/lib/python3.10/subprocess.py:524: CalledProcessError

During handling of the above exception, another exception occurred:

request = <SubRequest 'experiments' for <Function test_repro_historical>>
output_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204')
control_path = PosixPath('/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/base-experiment')
keep_archive = False

    @pytest.fixture(scope="class")
    def experiments(
        request, output_path: Path, control_path: Path, keep_archive: Optional[bool]
    ):
        """
        Parse the experiments markers from the requested tests and
        submit all necessary experiments at the same time.
    
        The scope is class so the experiments are only run once before all repro
        tests.
        """
    
        # Parse the experiments markers from the requested tests
        experiments_markers = []
        for item in request.session.items:
            if item.parent == request.node:
                marker = item.get_closest_marker("experiments")
                if marker:
                    experiments_markers.append(marker.args[0])
    
>       return _experiments(experiments_markers, output_path, control_path, keep_archive)

../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:147: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../test-venv/lib/python3.10/site-packages/model_config_tests/config_tests/test_bit_reproducibility.py:113: in _experiments
    experiments.setup_and_submit(
../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:279: in setup_and_submit
    exp.submit_payu_run(n_runs=n_runs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <model_config_tests.exp_test_helper.ExpTestHelper object at 0x7f9a8ac6d1b0>
n_runs = 1

    def submit_payu_run(self, n_runs: int = None) -> str:
        """
        Submit a payu run job.
    
        Parameters
        ----------
        n_runs: int
            The number of runs to submit with --nruns.
    
        Returns
        ----------
        str
            The job ID of the submitted payu run job
        """
        if self.disable_payu_run:
            return
    
        owd = Path.cwd()
        try:
            # Change to experiment directory and run.
            os.chdir(self.control_path)
    
            print("Running payu setup")
            result = sp.run(
                ["payu", "setup", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
            )
            if result.returncode != 0:
                # Add additional error messaging for debugging
                error_msg = (
                    "Failed to run payu setup:\n"
                    f"Return code: {result.returncode}\n"
                    f"--- stdout ---\n{result.stdout}\n"
                    f"--- stderr ---\n{result.stderr}"
                )
                print(error_msg)
                raise RuntimeError(error_msg)
    
            print("Running payu sweep")
            sp.run(
                ["payu", "sweep", "--lab", str(self.lab_path)],
                capture_output=True,
                text=True,
                check=True,
            )
    
            run_command = ["payu", "run", "--lab", str(self.lab_path)]
            if n_runs:
                run_command.extend(["--nruns", str(n_runs)])
            print(f"Running payu run command: {' '.join(run_command)}")
            result = sp.run(run_command, capture_output=True, text=True, check=True)
            self.run_id = parse_run_id(result.stdout)
            print(f"Run Job ID: {self.run_id}")
        except sp.CalledProcessError as e:
>           raise RuntimeError(f"Failed to submit payu run. Error: {e}")
E           RuntimeError: Failed to submit payu run. Error: Command '['payu', 'run', '--lab', '/scratch/tm70/repro-ci/experiments/access-esm1.6-configs/c26432586456d391f4c8bcdd0f36e3367c55d204/lab', '--nruns', '1']' returned non-zero exit status 1.

../test-venv/lib/python3.10/site-packages/model_config_tests/exp_test_helper.py:172: RuntimeError