# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Manage submission, poll and kill of a job to the job runners.

Export the JobRunnerManager class.

Job runner handler (a.k.a. job submission method) modules should be placed
under the "cylc.flow.job_runner_handlers" package. Each module should export
the symbol "JOB_RUNNER_HANDLER" for the singleton instance that implements the
job system handler logic.

Each job runner handler class should instantiate with no argument, and may
have the following constants and methods:

job_runner.filter_poll_many_output(out) => job_ids
    * Called after the job runner's poll many command. The method should read
      the output and return a list of job IDs that are still in the
      job runner.

job_runner.filter_submit_output(out, err) => new_out, new_err
    * Filter the standard output and standard error of the job submission
      command. This is useful if the job submission command returns information
      that should just be ignored. See also "job_runner.SUBMIT_CMD_TMPL".

job_runner.format_directives(job_conf) => lines
    * If relevant, this method formats the job directives for a job file, if
      job file directives are relevant for the job runner. The argument
      "job_conf" is a dict containing the job configuration.

job_runner.get_poll_many_cmd(job-id-list) => list
    * Return a list containing the shell command to poll the jobs in the
      argument list.

job_runner.get_submit_stdin(job_file_path: str, submit_opts: dict) => tuple
    * Return a 2-element tuple `(proc_stdin_arg, proc_stdin_value)`.
      Element 1 is suitable for the `stdin=...` argument of `subprocess.Popen`
      so it can be a file handle, `subprocess.PIPE` or `None`. Element 2 is the
      string content to pipe to STDIN of the submit command (relevant only if
      `proc_stdin_arg` is `subprocess.PIPE`.

job_runner.get_vacation_signal(job_conf) => str
    * If relevant, return a string containing the name of the signal that
      indicates the job has been vacated by the job runner.

job_runner.submit(job_file_path, submit_opts) => ret_code, out, err
    * Submit a job and return an instance of the Popen object for the
      submission. This method is useful if the job submission requires logic
      beyond just running a system or shell command. See also
      "job_runner.SUBMIT_CMD". You must pass "env=submit_opts.get('env')" to
      Popen - see background.py for example.

job_runner.manip_job_id(job_id) => job_id
    * Modify the job ID that is returned by the job submit command.

job_runner.FAIL_SIGNALS => tuple<str>
    * A tuple containing the names of signals to trap for reporting errors.
      Default is ("EXIT", "ERR", "TERM", "XCPU"). ERR and EXIT are always
      recommended.  EXIT is used to report premature stopping of the job
      script, and its trap is unset at the end of the script.

job_runner.KILL_CMD_TMPL
    *  A Python string template for getting the job runner command to remove
       and terminate a job ID. The command is formed using the logic:
           job_runner.KILL_CMD_TMPL % {"job_id": job_id}

job_runner.POLL_CANT_CONNECT_ERR
    * A string containing an error message. If this is defined, when a poll
      command returns a non-zero return code and its STDERR contains this
      string, then the poll result will not be trusted, because it is assumed
      that the job runner is currently unavailable. Jobs submitted to the
      job runner will be assumed OK until we are able to connect to the
      job runner again.

job_runner.SHOULD_KILL_PROC_GROUP
    * A boolean to indicate whether it is necessary to kill a job by sending
      a signal to its Unix process group. This boolean also indicates that
      a job submitted via this job runner will physically run on the same
      host it is submitted to.

job_runner.SHOULD_POLL_PROC_GROUP
    * A boolean to indicate whether it is necessary to poll a job by its PID
      as well as the job ID.

job_runner.REC_ID_FROM_SUBMIT_ERR
job_runner.REC_ID_FROM_SUBMIT_OUT
    * A regular expression (compiled) to extract the job "id" from the standard
      output or standard error of the job submission command.

job_runner.SUBMIT_CMD_ENV
    * A Python dict (or an iterable that can be used to update a dict)
      containing extra environment variables for getting the job runner
      command to submit a job file.

job_runner.SUBMIT_CMD_TMPL
    * A Python string template for getting the job runner command to submit a
      job file. The command is formed using the logic:
          job_runner.SUBMIT_CMD_TMPL % {"job": job_file_path}
      See also "job_runner._job_submit_impl".

"""

from contextlib import suppress
import json
import os
from pathlib import Path
import shlex
import stat
import sys
import traceback
from shutil import rmtree
from signal import SIGKILL
from subprocess import DEVNULL  # nosec

from cylc.flow.task_message import (
    CYLC_JOB_PID, CYLC_JOB_INIT_TIME, CYLC_JOB_EXIT_TIME, CYLC_JOB_EXIT,
    CYLC_MESSAGE)
from cylc.flow.cylc_subproc import procopen
from cylc.flow.task_job_logs import (
    JOB_LOG_ERR, JOB_LOG_JOB, JOB_LOG_OUT, JOB_LOG_STATUS)
from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED
from cylc.flow.wallclock import get_current_time_string
from cylc.flow.parsec.OrderedDict import OrderedDict


class JobPollContext():
    """Context object for a job poll."""
    CONTEXT_ATTRIBUTES = (
        'job_log_dir',  # cycle/task/submit_num
        'job_runner_name',
        'job_id',  # job id in job runner
        'job_runner_exit_polled',  # 0 for false, 1 for true
        'run_status',  # 0 for success, 1 for failure
        'run_signal',  # signal received on run failure
        'time_submit_exit',  # submit (exit) time
        'time_run',  # run start time
        'time_run_exit',  # run exit time
        'job_runner_call_no_lines',  # line count in job runner call stdout
    )

    __slots__ = CONTEXT_ATTRIBUTES + (
        'pid',
        'messages'
    )

    def __init__(self, job_log_dir, **attrs):
        self.job_log_dir = job_log_dir
        self.job_runner_name = None
        self.job_id = None
        self.job_runner_exit_polled = None
        self.pid = None
        self.run_status = None
        self.run_signal = None
        self.time_submit_exit = None
        self.time_run = None
        self.time_run_exit = None
        self.job_runner_call_no_lines = None
        self.messages = []

        if attrs:
            for key, value in attrs.items():
                if key not in self.CONTEXT_ATTRIBUTES:
                    raise ValueError('Invalid kwarg "%s"' % key)
                setattr(self, key, value)

    def update(self, other):
        """Update my data from given file context."""
        for i in self.__slots__:
            setattr(self, i, getattr(other, i))

    def get_summary_str(self):
        """Return the poll context as a summary string delimited by "|"."""
        ret = OrderedDict()
        for key in self.CONTEXT_ATTRIBUTES:
            value = getattr(self, key)
            if key == 'job_log_dir' or value is None:
                continue
            ret[key] = value
        return '%s|%s' % (self.job_log_dir, json.dumps(ret))


class JobRunnerManager():
    """Job submission, poll and kill.

    Manage the importing of job submission method modules.

    """

    CYLC_JOB_RUNNER_NAME = "CYLC_JOB_RUNNER_NAME"
    CYLC_JOB_ID = "CYLC_JOB_ID"
    CYLC_JOB_RUNNER_SUBMIT_TIME = "CYLC_JOB_RUNNER_SUBMIT_TIME"
    CYLC_JOB_RUNNER_EXIT_POLLED = "CYLC_JOB_RUNNER_EXIT_POLLED"
    FAIL_SIGNALS = ("EXIT", "ERR", "TERM", "XCPU")
    LINE_PREFIX_JOB_RUNNER_NAME = "# Job submit method: "
    LINE_PREFIX_JOB_RUNNER_CMD_TMPL = "# Job submit command template: "
    LINE_PREFIX_EXECUTION_TIME_LIMIT = "# Execution time limit: "
    LINE_PREFIX_EOF = "#EOF: "
    LINE_PREFIX_JOB_LOG_DIR = "# Job log directory: "
    OUT_PREFIX_COMMAND = "[TASK JOB COMMAND]"
    OUT_PREFIX_MESSAGE = "[TASK JOB MESSAGE]"
    OUT_PREFIX_SUMMARY = "[TASK JOB SUMMARY]"
    OUT_PREFIX_CMD_ERR = "[TASK JOB ERROR]"
    _INSTANCES: dict = {}

    @classmethod
    def configure_workflow_run_dir(cls, workflow_run_dir):
        """Add local python module paths if not already done."""
        for sub_dir in ["python", os.path.join("lib", "python")]:
            # TODO - eventually drop the deprecated "python" sub-dir.
            workflow_py = os.path.join(workflow_run_dir, sub_dir)
            if os.path.isdir(workflow_py) and workflow_py not in sys.path:
                sys.path.append(workflow_py)

    def __init__(self, clean_env=False, env=None, path=None):
        """Initialise JobRunnerManager."""
        # Job submission environment.
        self.clean_env = clean_env
        self.path = path
        self.env = env

    def _get_sys(self, job_runner_name):
        """Return an instance of the class for "job_runner_name"."""
        if job_runner_name in self._INSTANCES:
            return self._INSTANCES[job_runner_name]
        for key in [f"cylc.flow.job_runner_handlers.{job_runner_name}",
                    job_runner_name]:

            try:
                mod_of_name = __import__(key, fromlist=[key])
                self._INSTANCES[job_runner_name] = getattr(
                    mod_of_name, "JOB_RUNNER_HANDLER", None)
                return self._INSTANCES[job_runner_name]
            except ImportError:
                if key == job_runner_name:
                    raise

    def format_directives(self, job_conf):
        """Format the job directives for a job file, if relevant."""
        job_runner = self._get_sys(job_conf['platform']['job runner'])
        if hasattr(job_runner, "format_directives"):
            job_conf = {
                # strip $HOME from the job file path
                # paths in directives should be interpreted relative to $HOME
                # https://github.com/cylc/cylc-flow/issues/4247
                **job_conf,
                'job_file_path': (
                    job_conf["job_file_path"].replace(r"$HOME/", "")
                )
            }
            return job_runner.format_directives(job_conf)

    def get_fail_signals(self, job_conf):
        """Return a list of failure signal names to trap in the job file."""
        job_runner = self._get_sys(job_conf['platform']['job runner'])
        return getattr(job_runner, "FAIL_SIGNALS", self.FAIL_SIGNALS)

    def get_vacation_signal(self, job_conf):
        """Return the vacation signal name for a job file."""
        job_runner = self._get_sys(job_conf['platform']['job runner'])
        if hasattr(job_runner, "get_vacation_signal"):
            return job_runner.get_vacation_signal(job_conf)

    def is_job_local_to_host(self, job_runner_name):
        """Return True if job runner runs jobs local to the submit host."""
        return getattr(
            self._get_sys(job_runner_name), "SHOULD_KILL_PROC_GROUP", False)

    def jobs_kill(self, job_log_root, job_log_dirs):
        """Kill multiple jobs.

        job_log_root -- The log/job/ sub-directory of the workflow.
        job_log_dirs -- A list containing point/name/submit_num for task jobs.

        """
        # Note: The more efficient way to do this is to group the jobs by their
        # job runners, and call the kill command for each job runner once.
        # However, this will make it more difficult to determine if the kill
        # command for a particular job is successful or not.
        if "$" in job_log_root:
            job_log_root = os.path.expandvars(job_log_root)
        self.configure_workflow_run_dir(job_log_root.rsplit(os.sep, 2)[0])
        now = get_current_time_string()
        for job_log_dir in job_log_dirs:
            ret_code, err = self.job_kill(
                os.path.join(job_log_root, job_log_dir, JOB_LOG_STATUS))
            sys.stdout.write("%s%s|%s|%d\n" % (
                self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code))
            # Note: Print STDERR to STDOUT may look a bit strange, but it
            # requires less logic for the workflow to parse the output.
            if err.strip():
                for line in err.splitlines(True):
                    if not line.endswith("\n"):
                        line += "\n"
                    sys.stdout.write("%s%s|%s|%s" % (
                        self.OUT_PREFIX_CMD_ERR, now, job_log_dir, line))

    def jobs_poll(self, job_log_root, job_log_dirs):
        """Poll multiple jobs.

        job_log_root -- The log/job/ sub-directory of the workflow.
        job_log_dirs -- A list containing point/name/submit_num for task jobs.

        """
        if "$" in job_log_root:
            job_log_root = os.path.expandvars(job_log_root)
        self.configure_workflow_run_dir(job_log_root.rsplit(os.sep, 2)[0])

        ctx_list = []  # Contexts for all relevant jobs
        ctx_list_by_job_runner = {}  # {job_runner_name1: [ctx1, ...], ...}

        for job_log_dir in job_log_dirs:
            ctx = self._jobs_poll_status_files(job_log_root, job_log_dir)
            if ctx is None:
                continue
            ctx_list.append(ctx)

            if not ctx.job_runner_name or not ctx.job_id:
                # Lost job runner information for some reason.
                # Mark the job as if it is no longer in the job runner.
                ctx.job_runner_exit_polled = 1
                sys.stderr.write(
                    "%s/%s: incomplete job runner info\n" % (
                        ctx.job_log_dir, JOB_LOG_STATUS))

            # We can trust:
            # * Jobs previously polled to have exited the job runner.
            # * Jobs succeeded or failed with ERR/EXIT.
            if (ctx.job_runner_exit_polled or ctx.run_status == 0 or
                    ctx.run_signal in ["ERR", "EXIT"]):
                continue

            if ctx.job_runner_name not in ctx_list_by_job_runner:
                ctx_list_by_job_runner[ctx.job_runner_name] = []
            ctx_list_by_job_runner[ctx.job_runner_name].append(ctx)

        for job_runner_name, my_ctx_list in ctx_list_by_job_runner.items():
            self._jobs_poll_runner(
                job_log_root, job_runner_name, my_ctx_list)

        cur_time_str = get_current_time_string()
        for ctx in ctx_list:
            for message in ctx.messages:
                sys.stdout.write("%s%s|%s|%s\n" % (
                    self.OUT_PREFIX_MESSAGE,
                    cur_time_str,
                    ctx.job_log_dir,
                    message))
            sys.stdout.write("%s%s|%s\n" % (
                self.OUT_PREFIX_SUMMARY,
                cur_time_str,
                ctx.get_summary_str()))

    def jobs_submit(self, job_log_root, job_log_dirs, remote_mode=False,
                    utc_mode=False):
        """Submit multiple jobs.

        job_log_root -- The log/job/ sub-directory of the workflow.
        job_log_dirs -- A list containing point/name/submit_num for task jobs.
        remote_mode -- am I running on the remote job host?
        utc_mode -- is the workflow running in UTC mode?

        """
        if "$" in job_log_root:
            job_log_root = os.path.expandvars(job_log_root)
        self.configure_workflow_run_dir(job_log_root.rsplit(os.sep, 2)[0])
        if remote_mode:
            items = self._jobs_submit_prep_by_stdin(job_log_root, job_log_dirs)
        else:
            items = self._jobs_submit_prep_by_args(job_log_root, job_log_dirs)
        now = get_current_time_string(override_use_utc=utc_mode)
        for job_log_dir, job_runner_name, submit_opts in items:
            job_file_path = os.path.join(
                job_log_root, job_log_dir, JOB_LOG_JOB)
            if not job_runner_name:
                sys.stdout.write("%s%s|%s|1|\n" % (
                    self.OUT_PREFIX_SUMMARY, now, job_log_dir))
                continue
            ret_code, out, err, job_id = self._job_submit_impl(
                job_file_path, job_runner_name, submit_opts)
            sys.stdout.write("%s%s|%s|%d|%s\n" % (
                self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code, job_id))
            for key, value in [("STDERR", err), ("STDOUT", out)]:
                if value is None or not value.strip():
                    continue
                for line in value.splitlines(True):
                    if not value.endswith("\n"):
                        value += "\n"
                    sys.stdout.write("%s%s|%s|[%s] %s" % (
                        self.OUT_PREFIX_COMMAND, now, job_log_dir, key, line))

    def job_kill(self, st_file_path):
        """Ask job runner to terminate the job specified in "st_file_path".

        Return 0 on success, non-zero integer on failure.

        """
        # WORKFLOW_RUN_DIR/log/job/CYCLE/TASK/SUBMIT/job.status
        self.configure_workflow_run_dir(st_file_path.rsplit(os.sep, 6)[0])
        try:
            with open(st_file_path) as st_file:
                for line in st_file:
                    if line.startswith(f"{self.CYLC_JOB_RUNNER_NAME}="):
                        job_runner = self._get_sys(
                            line.strip().split("=", 1)[1]
                        )
                        break
                else:
                    return (
                        1,
                        "Cannot determine job runner from "
                        f"{JOB_LOG_STATUS} file"
                    )
                st_file.seek(0, 0)  # rewind
                if getattr(job_runner, "SHOULD_KILL_PROC_GROUP", False):
                    for line in st_file:
                        if line.startswith(CYLC_JOB_PID + "="):
                            pid = line.strip().split("=", 1)[1]
                            try:
                                os.killpg(os.getpgid(int(pid)), SIGKILL)
                            except (OSError, ValueError) as exc:
                                traceback.print_exc()
                                return (1, str(exc))
                            else:
                                return (0, "")
                st_file.seek(0, 0)  # rewind
                if hasattr(job_runner, "KILL_CMD_TMPL"):
                    for line in st_file:
                        if not line.startswith(f"{self.CYLC_JOB_ID}="):
                            continue
                        job_id = line.strip().split("=", 1)[1]
                        command = shlex.split(
                            job_runner.KILL_CMD_TMPL % {"job_id": job_id})
                        try:
                            proc = procopen(command, stdindevnull=True,
                                            stderrpipe=True)
                        except OSError as exc:
                            # subprocess.Popen has a bad habit of not setting
                            # the filename of the executable when it raises an
                            # OSError.
                            if not exc.filename:
                                exc.filename = command[0]
                            traceback.print_exc()
                            return (1, str(exc))
                        else:
                            return (
                                proc.wait(),
                                proc.communicate()[1].decode()
                            )
            return (1, f"Cannot determine job ID from {JOB_LOG_STATUS} file")
        except IOError as exc:
            return (1, str(exc))

    @classmethod
    def _create_nn(cls, job_file_path):
        """Create NN symbolic link, if necessary.

        If NN => 01, remove numbered directories with submit numbers greater
        than 01.
        Helper for "self._job_submit_impl".

        """
        job_file_dir = os.path.dirname(job_file_path)
        source = os.path.basename(job_file_dir)
        task_log_dir = os.path.dirname(job_file_dir)
        nn_path = os.path.join(task_log_dir, "NN")
        try:
            old_source = os.readlink(nn_path)
        except OSError:
            old_source = None
        if old_source is not None and old_source != source:
            os.unlink(nn_path)
            old_source = None
        if old_source is None:
            os.symlink(source, nn_path)
        # On submit 1, remove any left over digit directories from prev runs
        if source == "01":
            for name in os.listdir(task_log_dir):
                if name != source and name.isdigit():
                    # Ignore errors, not disastrous if rmtree fails
                    rmtree(
                        os.path.join(task_log_dir, name), ignore_errors=True)

    def _filter_submit_output(self, st_file_path, job_runner, out, err):
        """Filter submit command output, if relevant."""
        job_id = None
        if hasattr(job_runner, "REC_ID_FROM_SUBMIT_ERR"):
            text = err
            rec_id = job_runner.REC_ID_FROM_SUBMIT_ERR
        elif hasattr(job_runner, "REC_ID_FROM_SUBMIT_OUT"):
            text = out
            rec_id = job_runner.REC_ID_FROM_SUBMIT_OUT
        if rec_id:
            for line in str(text).splitlines():
                match = rec_id.match(line)
                if match:
                    job_id = match.group("id")
                    if hasattr(job_runner, "manip_job_id"):
                        job_id = job_runner.manip_job_id(job_id)
                    with open(st_file_path, "a") as job_status_file:
                        job_status_file.write("{0}={1}\n".format(
                            self.CYLC_JOB_ID, job_id))
                        job_status_file.write("{0}={1}\n".format(
                            self.CYLC_JOB_RUNNER_SUBMIT_TIME,
                            get_current_time_string()))
                    break
        if hasattr(job_runner, "filter_submit_output"):
            out, err = job_runner.filter_submit_output(out, err)
        return out, err, job_id

    def _jobs_poll_status_files(self, job_log_root, job_log_dir):
        """Helper 1 for self.jobs_poll(job_log_root, job_log_dirs)."""
        ctx = JobPollContext(job_log_dir)
        try:
            with open(
                os.path.join(job_log_root, ctx.job_log_dir, JOB_LOG_STATUS)
            ) as handle:
                for line in handle:
                    if "=" not in line:
                        continue
                    key, value = line.strip().split("=", 1)
                    if key == self.CYLC_JOB_RUNNER_NAME:
                        ctx.job_runner_name = value
                    elif key == self.CYLC_JOB_ID:
                        ctx.job_id = value
                    elif key == self.CYLC_JOB_RUNNER_EXIT_POLLED:
                        ctx.job_runner_exit_polled = 1
                    elif key == CYLC_JOB_PID:
                        ctx.pid = value
                    elif key == self.CYLC_JOB_RUNNER_SUBMIT_TIME:
                        ctx.time_submit_exit = value
                    elif key == CYLC_JOB_INIT_TIME:
                        ctx.time_run = value
                    elif key == CYLC_JOB_EXIT_TIME:
                        ctx.time_run_exit = value
                    elif key == CYLC_JOB_EXIT:
                        if value == TASK_OUTPUT_SUCCEEDED.upper():
                            ctx.run_status = 0
                        else:
                            ctx.run_status = 1
                            ctx.run_signal = value
                    elif key == CYLC_MESSAGE:
                        ctx.messages.append(value)
        except IOError as exc:
            sys.stderr.write(f"{exc}\n")
            return

        return ctx

    def _jobs_poll_runner(self, job_log_root, job_runner_name, my_ctx_list):
        """Helper 2 for self.jobs_poll(job_log_root, job_log_dirs)."""
        exp_job_ids = [ctx.job_id for ctx in my_ctx_list]
        bad_job_ids = list(exp_job_ids)
        exp_pids = []
        bad_pids = []
        items = [[self._get_sys(job_runner_name), exp_job_ids, bad_job_ids]]
        if getattr(items[0][0], "SHOULD_POLL_PROC_GROUP", False):
            exp_pids = [ctx.pid for ctx in my_ctx_list if ctx.pid is not None]
            bad_pids.extend(exp_pids)
            items.append([self._get_sys("background"), exp_pids, bad_pids])
        debug_messages = []
        for job_runner, exp_ids, bad_ids in items:
            if hasattr(job_runner, "get_poll_many_cmd"):
                # Some poll commands may not be as simple
                cmd = job_runner.get_poll_many_cmd(exp_ids)
            else:  # if hasattr(job_runner, "POLL_CMD"):
                # Simple poll command that takes a list of job IDs
                cmd = [job_runner.POLL_CMD, *exp_ids]
            try:
                proc = procopen(cmd, stdindevnull=True,
                                stderrpipe=True, stdoutpipe=True)
            except OSError as exc:
                # subprocess.Popen has a bad habit of not setting the
                # filename of the executable when it raises an OSError.
                if not exc.filename:
                    exc.filename = cmd[0]
                sys.stderr.write(f"{exc}\n")
                return
            ret_code = proc.wait()
            out, err = (f.decode() for f in proc.communicate())
            debug_messages.append('{0} - {1}'.format(
                job_runner, len(out.split('\n')))
            )
            sys.stderr.write(err)
            if (ret_code and hasattr(job_runner, "POLL_CANT_CONNECT_ERR") and
                    job_runner.POLL_CANT_CONNECT_ERR in err):
                # Poll command failed because it cannot connect to job runner
                # Assume jobs are still healthy until the job runner is back.
                bad_ids[:] = []
            elif hasattr(job_runner, "filter_poll_many_output"):
                # Allow custom filter
                for id_ in job_runner.filter_poll_many_output(out):
                    with suppress(ValueError):
                        bad_ids.remove(id_)
            else:
                # Just about all poll commands return a table, with column 1
                # being the job ID. The logic here should be sufficient to
                # ensure that any table header is ignored.
                for line in out.splitlines():
                    try:
                        head = line.split(None, 1)[0]
                    except IndexError:
                        continue
                    if head in exp_ids:
                        with suppress(ValueError):
                            bad_ids.remove(head)

        debug_flag = False
        for ctx in my_ctx_list:
            ctx.job_runner_exit_polled = int(
                ctx.job_id in bad_job_ids)
            # Exited job runner, but process still running
            # This can happen to jobs in some "at" implementation
            if ctx.job_runner_exit_polled and ctx.pid in exp_pids:
                if ctx.pid not in bad_pids:
                    ctx.job_runner_exit_polled = 0
                else:
                    debug_flag = True
            # Add information to "job.status"
            if ctx.job_runner_exit_polled:
                try:
                    with open(os.path.join(
                        job_log_root, ctx.job_log_dir, JOB_LOG_STATUS), "a"
                    ) as handle:
                        handle.write("{0}={1}\n".format(
                            self.CYLC_JOB_RUNNER_EXIT_POLLED,
                            get_current_time_string())
                        )
                except IOError as exc:
                    sys.stderr.write(f"{exc}\n")

                # Re-read the status file in case the job started and exited
                # between the file and batch system checks, which would be
                # interpreted as submit-failed (job exited without starting).
                # Possible if polling many jobs and/or system heavily loaded.
                file_ctx = self._jobs_poll_status_files(
                    job_log_root, ctx.job_log_dir)
                ctx.update(file_ctx)

        if debug_flag:
            ctx.job_runner_call_no_lines = ', '.join(debug_messages)

    def _job_submit_impl(
            self, job_file_path, job_runner_name, submit_opts):
        """Helper for self.jobs_submit() and self.job_submit()."""

        # Create NN symbolic link, if necessary
        self._create_nn(job_file_path)
        for name in JOB_LOG_ERR, JOB_LOG_OUT:
            with suppress(OSError):
                os.unlink(os.path.join(job_file_path, name))

        # Start new status file
        with open(f"{job_file_path}.status", "w") as job_status_file:
            job_status_file.write(
                "{0}={1}\n".format(
                    self.CYLC_JOB_RUNNER_NAME,
                    job_runner_name
                )
            )

        # Submit job
        job_runner = self._get_sys(job_runner_name)
        if not self.clean_env:
            # Pass the whole environment to the job submit subprocess.
            # (Note this runs on the job host).
            env = os.environ
        else:
            # $HOME is required by job.sh on the job host.
            env = {'HOME': os.environ.get('HOME', '')}
        # Pass selected extra variables to the job submit subprocess.
        for var in self.env:
            env[var] = os.environ.get(var, '')
        if self.path is not None:
            # Append to avoid overriding an inherited PATH (e.g. in a venv)
            env['PATH'] = env.get('PATH', '') + ':' + ':'.join(self.path)
        if hasattr(job_runner, "submit"):
            submit_opts['env'] = env
            # job_runner.submit should handle OSError, if relevant.
            ret_code, out, err = job_runner.submit(job_file_path, submit_opts)
        else:
            proc_stdin_arg = None
            # Set command STDIN to DEVNULL by default to prevent leakage of
            # STDIN from current environment.
            proc_stdin_value = DEVNULL  # nosec
            if hasattr(job_runner, "get_submit_stdin"):
                proc_stdin_arg, proc_stdin_value = job_runner.get_submit_stdin(
                    job_file_path, submit_opts)
                if isinstance(proc_stdin_value, str):
                    proc_stdin_value = proc_stdin_value.encode()
            if hasattr(job_runner, "SUBMIT_CMD_ENV"):
                env.update(job_runner.SUBMIT_CMD_ENV)
            job_runner_cmd_tmpl = submit_opts.get("job_runner_cmd_tmpl")
            if job_runner_cmd_tmpl:
                # No need to catch OSError when using shell. It is unlikely
                # that we do not have a shell, and still manage to get as far
                # as here.
                job_runner_cmd = job_runner_cmd_tmpl % {"job": job_file_path}
                proc = procopen(job_runner_cmd, stdin=proc_stdin_arg,
                                stdoutpipe=True, stderrpipe=True, usesh=True,
                                env=env)
                # calls to open a shell are aggregated in
                # cylc_subproc.procopen()
            else:
                command = shlex.split(
                    job_runner.SUBMIT_CMD_TMPL % {"job": job_file_path})
                try:
                    proc = procopen(
                        command,
                        stdin=proc_stdin_arg,
                        stdoutpipe=True,
                        stderrpipe=True,
                        env=env,
                        # paths in directives should be interpreted relative to
                        # $HOME
                        # https://github.com/cylc/cylc-flow/issues/4247
                        cwd=Path('~').expanduser()
                    )
                except OSError as exc:
                    # subprocess.Popen has a bad habit of not setting the
                    # filename of the executable when it raises an OSError.
                    if not exc.filename:
                        exc.filename = command[0]
                    return 1, "", str(exc), ""
            out, err = (f.decode() for f in proc.communicate(proc_stdin_value))
            ret_code = proc.wait()
            with suppress(AttributeError, IOError):
                proc_stdin_arg.close()

        # Filter submit command output, if relevant
        # Get job ID, if possible
        job_id = None
        if out or err:
            try:
                out, err, job_id = self._filter_submit_output(
                    f"{job_file_path}.status", job_runner, out, err)
            except OSError:
                ret_code = 1
                self.job_kill(f"{job_file_path}.status")

        return ret_code, out, err, job_id

    def _jobs_submit_prep_by_args(self, job_log_root, job_log_dirs):
        """Prepare job files for submit by reading files in arguments.

        Job files are specified in the arguments in local mode. Extract job
        submission methods and job submission command templates from each job
        file.

        Return a list, where each element contains something like:
        (job_log_dir, job_runner_name, submit_opts)

        """
        items = []
        for job_log_dir in job_log_dirs:
            job_file_path = os.path.join(job_log_root, job_log_dir, "job")
            job_runner_name = None
            submit_opts = {}
            with open(job_file_path, 'r') as job_file:
                for line in job_file:
                    if line.startswith(self.LINE_PREFIX_JOB_RUNNER_NAME):
                        job_runner_name = line.replace(
                            self.LINE_PREFIX_JOB_RUNNER_NAME, "").strip()
                    elif line.startswith(self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL):
                        submit_opts["job_runner_cmd_tmpl"] = line.replace(
                            self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL, "").strip()
                    elif line.startswith(
                        self.LINE_PREFIX_EXECUTION_TIME_LIMIT
                    ):
                        submit_opts["execution_time_limit"] = float(
                            line.replace(
                                self.LINE_PREFIX_EXECUTION_TIME_LIMIT, ""
                            ).strip()
                        )
            items.append((job_log_dir, job_runner_name, submit_opts))
        return items

    def _jobs_submit_prep_by_stdin(self, job_log_root, job_log_dirs):
        """Prepare job files for submit by reading from STDIN.

        Job files are uploaded via STDIN in remote mode. Extract job submission
        methods and job submission command templates from each job file.

        Return a list, where each element contains something like:
        (job_log_dir, job_runner_name, submit_opts)

        """
        items = [[job_log_dir, None, {}] for job_log_dir in job_log_dirs]
        items_map = {}
        for item in items:
            items_map[item[0]] = item
        handle = None
        job_runner_name = None
        submit_opts = {}
        job_log_dir = None
        lines = []
        # Get job files from STDIN.
        # Get job runner name and job runner command template from each job
        # file.
        # Write job file in correct location.
        while True:  # Note: "for cur_line in sys.stdin:" may hang
            cur_line = sys.stdin.readline()
            if not cur_line:
                if handle is not None:
                    handle.close()
                break
            if cur_line.startswith(self.LINE_PREFIX_JOB_RUNNER_NAME):
                job_runner_name = cur_line.replace(
                    self.LINE_PREFIX_JOB_RUNNER_NAME, "").strip()
            elif cur_line.startswith(self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL):
                submit_opts["job_runner_cmd_tmpl"] = cur_line.replace(
                    self.LINE_PREFIX_JOB_RUNNER_CMD_TMPL, "").strip()
            elif cur_line.startswith(self.LINE_PREFIX_EXECUTION_TIME_LIMIT):
                submit_opts["execution_time_limit"] = float(cur_line.replace(
                    self.LINE_PREFIX_EXECUTION_TIME_LIMIT, "").strip())
            elif cur_line.startswith(self.LINE_PREFIX_JOB_LOG_DIR):
                job_log_dir = cur_line.replace(
                    self.LINE_PREFIX_JOB_LOG_DIR, "").strip()
                os.makedirs(
                    os.path.join(job_log_root, job_log_dir),
                    exist_ok=True)
                if handle is not None:
                    handle.close()
                handle = open(  # noqa: SIM115 (can't convert to with open)
                    os.path.join(job_log_root, job_log_dir, "job.tmp"), "wb")

            if handle is None:
                lines.append(cur_line)
            else:
                for line in lines + [cur_line]:
                    handle.write(line.encode())
                lines = []
                if cur_line.startswith(self.LINE_PREFIX_EOF + job_log_dir):
                    handle.close()
                    # Make it executable
                    os.chmod(handle.name, (
                        os.stat(handle.name).st_mode |
                        stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH))
                    # Rename from "*/job.tmp" to "*/job"
                    os.rename(handle.name, handle.name[:-4])
                    try:
                        items_map[job_log_dir][1] = job_runner_name
                        items_map[job_log_dir][2] = submit_opts
                    except KeyError:
                        pass
                    handle = None
                    job_log_dir = None
                    job_runner_name = None
                    submit_opts = {}
        return items
