path: root/src/core/job.c

/* SPDX-License-Identifier: LGPL-2.1+ */
/***
  Copyright 2010 Lennart Poettering
***/

#include <errno.h>

#include "sd-id128.h"
#include "sd-messages.h"

#include "alloc-util.h"
#include "async.h"
#include "dbus-job.h"
#include "dbus.h"
#include "escape.h"
#include "job.h"
#include "log.h"
#include "macro.h"
#include "parse-util.h"
#include "set.h"
#include "special.h"
#include "stdio-util.h"
#include "string-table.h"
#include "string-util.h"
#include "strv.h"
#include "terminal-util.h"
#include "unit.h"
#include "virt.h"

Job* job_new_raw(Unit *unit) {
        Job *j;

        /* used for deserialization */

        assert(unit);

        j = new0(Job, 1);
        if (!j)
                return NULL;

        j->manager = unit->manager;
        j->unit = unit;
        j->type = _JOB_TYPE_INVALID;
        j->reloaded = false;

        return j;
}

Job* job_new(Unit *unit, JobType type) {
        Job *j;

        assert(type < _JOB_TYPE_MAX);

        j = job_new_raw(unit);
        if (!j)
                return NULL;

        j->id = j->manager->current_job_id++;
        j->type = type;

        /* We don't link it here, that's what job_dependency() is for */

        return j;
}

void job_unlink(Job *j) {
        assert(j);
        assert(!j->installed);
        assert(!j->transaction_prev);
        assert(!j->transaction_next);
        assert(!j->subject_list);
        assert(!j->object_list);

        if (j->in_run_queue) {
                LIST_REMOVE(run_queue, j->manager->run_queue, j);
                j->in_run_queue = false;
        }

        if (j->in_dbus_queue) {
                LIST_REMOVE(dbus_queue, j->manager->dbus_job_queue, j);
                j->in_dbus_queue = false;
        }

        if (j->in_gc_queue) {
                LIST_REMOVE(gc_queue, j->manager->gc_job_queue, j);
                j->in_gc_queue = false;
        }

        j->timer_event_source = sd_event_source_unref(j->timer_event_source);
}

void job_free(Job *j) {
        assert(j);
        assert(!j->installed);
        assert(!j->transaction_prev);
        assert(!j->transaction_next);
        assert(!j->subject_list);
        assert(!j->object_list);

        job_unlink(j);

        sd_bus_track_unref(j->bus_track);
        strv_free(j->deserialized_clients);

        free(j);
}

static void job_set_state(Job *j, JobState state) {
        assert(j);
        assert(state >= 0);
        assert(state < _JOB_STATE_MAX);

        if (j->state == state)
                return;

        j->state = state;

        if (!j->installed)
                return;

        if (j->state == JOB_RUNNING)
                j->unit->manager->n_running_jobs++;
        else {
                assert(j->state == JOB_WAITING);
                assert(j->unit->manager->n_running_jobs > 0);

                j->unit->manager->n_running_jobs--;

                if (j->unit->manager->n_running_jobs <= 0)
                        j->unit->manager->jobs_in_progress_event_source = sd_event_source_unref(j->unit->manager->jobs_in_progress_event_source);
        }
}

void job_uninstall(Job *j) {
        Job **pj;

        assert(j->installed);

        job_set_state(j, JOB_WAITING);

        pj = (j->type == JOB_NOP) ? &j->unit->nop_job : &j->unit->job;
        assert(*pj == j);

        /* Detach from next 'bigger' objects */

        /* daemon-reload should be transparent to job observers */
        if (!MANAGER_IS_RELOADING(j->manager))
                bus_job_send_removed_signal(j);

        *pj = NULL;

        unit_add_to_gc_queue(j->unit);

        hashmap_remove(j->manager->jobs, UINT32_TO_PTR(j->id));
        j->installed = false;
}

static bool job_type_allows_late_merge(JobType t) {
        /* Tells whether it is OK to merge a job of type 't' with an already
         * running job.
         * Reloads cannot be merged this way. Think of the sequence:
         * 1. Reload of a daemon is in progress; the daemon has already loaded
         *    its config file, but hasn't completed the reload operation yet.
         * 2. Edit foo's config file.
         * 3. Trigger another reload to have the daemon use the new config.
         * Should the second reload job be merged into the first one, the daemon
         * would not know about the new config.
         * JOB_RESTART jobs on the other hand can be merged, because they get
         * patched into JOB_START after stopping the unit. So if we see a
         * JOB_RESTART running, it means the unit hasn't stopped yet and at
         * this time the merge is still allowed. */
        return t != JOB_RELOAD;
}

static void job_merge_into_installed(Job *j, Job *other) {
        assert(j->installed);
        assert(j->unit == other->unit);

        if (j->type != JOB_NOP)
                job_type_merge_and_collapse(&j->type, other->type, j->unit);
        else
                assert(other->type == JOB_NOP);

        j->irreversible = j->irreversible || other->irreversible;
        j->ignore_order = j->ignore_order || other->ignore_order;
}

Job* job_install(Job *j) {
        Job **pj;
        Job *uj;

        assert(!j->installed);
        assert(j->type < _JOB_TYPE_MAX_IN_TRANSACTION);
        assert(j->state == JOB_WAITING);

        pj = (j->type == JOB_NOP) ? &j->unit->nop_job : &j->unit->job;
        uj = *pj;

        if (uj) {
                if (job_type_is_conflicting(uj->type, j->type))
                        job_finish_and_invalidate(uj, JOB_CANCELED, false, false);
                else {
                        /* not conflicting, i.e. mergeable */

                        if (uj->state == JOB_WAITING ||
                            (job_type_allows_late_merge(j->type) && job_type_is_superset(uj->type, j->type))) {
                                job_merge_into_installed(uj, j);
                                log_unit_debug(uj->unit,
                                               "Merged into installed job %s/%s as %u",
                                               uj->unit->id, job_type_to_string(uj->type), (unsigned) uj->id);
                                return uj;
                        } else {
                                /* already running and not safe to merge into */
                                /* Patch uj to become a merged job and re-run it. */
                                /* XXX It should be safer to queue j to run after uj finishes, but it is
                                 * not currently possible to have more than one installed job per unit. */
                                job_merge_into_installed(uj, j);
                                log_unit_debug(uj->unit,
                                               "Merged into running job, re-running: %s/%s as %u",
                                               uj->unit->id, job_type_to_string(uj->type), (unsigned) uj->id);

                                job_set_state(uj, JOB_WAITING);
                                return uj;
                        }
                }
        }

        /* Install the job */
        *pj = j;
        j->installed = true;

        j->manager->n_installed_jobs++;
        log_unit_debug(j->unit,
                       "Installed new job %s/%s as %u",
                       j->unit->id, job_type_to_string(j->type), (unsigned) j->id);

        job_add_to_gc_queue(j);

        return j;
}

int job_install_deserialized(Job *j) {
        Job **pj;

        assert(!j->installed);

        if (j->type < 0 || j->type >= _JOB_TYPE_MAX_IN_TRANSACTION) {
                log_debug("Invalid job type %s in deserialization.", strna(job_type_to_string(j->type)));
                return -EINVAL;
        }

        pj = (j->type == JOB_NOP) ? &j->unit->nop_job : &j->unit->job;
        if (*pj) {
                log_unit_debug(j->unit, "Unit already has a job installed. Not installing deserialized job.");
                return -EEXIST;
        }

        *pj = j;
        j->installed = true;
        j->reloaded = true;

        if (j->state == JOB_RUNNING)
                j->unit->manager->n_running_jobs++;

        log_unit_debug(j->unit,
                       "Reinstalled deserialized job %s/%s as %u",
                       j->unit->id, job_type_to_string(j->type), (unsigned) j->id);
        return 0;
}

JobDependency* job_dependency_new(Job *subject, Job *object, bool matters, bool conflicts) {
        JobDependency *l;

        assert(object);

        /* Adds a new job link, which encodes that the 'subject' job
         * needs the 'object' job in some way. If 'subject' is NULL
         * this means the 'anchor' job (i.e. the one the user
         * explicitly asked for) is the requester. */

        l = new0(JobDependency, 1);
        if (!l)
                return NULL;

        l->subject = subject;
        l->object = object;
        l->matters = matters;
        l->conflicts = conflicts;

        if (subject)
                LIST_PREPEND(subject, subject->subject_list, l);

        LIST_PREPEND(object, object->object_list, l);

        return l;
}

void job_dependency_free(JobDependency *l) {
        assert(l);

        if (l->subject)
                LIST_REMOVE(subject, l->subject->subject_list, l);

        LIST_REMOVE(object, l->object->object_list, l);

        free(l);
}

void job_dump(Job *j, FILE*f, const char *prefix) {
        assert(j);
        assert(f);

        prefix = strempty(prefix);

        fprintf(f,
                "%s-> Job %u:\n"
                "%s\tAction: %s -> %s\n"
                "%s\tState: %s\n"
                "%s\tIrreversible: %s\n"
                "%s\tMay GC: %s\n",
                prefix, j->id,
                prefix, j->unit->id, job_type_to_string(j->type),
                prefix, job_state_to_string(j->state),
                prefix, yes_no(j->irreversible),
                prefix, yes_no(job_may_gc(j)));
}

/*
 * Merging is commutative, so imagine the matrix as symmetric. We store only
 * its lower triangle to avoid duplication. We don't store the main diagonal,
 * because A merged with A is simply A.
 *
 * If the resulting type is collapsed immediately afterwards (to get rid of
 * the JOB_RELOAD_OR_START, which lies outside the lookup function's domain),
 * the following properties hold:
 *
 * Merging is associative! A merged with B, and then merged with C is the same
 * as A merged with the result of B merged with C.
 *
 * Mergeability is transitive! If A can be merged with B and B with C then
 * A also with C.
 *
 * Also, if A merged with B cannot be merged with C, then either A or B cannot
 * be merged with C either.
 */
static const JobType job_merging_table[] = {
/* What \ With       *  JOB_START         JOB_VERIFY_ACTIVE  JOB_STOP JOB_RELOAD */
/*********************************************************************************/
/*JOB_START          */
/*JOB_VERIFY_ACTIVE  */ JOB_START,
/*JOB_STOP           */ -1,                  -1,
/*JOB_RELOAD         */ JOB_RELOAD_OR_START, JOB_RELOAD,          -1,
/*JOB_RESTART        */ JOB_RESTART,         JOB_RESTART,         -1, JOB_RESTART,
};

JobType job_type_lookup_merge(JobType a, JobType b) {
        assert_cc(ELEMENTSOF(job_merging_table) == _JOB_TYPE_MAX_MERGING * (_JOB_TYPE_MAX_MERGING - 1) / 2);
        assert(a >= 0 && a < _JOB_TYPE_MAX_MERGING);
        assert(b >= 0 && b < _JOB_TYPE_MAX_MERGING);

        if (a == b)
                return a;

        if (a < b) {
                JobType tmp = a;
                a = b;
                b = tmp;
        }

        return job_merging_table[(a - 1) * a / 2 + b];
}

bool job_type_is_redundant(JobType a, UnitActiveState b) {
        switch (a) {

        case JOB_START:
                return IN_SET(b, UNIT_ACTIVE, UNIT_RELOADING);

        case JOB_STOP:
                return IN_SET(b, UNIT_INACTIVE, UNIT_FAILED);

        case JOB_VERIFY_ACTIVE:
                return IN_SET(b, UNIT_ACTIVE, UNIT_RELOADING);

        case JOB_RELOAD:
                return
                        b == UNIT_RELOADING;

        case JOB_RESTART:
                return
                        b == UNIT_ACTIVATING;

        case JOB_NOP:
                return true;

        default:
                assert_not_reached("Invalid job type");
        }
}

JobType job_type_collapse(JobType t, Unit *u) {
        UnitActiveState s;

        switch (t) {

        case JOB_TRY_RESTART:
                s = unit_active_state(u);
                if (UNIT_IS_INACTIVE_OR_DEACTIVATING(s))
                        return JOB_NOP;

                return JOB_RESTART;

        case JOB_TRY_RELOAD:
                s = unit_active_state(u);
                if (UNIT_IS_INACTIVE_OR_DEACTIVATING(s))
                        return JOB_NOP;

                return JOB_RELOAD;

        case JOB_RELOAD_OR_START:
                s = unit_active_state(u);
                if (UNIT_IS_INACTIVE_OR_DEACTIVATING(s))
                        return JOB_START;

                return JOB_RELOAD;

        default:
                return t;
        }
}

int job_type_merge_and_collapse(JobType *a, JobType b, Unit *u) {
        JobType t;

        t = job_type_lookup_merge(*a, b);
        if (t < 0)
                return -EEXIST;

        *a = job_type_collapse(t, u);
        return 0;
}

static bool job_is_runnable(Job *j) {
        Iterator i;
        Unit *other;
        void *v;

        assert(j);
        assert(j->installed);

        /* Checks whether there is any job running for the units this
         * job needs to be running after (in the case of a 'positive'
         * job type) or before (in the case of a 'negative' job
         * type. */

        /* Note that unit types have a say in what is runnable,
         * too. For example, if they return -EAGAIN from
         * unit_start() they can indicate they are not
         * runnable yet. */

        /* First check if there is an override */
        if (j->ignore_order)
                return true;

        if (j->type == JOB_NOP)
                return true;

        if (IN_SET(j->type, JOB_START, JOB_VERIFY_ACTIVE, JOB_RELOAD)) {
                /* Immediate result is that the job is or might be
                 * started. In this case let's wait for the
                 * dependencies, regardless whether they are
                 * starting or stopping something. */

                HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_AFTER], i)
                        if (other->job)
                                return false;
        }

        /* Also, if something else is being stopped and we should
         * change state after it, then let's wait. */

        HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_BEFORE], i)
                if (other->job &&
                    IN_SET(other->job->type, JOB_STOP, JOB_RESTART))
                        return false;

        /* This means that for a service a and a service b where b
         * shall be started after a:
         *
         *  start a + start b → 1st step start a, 2nd step start b
         *  start a + stop b  → 1st step stop b,  2nd step start a
         *  stop a  + start b → 1st step stop a,  2nd step start b
         *  stop a  + stop b  → 1st step stop b,  2nd step stop a
         *
         *  This has the side effect that restarts are properly
         *  synchronized too. */

        return true;
}

static void job_change_type(Job *j, JobType newtype) {
        assert(j);

        log_unit_debug(j->unit,
                       "Converting job %s/%s -> %s/%s",
                       j->unit->id, job_type_to_string(j->type),
                       j->unit->id, job_type_to_string(newtype));

        j->type = newtype;
}

static int job_perform_on_unit(Job **j) {
        uint32_t id;
        Manager *m;
        JobType t;
        Unit *u;
        int r;

        /* While we execute this operation the job might go away (for
         * example: because it finishes immediately or is replaced by
         * a new, conflicting job.) To make sure we don't access a
         * freed job later on we store the id here, so that we can
         * verify the job is still valid. */

        assert(j);
        assert(*j);

        m = (*j)->manager;
        u = (*j)->unit;
        t = (*j)->type;
        id = (*j)->id;

        switch (t) {
                case JOB_START:
                        r = unit_start(u);
                        break;

                case JOB_RESTART:
                        t = JOB_STOP;
                        _fallthrough_;
                case JOB_STOP:
                        r = unit_stop(u);
                        break;

                case JOB_RELOAD:
                        r = unit_reload(u);
                        break;

                default:
                        assert_not_reached("Invalid job type");
        }

        /* Log if the job still exists and the start/stop/reload function
         * actually did something. */
        *j = manager_get_job(m, id);
        if (*j && r > 0)
                unit_status_emit_starting_stopping_reloading(u, t);

        return r;
}

int job_run_and_invalidate(Job *j) {
        int r;

        assert(j);
        assert(j->installed);
        assert(j->type < _JOB_TYPE_MAX_IN_TRANSACTION);
        assert(j->in_run_queue);

        LIST_REMOVE(run_queue, j->manager->run_queue, j);
        j->in_run_queue = false;

        if (j->state != JOB_WAITING)
                return 0;

        if (!job_is_runnable(j))
                return -EAGAIN;

        job_start_timer(j, true);
        job_set_state(j, JOB_RUNNING);
        job_add_to_dbus_queue(j);

        switch (j->type) {

                case JOB_VERIFY_ACTIVE: {
                        UnitActiveState t = unit_active_state(j->unit);
                        if (UNIT_IS_ACTIVE_OR_RELOADING(t))
                                r = -EALREADY;
                        else if (t == UNIT_ACTIVATING)
                                r = -EAGAIN;
                        else
                                r = -EBADR;
                        break;
                }

                case JOB_START:
                case JOB_STOP:
                case JOB_RESTART:
                        r = job_perform_on_unit(&j);

                        /* If the unit type does not support starting/stopping,
                         * then simply wait. */
                        if (r == -EBADR)
                                r = 0;
                        break;

                case JOB_RELOAD:
                        r = job_perform_on_unit(&j);
                        break;

                case JOB_NOP:
                        r = -EALREADY;
                        break;

                default:
                        assert_not_reached("Unknown job type");
        }

        if (j) {
                if (r == -EALREADY)
                        r = job_finish_and_invalidate(j, JOB_DONE, true, true);
                else if (r == -EBADR)
                        r = job_finish_and_invalidate(j, JOB_SKIPPED, true, false);
                else if (r == -ENOEXEC)
                        r = job_finish_and_invalidate(j, JOB_INVALID, true, false);
                else if (r == -EPROTO)
                        r = job_finish_and_invalidate(j, JOB_ASSERT, true, false);
                else if (r == -EOPNOTSUPP)
                        r = job_finish_and_invalidate(j, JOB_UNSUPPORTED, true, false);
                else if (r == -ENOLINK)
                        r = job_finish_and_invalidate(j, JOB_DEPENDENCY, true, false);
                else if (r == -ESTALE)
                        r = job_finish_and_invalidate(j, JOB_ONCE, true, false);
                else if (r == -EAGAIN)
                        job_set_state(j, JOB_WAITING);
                else if (r < 0)
                        r = job_finish_and_invalidate(j, JOB_FAILED, true, false);
        }

        return r;
}

_pure_ static const char *job_get_status_message_format(Unit *u, JobType t, JobResult result) {

        static const char *const generic_finished_start_job[_JOB_RESULT_MAX] = {
                [JOB_DONE]        = "Started %s.",
                [JOB_TIMEOUT]     = "Timed out starting %s.",
                [JOB_FAILED]      = "Failed to start %s.",
                [JOB_DEPENDENCY]  = "Dependency failed for %s.",
                [JOB_ASSERT]      = "Assertion failed for %s.",
                [JOB_UNSUPPORTED] = "Starting of %s not supported.",
                [JOB_COLLECTED]   = "Unnecessary job for %s was removed.",
                [JOB_ONCE]        = "Unit %s has been started before and cannot be started again."
        };
        static const char *const generic_finished_stop_job[_JOB_RESULT_MAX] = {
                [JOB_DONE]        = "Stopped %s.",
                [JOB_FAILED]      = "Stopped (with error) %s.",
                [JOB_TIMEOUT]     = "Timed out stopping %s.",
        };
        static const char *const generic_finished_reload_job[_JOB_RESULT_MAX] = {
                [JOB_DONE]        = "Reloaded %s.",
                [JOB_FAILED]      = "Reload failed for %s.",
                [JOB_TIMEOUT]     = "Timed out reloading %s.",
        };
        /* When verify-active detects the unit is inactive, report it.
         * Most likely a DEPEND warning from a requisiting unit will
         * occur next and it's nice to see what was requisited. */
        static const char *const generic_finished_verify_active_job[_JOB_RESULT_MAX] = {
                [JOB_SKIPPED]     = "%s is not active.",
        };

        const UnitStatusMessageFormats *format_table;
        const char *format;

        assert(u);
        assert(t >= 0);
        assert(t < _JOB_TYPE_MAX);

        if (IN_SET(t, JOB_START, JOB_STOP, JOB_RESTART)) {
                format_table = &UNIT_VTABLE(u)->status_message_formats;
                if (format_table) {
                        format = t == JOB_START ? format_table->finished_start_job[result] :
                                                  format_table->finished_stop_job[result];
                        if (format)
                                return format;
                }
        }

        /* Return generic strings */
        if (t == JOB_START)
                return generic_finished_start_job[result];
        else if (IN_SET(t, JOB_STOP, JOB_RESTART))
                return generic_finished_stop_job[result];
        else if (t == JOB_RELOAD)
                return generic_finished_reload_job[result];
        else if (t == JOB_VERIFY_ACTIVE)
                return generic_finished_verify_active_job[result];

        return NULL;
}

static const struct {
        const char *color, *word;
} job_print_status_messages [_JOB_RESULT_MAX] = {
        [JOB_DONE]        = { ANSI_OK_COLOR,         "  OK  " },
        [JOB_TIMEOUT]     = { ANSI_HIGHLIGHT_RED,    " TIME " },
        [JOB_FAILED]      = { ANSI_HIGHLIGHT_RED,    "FAILED" },
        [JOB_DEPENDENCY]  = { ANSI_HIGHLIGHT_YELLOW, "DEPEND" },
        [JOB_SKIPPED]     = { ANSI_HIGHLIGHT,        " INFO " },
        [JOB_ASSERT]      = { ANSI_HIGHLIGHT_YELLOW, "ASSERT" },
        [JOB_UNSUPPORTED] = { ANSI_HIGHLIGHT_YELLOW, "UNSUPP" },
        /* JOB_COLLECTED */
        [JOB_ONCE]        = { ANSI_HIGHLIGHT_RED,    " ONCE " },
};

static void job_print_status_message(Unit *u, JobType t, JobResult result) {
        const char *format;
        const char *status;

        assert(u);
        assert(t >= 0);
        assert(t < _JOB_TYPE_MAX);

        /* Reload status messages have traditionally not been printed to console. */
        if (t == JOB_RELOAD)
                return;

        if (!job_print_status_messages[result].word)
                return;

        format = job_get_status_message_format(u, t, result);
        if (!format)
                return;

        if (log_get_show_color())
                status = strjoina(job_print_status_messages[result].color,
                                  job_print_status_messages[result].word,
                                  ANSI_NORMAL);
        else
                status = job_print_status_messages[result].word;

        if (result != JOB_DONE)
                manager_flip_auto_status(u->manager, true);

        DISABLE_WARNING_FORMAT_NONLITERAL;
        unit_status_printf(u, status, format);
        REENABLE_WARNING;

        if (t == JOB_START && result == JOB_FAILED) {
                _cleanup_free_ char *quoted;

                quoted = shell_maybe_quote(u->id, ESCAPE_BACKSLASH);
                manager_status_printf(u->manager, STATUS_TYPE_NORMAL, NULL, "See 'systemctl status %s' for details.", strna(quoted));
        }
}

static void job_log_status_message(Unit *u, JobType t, JobResult result) {
        const char *format, *mid;
        char buf[LINE_MAX];
        static const int job_result_log_level[_JOB_RESULT_MAX] = {
                [JOB_DONE]        = LOG_INFO,
                [JOB_CANCELED]    = LOG_INFO,
                [JOB_TIMEOUT]     = LOG_ERR,
                [JOB_FAILED]      = LOG_ERR,
                [JOB_DEPENDENCY]  = LOG_WARNING,
                [JOB_SKIPPED]     = LOG_NOTICE,
                [JOB_INVALID]     = LOG_INFO,
                [JOB_ASSERT]      = LOG_WARNING,
                [JOB_UNSUPPORTED] = LOG_WARNING,
                [JOB_COLLECTED]   = LOG_INFO,
                [JOB_ONCE]        = LOG_ERR,
        };

        assert(u);
        assert(t >= 0);
        assert(t < _JOB_TYPE_MAX);

        /* Skip printing if output goes to the console, and job_print_status_message()
           will actually print something to the console. */
        if (log_on_console() && job_print_status_messages[result].word)
                return;

        format = job_get_status_message_format(u, t, result);
        if (!format)
                return;

        /* The description might be longer than the buffer, but that's OK,
         * we'll just truncate it here. Note that we use snprintf() rather than
         * xsprintf() on purpose here: we are fine with truncation and don't
         * consider that an error. */
        DISABLE_WARNING_FORMAT_NONLITERAL;
        (void) snprintf(buf, sizeof(buf), format, unit_description(u));
        REENABLE_WARNING;

        switch (t) {

        case JOB_START:
                if (result == JOB_DONE)
                        mid = "MESSAGE_ID=" SD_MESSAGE_UNIT_STARTED_STR;
                else
                        mid = "MESSAGE_ID=" SD_MESSAGE_UNIT_FAILED_STR;
                break;

        case JOB_RELOAD:
                mid = "MESSAGE_ID=" SD_MESSAGE_UNIT_RELOADED_STR;
                break;

        case JOB_STOP:
        case JOB_RESTART:
                mid = "MESSAGE_ID=" SD_MESSAGE_UNIT_STOPPED_STR;
                break;

        default:
                log_struct(job_result_log_level[result],
                           LOG_MESSAGE("%s", buf),
                           "JOB_TYPE=%s", job_type_to_string(t),
                           "JOB_RESULT=%s", job_result_to_string(result),
                           LOG_UNIT_ID(u),
                           LOG_UNIT_INVOCATION_ID(u));
                return;
        }

        log_struct(job_result_log_level[result],
                   LOG_MESSAGE("%s", buf),
                   "JOB_TYPE=%s", job_type_to_string(t),
                   "JOB_RESULT=%s", job_result_to_string(result),
                   LOG_UNIT_ID(u),
                   LOG_UNIT_INVOCATION_ID(u),
                   mid);
}

static void job_emit_status_message(Unit *u, JobType t, JobResult result) {
        assert(u);

        /* No message if the job did not actually do anything due to failed condition. */
        if (t == JOB_START && result == JOB_DONE && !u->condition_result)
                return;

        job_log_status_message(u, t, result);
        job_print_status_message(u, t, result);
}

static void job_fail_dependencies(Unit *u, UnitDependency d) {
        Unit *other;
        Iterator i;
        void *v;

        assert(u);

        HASHMAP_FOREACH_KEY(v, other, u->dependencies[d], i) {
                Job *j = other->job;

                if (!j)
                        continue;
                if (!IN_SET(j->type, JOB_START, JOB_VERIFY_ACTIVE))
                        continue;

                job_finish_and_invalidate(j, JOB_DEPENDENCY, true, false);
        }
}

static int job_save_pending_finished_job(Job *j) {
        int r;

        assert(j);

        r = set_ensure_allocated(&j->manager->pending_finished_jobs, NULL);
        if (r < 0)
                return r;

        job_unlink(j);
        return set_put(j->manager->pending_finished_jobs, j);
}

int job_finish_and_invalidate(Job *j, JobResult result, bool recursive, bool already) {
        Unit *u;
        Unit *other;
        JobType t;
        Iterator i;
        void *v;

        assert(j);
        assert(j->installed);
        assert(j->type < _JOB_TYPE_MAX_IN_TRANSACTION);

        u = j->unit;
        t = j->type;

        j->result = result;

        log_unit_debug(u, "Job %s/%s finished, result=%s", u->id, job_type_to_string(t), job_result_to_string(result));

        /* If this job did nothing to respective unit we don't log the status message */
        if (!already)
                job_emit_status_message(u, t, result);

        /* Patch restart jobs so that they become normal start jobs */
        if (result == JOB_DONE && t == JOB_RESTART) {

                job_change_type(j, JOB_START);
                job_set_state(j, JOB_WAITING);

                job_add_to_dbus_queue(j);
                job_add_to_run_queue(j);
                job_add_to_gc_queue(j);

                goto finish;
        }

        if (IN_SET(result, JOB_FAILED, JOB_INVALID))
                j->manager->n_failed_jobs++;

        job_uninstall(j);
        /* Keep jobs started before the reload to send singal later, free all others */
        if (!MANAGER_IS_RELOADING(j->manager) ||
            !j->reloaded ||
            job_save_pending_finished_job(j) < 0)
                job_free(j);

        /* Fail depending jobs on failure */
        if (result != JOB_DONE && recursive) {
                if (IN_SET(t, JOB_START, JOB_VERIFY_ACTIVE)) {
                        job_fail_dependencies(u, UNIT_REQUIRED_BY);
                        job_fail_dependencies(u, UNIT_REQUISITE_OF);
                        job_fail_dependencies(u, UNIT_BOUND_BY);
                } else if (t == JOB_STOP)
                        job_fail_dependencies(u, UNIT_CONFLICTED_BY);
        }

        /* Trigger OnFailure dependencies that are not generated by
         * the unit itself. We don't treat JOB_CANCELED as failure in
         * this context. And JOB_FAILURE is already handled by the
         * unit itself. */
        if (IN_SET(result, JOB_TIMEOUT, JOB_DEPENDENCY)) {
                log_struct(LOG_NOTICE,
                           "JOB_TYPE=%s", job_type_to_string(t),
                           "JOB_RESULT=%s", job_result_to_string(result),
                           LOG_UNIT_ID(u),
                           LOG_UNIT_MESSAGE(u, "Job %s/%s failed with result '%s'.",
                                            u->id,
                                            job_type_to_string(t),
                                            job_result_to_string(result)));

                unit_start_on_failure(u);
        }

        unit_trigger_notify(u);

finish:
        /* Try to start the next jobs that can be started */
        HASHMAP_FOREACH_KEY(v, other, u->dependencies[UNIT_AFTER], i)
                if (other->job) {
                        job_add_to_run_queue(other->job);
                        job_add_to_gc_queue(other->job);
                }
        HASHMAP_FOREACH_KEY(v, other, u->dependencies[UNIT_BEFORE], i)
                if (other->job) {
                        job_add_to_run_queue(other->job);
                        job_add_to_gc_queue(other->job);
                }

        manager_check_finished(u->manager);

        return 0;
}

static int job_dispatch_timer(sd_event_source *s, uint64_t monotonic, void *userdata) {
        Job *j = userdata;
        Unit *u;

        assert(j);
        assert(s == j->timer_event_source);

        log_unit_warning(j->unit, "Job %s/%s timed out.", j->unit->id, job_type_to_string(j->type));

        u = j->unit;
        job_finish_and_invalidate(j, JOB_TIMEOUT, true, false);

        emergency_action(u->manager, u->job_timeout_action, u->job_timeout_reboot_arg, "job timed out");

        return 0;
}

int job_start_timer(Job *j, bool job_running) {
        int r;
        usec_t timeout_time, old_timeout_time;

        if (job_running) {
                j->begin_running_usec = now(CLOCK_MONOTONIC);

                if (j->unit->job_running_timeout == USEC_INFINITY)
                        return 0;

                timeout_time = usec_add(j->begin_running_usec, j->unit->job_running_timeout);

                if (j->timer_event_source) {
                        /* Update only if JobRunningTimeoutSec= results in earlier timeout */
                        r = sd_event_source_get_time(j->timer_event_source, &old_timeout_time);
                        if (r < 0)
                                return r;

                        if (old_timeout_time <= timeout_time)
                                return 0;

                        return sd_event_source_set_time(j->timer_event_source, timeout_time);
                }
        } else {
                if (j->timer_event_source)
                        return 0;

                j->begin_usec = now(CLOCK_MONOTONIC);

                if (j->unit->job_timeout == USEC_INFINITY)
                        return 0;

                timeout_time = usec_add(j->begin_usec, j->unit->job_timeout);
        }

        r = sd_event_add_time(
                        j->manager->event,
                        &j->timer_event_source,
                        CLOCK_MONOTONIC,
                        timeout_time, 0,
                        job_dispatch_timer, j);
        if (r < 0)
                return r;

        (void) sd_event_source_set_description(j->timer_event_source, "job-start");

        return 0;
}

void job_add_to_run_queue(Job *j) {
        assert(j);
        assert(j->installed);

        if (j->in_run_queue)
                return;

        if (!j->manager->run_queue)
                sd_event_source_set_enabled(j->manager->run_queue_event_source, SD_EVENT_ONESHOT);

        LIST_PREPEND(run_queue, j->manager->run_queue, j);
        j->in_run_queue = true;
}

void job_add_to_dbus_queue(Job *j) {
        assert(j);
        assert(j->installed);

        if (j->in_dbus_queue)
                return;

        /* We don't check if anybody is subscribed here, since this
         * job might just have been created and not yet assigned to a
         * connection/client. */

        LIST_PREPEND(dbus_queue, j->manager->dbus_job_queue, j);
        j->in_dbus_queue = true;
}

char *job_dbus_path(Job *j) {
        char *p;

        assert(j);

        if (asprintf(&p, "/org/freedesktop/systemd1/job/%"PRIu32, j->id) < 0)
                return NULL;

        return p;
}

int job_serialize(Job *j, FILE *f) {
        assert(j);
        assert(f);

        fprintf(f, "job-id=%u\n", j->id);
        fprintf(f, "job-type=%s\n", job_type_to_string(j->type));
        fprintf(f, "job-state=%s\n", job_state_to_string(j->state));
        fprintf(f, "job-irreversible=%s\n", yes_no(j->irreversible));
        fprintf(f, "job-sent-dbus-new-signal=%s\n", yes_no(j->sent_dbus_new_signal));
        fprintf(f, "job-ignore-order=%s\n", yes_no(j->ignore_order));

        if (j->begin_usec > 0)
                fprintf(f, "job-begin="USEC_FMT"\n", j->begin_usec);
        if (j->begin_running_usec > 0)
                fprintf(f, "job-begin-running="USEC_FMT"\n", j->begin_running_usec);

        bus_track_serialize(j->bus_track, f, "subscribed");

        /* End marker */
        fputc('\n', f);
        return 0;
}

int job_deserialize(Job *j, FILE *f) {
        assert(j);
        assert(f);

        for (;;) {
                char line[LINE_MAX], *l, *v;
                size_t k;

                if (!fgets(line, sizeof(line), f)) {
                        if (feof(f))
                                return 0;
                        return -errno;
                }

                char_array_0(line);
                l = strstrip(line);

                /* End marker */
                if (l[0] == 0)
                        return 0;

                k = strcspn(l, "=");

                if (l[k] == '=') {
                        l[k] = 0;
                        v = l+k+1;
                } else
                        v = l+k;

                if (streq(l, "job-id")) {

                        if (safe_atou32(v, &j->id) < 0)
                                log_debug("Failed to parse job id value %s", v);

                } else if (streq(l, "job-type")) {
                        JobType t;

                        t = job_type_from_string(v);
                        if (t < 0)
                                log_debug("Failed to parse job type %s", v);
                        else if (t >= _JOB_TYPE_MAX_IN_TRANSACTION)
                                log_debug("Cannot deserialize job of type %s", v);
                        else
                                j->type = t;

                } else if (streq(l, "job-state")) {
                        JobState s;

                        s = job_state_from_string(v);
                        if (s < 0)
                                log_debug("Failed to parse job state %s", v);
                        else
                                job_set_state(j, s);

                } else if (streq(l, "job-irreversible")) {
                        int b;

                        b = parse_boolean(v);
                        if (b < 0)
                                log_debug("Failed to parse job irreversible flag %s", v);
                        else
                                j->irreversible = j->irreversible || b;

                } else if (streq(l, "job-sent-dbus-new-signal")) {
                        int b;

                        b = parse_boolean(v);
                        if (b < 0)
                                log_debug("Failed to parse job sent_dbus_new_signal flag %s", v);
                        else
                                j->sent_dbus_new_signal = j->sent_dbus_new_signal || b;

                } else if (streq(l, "job-ignore-order")) {
                        int b;

                        b = parse_boolean(v);
                        if (b < 0)
                                log_debug("Failed to parse job ignore_order flag %s", v);
                        else
                                j->ignore_order = j->ignore_order || b;

                } else if (streq(l, "job-begin")) {
                        unsigned long long ull;

                        if (sscanf(v, "%llu", &ull) != 1)
                                log_debug("Failed to parse job-begin value %s", v);
                        else
                                j->begin_usec = ull;

                } else if (streq(l, "job-begin-running")) {
                        unsigned long long ull;

                        if (sscanf(v, "%llu", &ull) != 1)
                                log_debug("Failed to parse job-begin-running value %s", v);
                        else
                                j->begin_running_usec = ull;

                } else if (streq(l, "subscribed")) {

                        if (strv_extend(&j->deserialized_clients, v) < 0)
                                log_oom();
                }
        }
}

int job_coldplug(Job *j) {
        int r;
        usec_t timeout_time = USEC_INFINITY;

        assert(j);

        /* After deserialization is complete and the bus connection
         * set up again, let's start watching our subscribers again */
        (void) bus_job_coldplug_bus_track(j);

        if (j->state == JOB_WAITING)
                job_add_to_run_queue(j);

        /* Maybe due to new dependencies we don't actually need this job anymore? */
        job_add_to_gc_queue(j);

        /* Create timer only when job began or began running and the respective timeout is finite.
         * Follow logic of job_start_timer() if both timeouts are finite */
        if (j->begin_usec == 0)
                return 0;

        if (j->unit->job_timeout != USEC_INFINITY)
                timeout_time = usec_add(j->begin_usec, j->unit->job_timeout);

        if (j->begin_running_usec > 0 && j->unit->job_running_timeout != USEC_INFINITY)
                timeout_time = MIN(timeout_time, usec_add(j->begin_running_usec, j->unit->job_running_timeout));

        if (timeout_time == USEC_INFINITY)
                return 0;

        j->timer_event_source = sd_event_source_unref(j->timer_event_source);

        r = sd_event_add_time(
                        j->manager->event,
                        &j->timer_event_source,
                        CLOCK_MONOTONIC,
                        timeout_time, 0,
                        job_dispatch_timer, j);
        if (r < 0)
                log_debug_errno(r, "Failed to restart timeout for job: %m");

        (void) sd_event_source_set_description(j->timer_event_source, "job-timeout");

        return r;
}

void job_shutdown_magic(Job *j) {
        assert(j);

        /* The shutdown target gets some special treatment here: we
         * tell the kernel to begin with flushing its disk caches, to
         * optimize shutdown time a bit. Ideally we wouldn't hardcode
         * this magic into PID 1. However all other processes aren't
         * options either since they'd exit much sooner than PID 1 and
         * asynchronous sync() would cause their exit to be
         * delayed. */

        if (j->type != JOB_START)
                return;

        if (!MANAGER_IS_SYSTEM(j->unit->manager))
                return;

        if (!unit_has_name(j->unit, SPECIAL_SHUTDOWN_TARGET))
                return;

        /* In case messages on console has been disabled on boot */
        j->unit->manager->no_console_output = false;

        if (detect_container() > 0)
                return;

        (void) asynchronous_sync(NULL);
}

int job_get_timeout(Job *j, usec_t *timeout) {
        usec_t x = USEC_INFINITY, y = USEC_INFINITY;
        Unit *u = j->unit;
        int r;

        assert(u);

        if (j->timer_event_source) {
                r = sd_event_source_get_time(j->timer_event_source, &x);
                if (r < 0)
                        return r;
        }

        if (UNIT_VTABLE(u)->get_timeout) {
                r = UNIT_VTABLE(u)->get_timeout(u, &y);
                if (r < 0)
                        return r;
        }

        if (x == USEC_INFINITY && y == USEC_INFINITY)
                return 0;

        *timeout = MIN(x, y);
        return 1;
}

bool job_may_gc(Job *j) {
        Unit *other;
        Iterator i;
        void *v;

        assert(j);

        /* Checks whether this job should be GC'ed away. We only do this for jobs of units that have no effect on their
         * own and just track external state. For now the only unit type that qualifies for this are .device units.
         * Returns true if the job can be collected. */

        if (!UNIT_VTABLE(j->unit)->gc_jobs)
                return false;

        if (sd_bus_track_count(j->bus_track) > 0)
                return false;

        /* FIXME: So this is a bit ugly: for now we don't properly track references made via private bus connections
         * (because it's nasty, as sd_bus_track doesn't apply to it). We simply remember that the job was once
         * referenced by one, and reset this whenever we notice that no private bus connections are around. This means
         * the GC is a bit too conservative when it comes to jobs created by private bus connections. */
        if (j->ref_by_private_bus) {
                if (set_isempty(j->unit->manager->private_buses))
                        j->ref_by_private_bus = false;
                else
                        return false;
        }

        if (j->type == JOB_NOP)
                return false;

        /* If a job is ordered after ours, and is to be started, then it needs to wait for us, regardless if we stop or
         * start, hence let's not GC in that case. */
        HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_BEFORE], i) {
                if (!other->job)
                        continue;

                if (other->job->ignore_order)
                        continue;

                if (IN_SET(other->job->type, JOB_START, JOB_VERIFY_ACTIVE, JOB_RELOAD))
                        return false;
        }

        /* If we are going down, but something else is ordered After= us, then it needs to wait for us */
        if (IN_SET(j->type, JOB_STOP, JOB_RESTART))
                HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_AFTER], i) {
                        if (!other->job)
                                continue;

                        if (other->job->ignore_order)
                                continue;

                        return false;
                }

        /* The logic above is kinda the inverse of the job_is_runnable() logic. Specifically, if the job "we" is
         * ordered before the job "other":
         *
         *  we start + other start → stay
         *  we start + other stop  → gc
         *  we stop  + other start → stay
         *  we stop  + other stop  → gc
         *
         * "we" are ordered after "other":
         *
         *  we start + other start → gc
         *  we start + other stop  → gc
         *  we stop  + other start → stay
         *  we stop  + other stop  → stay
         *
         */

        return true;
}

void job_add_to_gc_queue(Job *j) {
        assert(j);

        if (j->in_gc_queue)
                return;

        if (!job_may_gc(j))
                return;

        LIST_PREPEND(gc_queue, j->unit->manager->gc_job_queue, j);
        j->in_gc_queue = true;
}

static int job_compare(const void *a, const void *b) {
        Job *x = *(Job**) a, *y = *(Job**) b;

        if (x->id < y->id)
                return -1;
        if (x->id > y->id)
                return 1;

        return 0;
}

static size_t sort_job_list(Job **list, size_t n) {
        Job *previous = NULL;
        size_t a, b;

        /* Order by numeric IDs */
        qsort_safe(list, n, sizeof(Job*), job_compare);

        /* Filter out duplicates */
        for (a = 0, b = 0; a < n; a++) {

                if (previous == list[a])
                        continue;

                previous = list[b++] = list[a];
        }

        return b;
}

int job_get_before(Job *j, Job*** ret) {
        _cleanup_free_ Job** list = NULL;
        size_t n = 0, n_allocated = 0;
        Unit *other = NULL;
        Iterator i;
        void *v;

        /* Returns a list of all pending jobs that need to finish before this job may be started. */

        assert(j);
        assert(ret);

        if (j->ignore_order) {
                *ret = NULL;
                return 0;
        }

        if (IN_SET(j->type, JOB_START, JOB_VERIFY_ACTIVE, JOB_RELOAD)) {

                HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_AFTER], i) {
                        if (!other->job)
                                continue;

                        if (!GREEDY_REALLOC(list, n_allocated, n+1))
                                return -ENOMEM;
                        list[n++] = other->job;
                }
        }

        HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_BEFORE], i) {
                if (!other->job)
                        continue;

                if (!IN_SET(other->job->type, JOB_STOP, JOB_RESTART))
                        continue;

                if (!GREEDY_REALLOC(list, n_allocated, n+1))
                        return -ENOMEM;
                list[n++] = other->job;
        }

        n = sort_job_list(list, n);

        *ret = TAKE_PTR(list);

        return (int) n;
}

int job_get_after(Job *j, Job*** ret) {
        _cleanup_free_ Job** list = NULL;
        size_t n = 0, n_allocated = 0;
        Unit *other = NULL;
        void *v;
        Iterator i;

        assert(j);
        assert(ret);

        /* Returns a list of all pending jobs that are waiting for this job to finish. */

        HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_BEFORE], i) {
                if (!other->job)
                        continue;

                if (other->job->ignore_order)
                        continue;

                if (!IN_SET(other->job->type, JOB_START, JOB_VERIFY_ACTIVE, JOB_RELOAD))
                        continue;

                if (!GREEDY_REALLOC(list, n_allocated, n+1))
                        return -ENOMEM;
                list[n++] = other->job;
        }

        if (IN_SET(j->type, JOB_STOP, JOB_RESTART)) {

                HASHMAP_FOREACH_KEY(v, other, j->unit->dependencies[UNIT_AFTER], i) {
                        if (!other->job)
                                continue;

                        if (other->job->ignore_order)
                                continue;

                        if (!GREEDY_REALLOC(list, n_allocated, n+1))
                                return -ENOMEM;
                        list[n++] = other->job;
                }
        }

        n = sort_job_list(list, n);

        *ret = TAKE_PTR(list);

        return (int) n;
}

static const char* const job_state_table[_JOB_STATE_MAX] = {
        [JOB_WAITING] = "waiting",
        [JOB_RUNNING] = "running",
};

DEFINE_STRING_TABLE_LOOKUP(job_state, JobState);

static const char* const job_type_table[_JOB_TYPE_MAX] = {
        [JOB_START] = "start",
        [JOB_VERIFY_ACTIVE] = "verify-active",
        [JOB_STOP] = "stop",
        [JOB_RELOAD] = "reload",
        [JOB_RELOAD_OR_START] = "reload-or-start",
        [JOB_RESTART] = "restart",
        [JOB_TRY_RESTART] = "try-restart",
        [JOB_TRY_RELOAD] = "try-reload",
        [JOB_NOP] = "nop",
};

DEFINE_STRING_TABLE_LOOKUP(job_type, JobType);

static const char* const job_mode_table[_JOB_MODE_MAX] = {
        [JOB_FAIL] = "fail",
        [JOB_REPLACE] = "replace",
        [JOB_REPLACE_IRREVERSIBLY] = "replace-irreversibly",
        [JOB_ISOLATE] = "isolate",
        [JOB_FLUSH] = "flush",
        [JOB_IGNORE_DEPENDENCIES] = "ignore-dependencies",
        [JOB_IGNORE_REQUIREMENTS] = "ignore-requirements",
};

DEFINE_STRING_TABLE_LOOKUP(job_mode, JobMode);

static const char* const job_result_table[_JOB_RESULT_MAX] = {
        [JOB_DONE] = "done",
        [JOB_CANCELED] = "canceled",
        [JOB_TIMEOUT] = "timeout",
        [JOB_FAILED] = "failed",
        [JOB_DEPENDENCY] = "dependency",
        [JOB_SKIPPED] = "skipped",
        [JOB_INVALID] = "invalid",
        [JOB_ASSERT] = "assert",
        [JOB_UNSUPPORTED] = "unsupported",
        [JOB_COLLECTED] = "collected",
        [JOB_ONCE] = "once",
};

DEFINE_STRING_TABLE_LOOKUP(job_result, JobResult);

const char* job_type_to_access_method(JobType t) {
        assert(t >= 0);
        assert(t < _JOB_TYPE_MAX);

        if (IN_SET(t, JOB_START, JOB_RESTART, JOB_TRY_RESTART))
                return "start";
        else if (t == JOB_STOP)
                return "stop";
        else
                return "reload";
}