From 71c32c3d48acac7622022c1baee982c0db29b059 Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Fri, 11 Jun 2021 17:20:01 +0200
Subject: [PATCH 1/7] Restart policy `Never` handled correctly

---
 CHANGELOG.adoc                                |   4 +
 docs/modules/ROOT/pages/pod_types/jobs.adoc   |  42 +++++
 .../ROOT/pages/pod_types/services.adoc        |  23 +++
 src/provider/kubernetes/accessor.rs           |  70 +++++++
 src/provider/kubernetes/mod.rs                |   4 +
 src/provider/kubernetes/status.rs             |  32 ++++
 src/provider/mod.rs                           |   3 +-
 src/provider/states.rs                        |  22 ---
 src/provider/states/pod.rs                    |   1 -
 src/provider/states/pod/failed.rs             |  48 -----
 src/provider/states/pod/running.rs            | 172 +++++++++++-------
 src/provider/states/pod/starting.rs           |  29 ++-
 src/provider/states/pod/terminated.rs         |  58 +++---
 src/provider/systemdmanager/manager.rs        |  17 +-
 src/provider/systemdmanager/systemd1_api.rs   |  99 +++++++++-
 15 files changed, 441 insertions(+), 183 deletions(-)
 create mode 100644 docs/modules/ROOT/pages/pod_types/jobs.adoc
 create mode 100644 docs/modules/ROOT/pages/pod_types/services.adoc
 create mode 100644 src/provider/kubernetes/accessor.rs
 create mode 100644 src/provider/kubernetes/mod.rs
 create mode 100644 src/provider/kubernetes/status.rs
 delete mode 100644 src/provider/states/pod/failed.rs

diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
index 1d9e8cc..8bd68fb 100644
--- a/CHANGELOG.adoc
+++ b/CHANGELOG.adoc
@@ -3,11 +3,15 @@
 == 0.4.0 - unreleased
 
 :188: https://github.com/stackabletech/agent/pull/188[#188]
+:204: https://github.com/stackabletech/agent/pull/204[#204]
 
 === Added
 * Annotation `featureLogs` added to the pods to indicate if logs can be
   retrieved with `kubectl logs` ({188}).
 
+=== Fixed
+* Pods with restart policy `Never` handled correctly ({204}).
+
 == 0.3.0 - 2021-05-27
 
 :165: https://github.com/stackabletech/agent/pull/165[#165]
diff --git a/docs/modules/ROOT/pages/pod_types/jobs.adoc b/docs/modules/ROOT/pages/pod_types/jobs.adoc
new file mode 100644
index 0000000..0ed7914
--- /dev/null
+++ b/docs/modules/ROOT/pages/pod_types/jobs.adoc
@@ -0,0 +1,42 @@
+= Jobs
+
+Jobs which terminate after performing a task, must be created with
+`restartPolicy` set to `Never`:
+
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: <pod-name>
+    spec:
+      containers:
+        - name: <job-name>
+          image: <job-image>
+          command:
+            - <job-command>
+      restartPolicy: Never
+
+If the job terminated successfully then the pod phase is set to
+`Succeeded` and the exit code of the container state is `0`:
+
+    status:
+      phase: Succeeded
+      message: Completed
+      reason: Completed
+      containerStatuses:
+        - state:
+            terminated:
+              exitCode: 0
+              message: Completed
+
+If the job failed then the pod phase is set to `Failed` and the exit
+code of the container state is `1`:
+
+    status:
+      phase: Failed
+      message: Error
+      reason: Error
+      containerStatuses:
+        - state:
+            terminated:
+              exitCode: 1
+              message: Error
diff --git a/docs/modules/ROOT/pages/pod_types/services.adoc b/docs/modules/ROOT/pages/pod_types/services.adoc
new file mode 100644
index 0000000..1887ae5
--- /dev/null
+++ b/docs/modules/ROOT/pages/pod_types/services.adoc
@@ -0,0 +1,23 @@
+= Services
+
+Services do not terminate on their own. They run until the pod is
+deleted. The `restartPolicy` must be set to `Always` or `OnFailure`. As
+services cannot terminate successfully, there is no difference between
+these two policies. `restartPolicy` defaults to `Always` and can
+therefore be omitted.
+
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: <pod-name>
+    spec:
+      containers:
+        - name: <service-name>
+          image: <service-image>
+          command:
+            - <service-command>
+      restartPolicy: Always
+
+After a container command is executed the agent waits for 10 seconds
+before the container status is set to running. When all containers are
+running, also the pod phase is switched from `Pending` to `Running`.
diff --git a/src/provider/kubernetes/accessor.rs b/src/provider/kubernetes/accessor.rs
new file mode 100644
index 0000000..4df28a7
--- /dev/null
+++ b/src/provider/kubernetes/accessor.rs
@@ -0,0 +1,70 @@
+//! Accessor methods for Kubernetes resources
+
+use std::str::FromStr;
+
+use kubelet::pod::Pod;
+use strum::{Display, EnumString, EnumVariantNames};
+
+/// Restart policy for all containers within the pod.
+#[derive(Clone, Debug, Display, EnumString, EnumVariantNames, Eq, PartialEq)]
+pub enum RestartPolicy {
+    Always,
+    OnFailure,
+    Never,
+}
+
+impl Default for RestartPolicy {
+    fn default() -> Self {
+        RestartPolicy::Always
+    }
+}
+
+/// Returns the restart policy for all containers within the pod.
+pub fn restart_policy(pod: &Pod) -> RestartPolicy {
+    pod.as_kube_pod()
+        .spec
+        .as_ref()
+        .and_then(|spec| spec.restart_policy.as_ref())
+        .and_then(|restart_policy| RestartPolicy::from_str(restart_policy).ok())
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::provider::test::TestPod;
+    use rstest::rstest;
+
+    #[rstest]
+    #[case::restart_policy_onfailure(
+        "
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              name: test
+            spec:
+              containers:
+                - name: test-container
+              restartPolicy: OnFailure
+        ",
+        RestartPolicy::OnFailure
+    )]
+    #[case::restart_policy_default(
+        "
+            apiVersion: v1
+            kind: Pod
+            metadata:
+              name: test
+            spec:
+              containers:
+                - name: test-container
+        ",
+        RestartPolicy::Always
+    )]
+    fn should_return_specified_restart_policy_or_default(
+        #[case] pod: TestPod,
+        #[case] expected_restart_policy: RestartPolicy,
+    ) {
+        assert_eq!(expected_restart_policy, restart_policy(&pod));
+    }
+}
diff --git a/src/provider/kubernetes/mod.rs b/src/provider/kubernetes/mod.rs
new file mode 100644
index 0000000..439aa47
--- /dev/null
+++ b/src/provider/kubernetes/mod.rs
@@ -0,0 +1,4 @@
+//! Utility functions for Kubernetes
+
+pub mod accessor;
+pub mod status;
diff --git a/src/provider/kubernetes/status.rs b/src/provider/kubernetes/status.rs
new file mode 100644
index 0000000..12b2956
--- /dev/null
+++ b/src/provider/kubernetes/status.rs
@@ -0,0 +1,32 @@
+//! Functions for patching the pod status
+
+use k8s_openapi::api::core::v1::Pod as KubePod;
+use kube::{Api, Client};
+use kubelet::{
+    container::{ContainerKey, Status},
+    pod::Pod,
+};
+use log::warn;
+
+/// Patches the pod status with the given container status.
+///
+/// If the patching fails then a warning is logged.
+pub async fn patch_container_status(
+    client: &Client,
+    pod: &Pod,
+    container_key: &ContainerKey,
+    status: &Status,
+) {
+    let api: Api<KubePod> = Api::namespaced(client.clone(), pod.namespace());
+
+    if let Err(error) =
+        kubelet::container::patch_container_status(&api, &pod, container_key, &status).await
+    {
+        warn!(
+            "Status of container [{}] in pod [{}] could not be patched. {}",
+            container_key,
+            pod.name(),
+            error
+        );
+    }
+}
diff --git a/src/provider/mod.rs b/src/provider/mod.rs
index 0f01cb3..7b57d24 100644
--- a/src/provider/mod.rs
+++ b/src/provider/mod.rs
@@ -42,6 +42,7 @@ pub struct StackableProvider {
 pub const CRDS: &[&str] = &["repositories.stable.stackable.de"];
 
 mod error;
+pub mod kubernetes;
 mod repository;
 mod states;
 pub mod systemdmanager;
@@ -366,7 +367,7 @@ impl Provider for StackableProvider {
 }
 
 #[cfg(test)]
-mod test {
+pub mod test {
     use super::*;
     use rstest::rstest;
     use std::ops::Deref;
diff --git a/src/provider/states.rs b/src/provider/states.rs
index 412520a..321acf1 100644
--- a/src/provider/states.rs
+++ b/src/provider/states.rs
@@ -1,8 +1,3 @@
-use k8s_openapi::api::core::v1::ContainerStatus as KubeContainerStatus;
-use k8s_openapi::api::core::v1::PodCondition as KubePodCondition;
-use kubelet::pod::state::prelude::*;
-use kubelet::pod::{Phase, Status};
-
 pub(crate) mod pod;
 
 /// When called in a state's `next` function, exits the state machine
@@ -15,20 +10,3 @@ macro_rules! fail_fatal {
         return Transition::Complete(Err(aerr));
     }};
 }
-
-/// Create basic Pod status patch with container status and pod conditions
-pub fn make_status_with_containers_and_condition(
-    phase: Phase,
-    reason: &str,
-    container_statuses: Vec<KubeContainerStatus>,
-    init_container_statuses: Vec<KubeContainerStatus>,
-    conditions: Vec<KubePodCondition>,
-) -> Status {
-    StatusBuilder::new()
-        .phase(phase)
-        .reason(reason)
-        .container_statuses(container_statuses)
-        .init_container_statuses(init_container_statuses)
-        .conditions(conditions)
-        .build()
-}
diff --git a/src/provider/states/pod.rs b/src/provider/states/pod.rs
index d581447..2897008 100644
--- a/src/provider/states/pod.rs
+++ b/src/provider/states/pod.rs
@@ -11,7 +11,6 @@ pub(crate) mod creating_config;
 pub(crate) mod creating_service;
 pub(crate) mod downloading;
 pub(crate) mod downloading_backoff;
-pub(crate) mod failed;
 pub(crate) mod installing;
 pub(crate) mod running;
 pub(crate) mod setup_failed;
diff --git a/src/provider/states/pod/failed.rs b/src/provider/states/pod/failed.rs
deleted file mode 100644
index af1e41d..0000000
--- a/src/provider/states/pod/failed.rs
+++ /dev/null
@@ -1,48 +0,0 @@
-use kubelet::pod::state::prelude::*;
-use log::{debug, info};
-
-use super::installing::Installing;
-use super::starting::Starting;
-use crate::provider::{PodState, ProviderState};
-
-#[derive(Default, Debug, TransitionTo)]
-#[transition_to(Starting, Installing)]
-pub struct Failed {
-    pub message: String,
-}
-
-impl Failed {
-    fn restart_enabled(&self, pod: &Pod) -> bool {
-        if let Some(pod_spec) = &pod.as_kube_pod().spec {
-            if let Some(restart_policy) = &pod_spec.restart_policy {
-                return restart_policy.eq("Always");
-            }
-        }
-        false
-    }
-}
-
-#[async_trait::async_trait]
-impl State<PodState> for Failed {
-    async fn next(
-        self: Box<Self>,
-        _provider_state: SharedState<ProviderState>,
-        _pod_state: &mut PodState,
-        pod: Manifest<Pod>,
-    ) -> Transition<PodState> {
-        let pod = pod.latest();
-
-        info!("Process entered failed state");
-        if self.restart_enabled(&pod) {
-            debug!("Restart policy is set to restart, starting...");
-            return Transition::next(self, Starting {});
-        } else {
-            debug!("Restart is disabled for process.");
-        }
-        Transition::Complete(Ok(()))
-    }
-
-    async fn status(&self, _pod_state: &mut PodState, _pod: &Pod) -> anyhow::Result<PodStatus> {
-        Ok(make_status(Phase::Failed, &self.message))
-    }
-}
diff --git a/src/provider/states/pod/running.rs b/src/provider/states/pod/running.rs
index 8edb813..e64d43d 100644
--- a/src/provider/states/pod/running.rs
+++ b/src/provider/states/pod/running.rs
@@ -1,22 +1,27 @@
 use anyhow::anyhow;
-use k8s_openapi::api::core::v1::{
-    ContainerState, ContainerStateRunning, ContainerStatus as KubeContainerStatus, PodCondition,
-};
-use krator::ObjectStatus;
-use kubelet::pod::state::prelude::*;
-use kubelet::pod::{Pod, PodKey};
-use log::{debug, info, trace};
-
-use super::failed::Failed;
-use super::installing::Installing;
-use crate::provider::states::make_status_with_containers_and_condition;
-use crate::provider::{PodHandle, PodState, ProviderState};
+use k8s_openapi::api::core::v1::PodCondition;
 use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time;
 use k8s_openapi::chrono;
+use krator::ObjectStatus;
+use kubelet::{
+    container::Status,
+    pod::state::prelude::*,
+    pod::{Pod, PodKey},
+};
+use log::{debug, info, trace, warn};
 use tokio::time::Duration;
 
+use super::{installing::Installing, starting::Starting, terminated::Terminated};
+use crate::provider::{
+    kubernetes::{
+        accessor::{restart_policy, RestartPolicy},
+        status::patch_container_status,
+    },
+    PodHandle, PodState, ProviderState,
+};
+
 #[derive(Debug, TransitionTo)]
-#[transition_to(Failed, Running, Installing)]
+#[transition_to(Installing, Starting, Terminated)]
 pub struct Running {
     pub transition_time: Time,
 }
@@ -38,83 +43,118 @@ impl State<PodState> for Running {
         pod: Manifest<Pod>,
     ) -> Transition<PodState> {
         let pod = pod.latest();
-        let pod_key = &PodKey::from(pod);
+        let pod_key = &PodKey::from(&pod);
 
-        let (systemd_manager, pod_handle) = {
+        let (client, systemd_manager, pod_handle) = {
             let provider_state = shared.read().await;
             let handles = provider_state.handles.read().await;
             (
+                provider_state.client.clone(),
                 provider_state.systemd_manager.clone(),
                 handles.get(&pod_key).map(PodHandle::to_owned),
             )
         };
 
-        // We loop here indefinitely and "wake up" periodically to check if the service is still
+        let mut running_containers = match &pod_handle {
+                Some(containers) => containers.to_owned(),
+                None => return Transition::Complete(Err(anyhow!("No systemd units found for service [{}], this should not happen, please report a bug for this!", pod_state.service_name))),
+            };
+
+        let mut container_failed = false;
+
+        // We loop here and "wake up" periodically to check if the service is still
         // up and running
         // Interruption of this loop is triggered externally by the Krustlet code when
         //   - the pod which this state machine refers to gets deleted
         //   - Krustlet shuts down
-        loop {
+        while !running_containers.is_empty() && !container_failed {
             tokio::time::sleep(Duration::from_secs(10)).await;
             trace!(
                 "Checking if service {} is still running.",
                 &pod_state.service_name
             );
 
-            // Iterate over all units and check their state
-            // if the [`service_units`] Option is a None variant, return a failed state
-            // as we need to run something otherwise we are not doing anything
-            let containers = match &pod_handle {
-                Some(containers) => containers,
-                None => return Transition::Complete(Err(anyhow!("No systemd units found for service [{}], this should not happen, please report a bug for this!", pod_state.service_name))),
-            };
+            let mut succeeded_containers = Vec::new();
+            let mut failed_containers = Vec::new();
 
-            for container_handle in containers.values() {
+            for (container_key, container_handle) in running_containers.iter() {
                 let service_unit = &container_handle.service_unit;
 
                 match systemd_manager.is_running(&service_unit).await {
-                    Ok(true) => trace!(
-                        "Unit [{}] of service [{}] still running ...",
-                        service_unit,
-                        pod_state.service_name
-                    ),
-                    Ok(false) => {
-                        info!("Unit [{}] for service [{}] failed unexpectedly, transitioning to failed state.", pod_state.service_name, service_unit);
-                        return Transition::next(
-                            self,
-                            Failed {
-                                message: "".to_string(),
-                            },
-                        );
-                    }
+                    Ok(true) => {}
+                    Ok(false) => match systemd_manager.failed(&service_unit).await {
+                        Ok(true) => failed_containers
+                            .push((container_key.to_owned(), container_handle.to_owned())),
+                        Ok(false) => succeeded_containers
+                            .push((container_key.to_owned(), container_handle.to_owned())),
+                        Err(dbus_error) => warn!(
+                            "Error querying Failed property for Unit [{}] of service [{}]: [{}]",
+                            service_unit, pod_state.service_name, dbus_error
+                        ),
+                    },
                     Err(dbus_error) => {
-                        info!(
+                        warn!(
                             "Error querying ActiveState for Unit [{}] of service [{}]: [{}].",
-                            pod_state.service_name, service_unit, dbus_error
+                            service_unit, pod_state.service_name, dbus_error
                         );
-                        return Transition::Complete(Err(dbus_error));
                     }
                 }
             }
+
+            for (container_key, container_handle) in &succeeded_containers {
+                info!(
+                    "Unit [{}] for service [{}] terminated successfully.",
+                    pod_state.service_name, container_handle.service_unit
+                );
+                patch_container_status(
+                    &client,
+                    &pod,
+                    container_key,
+                    &Status::terminated("Completed", false),
+                )
+                .await;
+                running_containers.remove(container_key);
+            }
+
+            for (container_key, container_handle) in &failed_containers {
+                info!(
+                    "Unit [{}] for service [{}] failed unexpectedly.",
+                    pod_state.service_name, container_handle.service_unit
+                );
+                patch_container_status(
+                    &client,
+                    &pod,
+                    container_key,
+                    &Status::terminated("Error", true),
+                )
+                .await;
+                running_containers.remove(container_key);
+            }
+
+            for container_handle in running_containers.values() {
+                trace!(
+                    "Unit [{}] of service [{}] still running ...",
+                    container_handle.service_unit,
+                    pod_state.service_name
+                );
+            }
+
+            container_failed = !failed_containers.is_empty();
         }
-    }
 
-    // test
-    async fn status(&self, pod_state: &mut PodState, pod: &Pod) -> anyhow::Result<PodStatus> {
-        let state = ContainerState {
-            running: Some(ContainerStateRunning { started_at: None }),
-            ..Default::default()
-        };
+        if container_failed {
+            if restart_policy(&pod) == RestartPolicy::Never {
+                Transition::next(self, Terminated { successful: false })
+            } else {
+                debug!("Restart policy is set to restart, starting...");
+                Transition::next(self, Starting {})
+            }
+        } else {
+            Transition::next(self, Terminated { successful: true })
+        }
+    }
 
-        let container = &pod.containers()[0];
-        // TODO: Change to support multiple containers
-        let container_status = vec![KubeContainerStatus {
-            name: container.name().to_string(),
-            ready: true,
-            started: Some(false),
-            state: Some(state),
-            ..Default::default()
-        }];
+    async fn status(&self, pod_state: &mut PodState, _pod: &Pod) -> anyhow::Result<PodStatus> {
         let condition = PodCondition {
             last_probe_time: None,
             last_transition_time: Some(self.transition_time.clone()),
@@ -123,15 +163,15 @@ impl State<PodState> for Running {
             status: "True".to_string(),
             type_: "Ready".to_string(),
         };
-        let status = make_status_with_containers_and_condition(
-            Phase::Running,
-            "Running",
-            container_status,
-            vec![],
-            vec![condition],
-        );
+
+        let status = StatusBuilder::new()
+            .phase(Phase::Running)
+            .reason("Running")
+            .conditions(vec![condition])
+            .build();
+
         debug!(
-            "Patching status for running servce [{}] with: [{}]",
+            "Patching status for running service [{}] with: [{}]",
             pod_state.service_name,
             status.json_patch()
         );
diff --git a/src/provider/states/pod/starting.rs b/src/provider/states/pod/starting.rs
index 1d4f40c..cf22e61 100644
--- a/src/provider/states/pod/starting.rs
+++ b/src/provider/states/pod/starting.rs
@@ -1,6 +1,11 @@
 use super::running::Running;
 use crate::provider::{
-    systemdmanager::manager::SystemdManager, PodHandle, PodState, ProviderState,
+    kubernetes::{
+        accessor::{restart_policy, RestartPolicy},
+        status::patch_container_status,
+    },
+    systemdmanager::manager::SystemdManager,
+    PodHandle, PodState, ProviderState,
 };
 
 use anyhow::{anyhow, Result};
@@ -8,11 +13,11 @@ use kube::{
     api::{Patch, PatchParams},
     Api, Client,
 };
-use kubelet::pod::state::prelude::*;
 use kubelet::{
     container::ContainerKey,
     pod::{Pod, PodKey},
 };
+use kubelet::{container::Status, pod::state::prelude::*};
 use log::{debug, error, info};
 use serde_json::json;
 use std::time::Instant;
@@ -83,14 +88,16 @@ async fn start_service_units(
             info!("Enabling systemd unit [{}]", service_unit);
             systemd_manager.enable(service_unit).await?;
 
-            // TODO: does this need to be configurable, or ar we happy with a hard coded value
-            //  for now. I've briefly looked at the podspec and couldn't identify a good field
-            //  to use for this - also, currently this starts containers (= systemd units) in
-            //  order and waits 10 seconds for every unit, so a service with five containers
-            //  would take 50 seconds until it reported running - which is totally fine in case
-            //  the units actually depend on each other, but a case could be made for waiting
-            //  once at the end
-            await_startup(&systemd_manager, service_unit, Duration::from_secs(10)).await?;
+            if restart_policy(pod) != RestartPolicy::Never {
+                // TODO: does this need to be configurable, or ar we happy with a hard coded value
+                //  for now. I've briefly looked at the podspec and couldn't identify a good field
+                //  to use for this - also, currently this starts containers (= systemd units) in
+                //  order and waits 10 seconds for every unit, so a service with five containers
+                //  would take 50 seconds until it reported running - which is totally fine in case
+                //  the units actually depend on each other, but a case could be made for waiting
+                //  once at the end
+                await_startup(&systemd_manager, service_unit, Duration::from_secs(10)).await?;
+            }
         }
 
         let maybe_invocation_id = systemd_manager.get_invocation_id(service_unit).await.ok();
@@ -104,6 +111,8 @@ async fn start_service_units(
             &maybe_invocation_id.is_some().to_string(),
         )
         .await?;
+
+        patch_container_status(&client, &pod, &container_key, &Status::running()).await;
     }
 
     Ok(())
diff --git a/src/provider/states/pod/terminated.rs b/src/provider/states/pod/terminated.rs
index df5cb9c..510f5ab 100644
--- a/src/provider/states/pod/terminated.rs
+++ b/src/provider/states/pod/terminated.rs
@@ -1,12 +1,12 @@
 use kubelet::pod::{state::prelude::*, PodKey};
-use log::{error, info, warn};
+use log::{debug, info, warn};
 
 use crate::provider::{PodState, ProviderState};
 
 #[derive(Default, Debug)]
 /// The pod object was deleted in Kubernetes
 pub struct Terminated {
-    pub message: String,
+    pub successful: bool,
 }
 
 #[async_trait::async_trait]
@@ -17,10 +17,7 @@ impl State<PodState> for Terminated {
         pod_state: &mut PodState,
         pod: Manifest<Pod>,
     ) -> Transition<PodState> {
-        info!(
-            "Pod {} was terminated, stopping service!",
-            &pod_state.service_name
-        );
+        info!("Pod {} was terminated", &pod_state.service_name);
 
         let pod = pod.latest();
         let pod_key = &PodKey::from(pod);
@@ -40,9 +37,9 @@ impl State<PodState> for Terminated {
             for container_handle in containers.values() {
                 let service_unit = &container_handle.service_unit;
 
-                info!("Stopping systemd unit [{}]", service_unit);
+                debug!("Stopping systemd unit [{}]", service_unit);
                 if let Err(stop_error) = systemd_manager.stop(service_unit).await {
-                    error!(
+                    warn!(
                         "Error occurred stopping systemd unit [{}]: [{}]",
                         service_unit, stop_error
                     );
@@ -50,33 +47,46 @@ impl State<PodState> for Terminated {
                 }
 
                 // Daemon reload is false here, we'll do that once after all units have been removed
-                info!("Removing systemd unit [{}]", service_unit);
+                debug!("Removing systemd unit [{}]", service_unit);
                 if let Err(remove_error) = systemd_manager.remove_unit(service_unit, false).await {
-                    error!(
+                    warn!(
                         "Error occurred removing systemd unit [{}]: [{}]",
                         service_unit, remove_error
                     );
                     return Transition::Complete(Err(remove_error));
                 }
             }
+
+            debug!("Performing daemon-reload");
+            if let Err(reload_error) = systemd_manager.reload().await {
+                warn!("Failed to perform daemon-reload: [{}]", reload_error);
+                return Transition::Complete(Err(reload_error));
+            };
         } else {
-            warn!(
-                "No unit definitions found, not stopping anything for pod [{}]!",
-                pod_state.service_name
-            );
+            debug!("Pod [{}] was already terminated", pod_state.service_name);
         }
 
-        info!("Performing daemon-reload");
-        return match systemd_manager.reload().await {
-            Ok(()) => Transition::Complete(Ok(())),
-            Err(reload_error) => {
-                error!("Failed to perform daemon-reload: [{}]", reload_error);
-                Transition::Complete(Err(reload_error))
-            }
-        };
+        Transition::Complete(Ok(()))
     }
 
-    async fn status(&self, _pod_state: &mut PodState, _pod: &Pod) -> anyhow::Result<PodStatus> {
-        Ok(make_status(Phase::Succeeded, &self.message))
+    async fn status(&self, _pod_state: &mut PodState, pod: &Pod) -> anyhow::Result<PodStatus> {
+        let phase = pod
+            .as_kube_pod()
+            .status
+            .as_ref()
+            .and_then(|status| status.phase.as_ref())
+            .map(String::as_ref);
+
+        let already_terminated = phase == Some("Succeeded") || phase == Some("Failed");
+
+        let status = if already_terminated {
+            Default::default() // no changes to the current status
+        } else if self.successful {
+            make_status(Phase::Succeeded, "Completed")
+        } else {
+            make_status(Phase::Failed, "Error")
+        };
+
+        Ok(status)
     }
 }
diff --git a/src/provider/systemdmanager/manager.rs b/src/provider/systemdmanager/manager.rs
index 7d1a312..09b0c07 100644
--- a/src/provider/systemdmanager/manager.rs
+++ b/src/provider/systemdmanager/manager.rs
@@ -4,10 +4,10 @@
 //! disable systemd units.
 //!
 use super::systemd1_api::{
-    ActiveState, AsyncJobProxy, AsyncManagerProxy, JobRemovedResult, JobRemovedSignal,
-    ManagerSignals, StartMode, StopMode,
+    ActiveState, AsyncJobProxy, AsyncManagerProxy, AsyncServiceProxy, JobRemovedResult,
+    JobRemovedSignal, ManagerSignals, StartMode, StopMode,
 };
-use crate::provider::systemdmanager::systemdunit::SystemDUnit;
+use crate::provider::systemdmanager::{systemd1_api::ServiceResult, systemdunit::SystemDUnit};
 use crate::provider::StackableError;
 use crate::provider::StackableError::RuntimeError;
 use anyhow::anyhow;
@@ -388,6 +388,17 @@ impl SystemdManager {
             .map_err(|e| anyhow!("Error receiving ActiveState of unit [{}]. {}", unit, e))
     }
 
+    /// Checks if the result of the given service unit is not set to success.
+    pub async fn failed(&self, unit: &str) -> anyhow::Result<bool> {
+        let unit_proxy = self.proxy.load_unit(unit).await?;
+        let service_proxy = AsyncServiceProxy::from(unit_proxy);
+        service_proxy
+            .result()
+            .await
+            .map(|state| state != ServiceResult::Success)
+            .map_err(|e| anyhow!("Error receiving Result of unit [{}]. {}", unit, e))
+    }
+
     /// Retrieves the invocation ID for the given unit.
     ///
     /// The invocation ID was introduced in systemd version 232.
diff --git a/src/provider/systemdmanager/systemd1_api.rs b/src/provider/systemdmanager/systemd1_api.rs
index 06c0e24..857cd88 100644
--- a/src/provider/systemdmanager/systemd1_api.rs
+++ b/src/provider/systemdmanager/systemd1_api.rs
@@ -79,6 +79,19 @@ macro_rules! impl_type_for_enum {
     };
 }
 
+macro_rules! impl_tryfrom_ownedvalue_for_enum {
+    ($t:ty) => {
+        impl TryFrom<OwnedValue> for $t {
+            type Error = zvariant::Error;
+
+            fn try_from(value: OwnedValue) -> Result<Self, Self::Error> {
+                FromStr::from_str(&String::try_from(value)?)
+                    .map_err(|e: strum::ParseError| Self::Error::Message(e.to_string()))
+            }
+        }
+    };
+}
+
 /// Type of an entry in a changes list
 #[derive(Clone, Debug, Display, EnumString, EnumVariantNames, Eq, PartialEq)]
 #[strum(serialize_all = "kebab-case")]
@@ -348,14 +361,7 @@ pub enum ActiveState {
     Deactivating,
 }
 
-impl TryFrom<OwnedValue> for ActiveState {
-    type Error = zvariant::Error;
-
-    fn try_from(value: OwnedValue) -> Result<Self, Self::Error> {
-        FromStr::from_str(&String::try_from(value)?)
-            .map_err(|e: strum::ParseError| Self::Error::Message(e.to_string()))
-    }
-}
+impl_tryfrom_ownedvalue_for_enum!(ActiveState);
 
 /// Unique ID for a runtime cycle of a unit
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -398,6 +404,83 @@ trait Unit {
     fn invocation_id(&self) -> zbus::Result<InvocationId>;
 }
 
+/// Execution result of the last run of the service.
+///
+/// It is useful to determine the reason a service failed if it is in
+/// the "failed" state (see ['ActiveState::Failed`]).
+#[derive(Clone, Debug, Display, EnumString, Eq, PartialEq)]
+#[strum(serialize_all = "kebab-case")]
+pub enum ServiceResult {
+    /// Is set if the unit didn't fail.
+    Success,
+
+    /// Indicates that not enough resources were available to fork off
+    /// and execute the service processes.
+    Resources,
+
+    /// Indicates that a timeout occurred while executing a service
+    /// operation.
+    Timeout,
+
+    /// Indicates that a service process exited with an unclean exit
+    /// code.
+    ExitCode,
+
+    /// Indicates that a service process exited with an uncaught signal.
+    Signal,
+
+    /// Indicates that a service process exited uncleanly and dumped
+    /// core.
+    CoreDump,
+
+    /// Indicates that a service did not send out watchdog ping messages
+    /// often enough.
+    Watchdog,
+
+    /// Indicates that a service has been started too frequently in a
+    /// specific time frame (as configured in `StartLimitInterval`,
+    /// `StartLimitBurst`).
+    StartLimit,
+}
+
+impl_tryfrom_ownedvalue_for_enum!(ServiceResult);
+
+/// A systemd service unit object
+///
+/// A [`ServiceUnitProxy`] can be created from a [`UnitProxy`] with
+/// [`ServiceProxy::from`].
+///
+/// Currently not all methods of the systemd object are exposed.
+#[dbus_proxy(
+    default_service = "org.freedesktop.systemd1",
+    interface = "org.freedesktop.systemd1.Service"
+)]
+trait Service {
+    /// Execution result of the last run of the service. It is useful to
+    /// determine the reason a service failed if it is in the "failed"
+    /// state (see ['ActiveState::Failed`]).
+    #[dbus_proxy(property)]
+    fn result(&self) -> zbus::Result<ServiceResult>;
+}
+
+impl<'c> From<UnitProxy<'c>> for ServiceProxy<'c> {
+    fn from(unit_proxy: UnitProxy<'c>) -> Self {
+        ServiceProxy::builder(unit_proxy.connection())
+            .path(unit_proxy.path().to_owned())
+            .unwrap()
+            .build()
+    }
+}
+
+impl<'c> From<AsyncUnitProxy<'c>> for AsyncServiceProxy<'c> {
+    fn from(unit_proxy: AsyncUnitProxy<'c>) -> Self {
+        AsyncServiceProxy::builder(unit_proxy.connection())
+            .path(unit_proxy.path().to_owned())
+            .unwrap()
+            .build()
+    }
+}
+
 /// A systemd job object
 ///
 /// The [`JobProxy`] is returned by various functions in [`ManagerProxy`].

From a6f97e822cdb5c1ff1d9a4557c9608a209ee8320 Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Tue, 22 Jun 2021 10:35:49 +0200
Subject: [PATCH 2/7] systemd version set to 0.9

---
 Cargo.lock | 211 +++++++++++++++++++++++++----------------------------
 Cargo.toml |   2 +-
 2 files changed, 100 insertions(+), 113 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 409624f..a270c56 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -48,9 +48,9 @@ checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271"
 
 [[package]]
 name = "async-broadcast"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8501c1102158e8cf7a73cab03102be0d2651b0f574a33a749ebc3aec72fcde9d"
+checksum = "8c53006e3a562799d414fad784fdb8789dcf94f78f9e66e642347e50492852f7"
 dependencies = [
  "easy-parallel",
  "event-listener",
@@ -112,9 +112,9 @@ dependencies = [
 
 [[package]]
 name = "async-stream"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e"
+checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
 dependencies = [
  "async-stream-impl",
  "futures-core",
@@ -122,9 +122,9 @@ dependencies = [
 
 [[package]]
 name = "async-stream-impl"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340"
+checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -251,9 +251,9 @@ checksum = "1522ac6ee801a11bf9ef3f80403f4ede6eb41291fac3dde3de09989679305f25"
 
 [[package]]
 name = "bumpalo"
-version = "3.6.1"
+version = "3.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
+checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631"
 
 [[package]]
 name = "byte-tools"
@@ -297,9 +297,9 @@ checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba"
 
 [[package]]
 name = "cc"
-version = "1.0.67"
+version = "1.0.68"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd"
+checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787"
 
 [[package]]
 name = "cfg-if"
@@ -376,9 +376,9 @@ checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
 
 [[package]]
 name = "cpufeatures"
-version = "0.1.4"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed00c67cb5d0a7d64a44f6ad2668db7e7530311dd53ea79bcd4fb022c64911c8"
+checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef"
 dependencies = [
  "libc",
 ]
@@ -404,11 +404,10 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278"
+checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db"
 dependencies = [
- "autocfg",
  "cfg-if 1.0.0",
  "lazy_static",
 ]
@@ -704,21 +703,11 @@ dependencies = [
  "percent-encoding 2.1.0",
 ]
 
-[[package]]
-name = "fsevent"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97f347202c95c98805c216f9e1df210e8ebaec9fdb2365700a43c10797a35e63"
-dependencies = [
- "bitflags",
- "fsevent-sys",
-]
-
 [[package]]
 name = "fsevent-sys"
-version = "3.1.0"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca6f5e6817058771c10f0eb0f05ddf1e35844266f972004fe8e4b21fda295bd5"
+checksum = "5c0e564d24da983c053beff1bb7178e237501206840a3e6bf4e267b9e8ae734a"
 dependencies = [
  "libc",
 ]
@@ -795,9 +784,9 @@ checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1"
 
 [[package]]
 name = "futures-lite"
-version = "1.11.3"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4481d0cd0de1d204a4fa55e7d45f07b1d958abcb06714b3446438e2eff695fb"
+checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48"
 dependencies = [
  "fastrand",
  "futures-core",
@@ -961,9 +950,9 @@ dependencies = [
 
 [[package]]
 name = "heck"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
 dependencies = [
  "unicode-segmentation",
 ]
@@ -1035,9 +1024,9 @@ checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
 
 [[package]]
 name = "httpdate"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9"
+checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
 
 [[package]]
 name = "humantime"
@@ -1056,9 +1045,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
 [[package]]
 name = "hyper"
-version = "0.14.7"
+version = "0.14.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54"
+checksum = "07d6baa1b441335f3ce5098ac421fb6547c46dda735ca1bc6d0153c838f9dd83"
 dependencies = [
  "bytes 1.0.1",
  "futures-channel",
@@ -1070,7 +1059,7 @@ dependencies = [
  "httparse",
  "httpdate",
  "itoa",
- "pin-project 1.0.7",
+ "pin-project-lite 0.2.6",
  "socket2",
  "tokio 1.7.1",
  "tower-service",
@@ -1199,9 +1188,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
+checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
 
 [[package]]
 name = "itertools"
@@ -1348,7 +1337,7 @@ dependencies = [
  "serde_yaml",
  "static_assertions",
  "thiserror",
- "time 0.2.26",
+ "time 0.2.27",
  "tokio 1.7.1",
  "url 2.2.2",
 ]
@@ -1459,14 +1448,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.95"
+version = "0.2.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36"
+checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6"
 
 [[package]]
 name = "libsystemd-sys"
-version = "0.8.0"
-source = "git+https://github.com/stackabletech/rust-systemd.git?branch=stackable_patches_v0.8.2#c0361c69362ebf9c073493bc2e4e82d0bd5552b3"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c68c9b25d61a54754b491e1cb848e644ff90b6c11f6f11f9d2c170b7d2cd83b3"
 dependencies = [
  "build-env",
  "libc",
@@ -1568,9 +1558,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.7.11"
+version = "0.7.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956"
+checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16"
 dependencies = [
  "libc",
  "log",
@@ -1691,18 +1681,17 @@ dependencies = [
 
 [[package]]
 name = "notify"
-version = "5.0.0-pre.8"
+version = "5.0.0-pre.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46bbbcd078f1f00ddb7a9abe70b96e91229b44b0b3afdec610f8e5137f8f014b"
+checksum = "51f18203a26893ca1d3526cf58084025d5639f91c44f8b70ab3b724f60e819a0"
 dependencies = [
  "bitflags",
  "crossbeam-channel",
  "filetime",
- "fsevent",
  "fsevent-sys",
  "inotify",
  "libc",
- "mio 0.7.11",
+ "mio 0.7.13",
  "walkdir",
  "winapi 0.3.9",
 ]
@@ -1766,9 +1755,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.7.2"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
+checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56"
 
 [[package]]
 name = "opaque-debug"
@@ -1784,9 +1773,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.34"
+version = "0.10.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8"
+checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
 dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
@@ -1804,9 +1793,9 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.63"
+version = "0.9.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98"
+checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
 dependencies = [
  "autocfg",
  "cc",
@@ -1817,9 +1806,9 @@ dependencies = [
 
 [[package]]
 name = "ordered-float"
-version = "2.2.0"
+version = "2.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b50b8919aecb97e5ee9aceef27e24f39c46b11831130f4a6b7b091ec5de0de12"
+checksum = "f100fcfb41e5385e0991f74981732049f9b896821542a219420491046baafdc2"
 dependencies = [
  "num-traits",
 ]
@@ -1972,14 +1961,14 @@ checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
 
 [[package]]
 name = "polling"
-version = "2.0.3"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fc12d774e799ee9ebae13f4076ca003b40d18a11ac0f3641e6f899618580b7b"
+checksum = "92341d779fa34ea8437ef4d82d440d5e1ce3f3ff7f824aa64424cd481f9a1f25"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
  "log",
- "wepoll-sys",
+ "wepoll-ffi",
  "winapi 0.3.9",
 ]
 
@@ -2143,14 +2132,14 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
 dependencies = [
  "libc",
- "rand_chacha 0.3.0",
- "rand_core 0.6.2",
- "rand_hc 0.3.0",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+ "rand_hc 0.3.1",
 ]
 
 [[package]]
@@ -2165,12 +2154,12 @@ dependencies = [
 
 [[package]]
 name = "rand_chacha"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
 dependencies = [
  "ppv-lite86",
- "rand_core 0.6.2",
+ "rand_core 0.6.3",
 ]
 
 [[package]]
@@ -2199,9 +2188,9 @@ dependencies = [
 
 [[package]]
 name = "rand_core"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
 dependencies = [
  "getrandom 0.2.3",
 ]
@@ -2217,11 +2206,11 @@ dependencies = [
 
 [[package]]
 name = "rand_hc"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73"
+checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
 dependencies = [
- "rand_core 0.6.2",
+ "rand_core 0.6.3",
 ]
 
 [[package]]
@@ -2247,9 +2236,9 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
+checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
 dependencies = [
  "bitflags",
 ]
@@ -2277,12 +2266,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4"
-dependencies = [
- "byteorder",
-]
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
 
 [[package]]
 name = "regex-syntax"
@@ -2301,9 +2287,9 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.3"
+version = "0.11.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124"
+checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
 dependencies = [
  "async-compression",
  "base64 0.13.0",
@@ -2468,9 +2454,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.2.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84"
+checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
 dependencies = [
  "bitflags",
  "core-foundation",
@@ -2481,9 +2467,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.2.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339"
+checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -2665,9 +2651,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook-registry"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6"
+checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0"
 dependencies = [
  "libc",
 ]
@@ -2777,8 +2763,8 @@ dependencies = [
 
 [[package]]
 name = "stackable_config"
-version = "0.1.0"
-source = "git+https://github.com/stackabletech/common.git?branch=main#548ff2ef3e04ead147b9db8adbde6377e56f6d61"
+version = "0.1.0-nightly"
+source = "git+https://github.com/stackabletech/common.git?branch=main#127b7d2cc349f59a5751354c5109a118ad74e0e5"
 dependencies = [
  "anyhow",
  "bstr",
@@ -2902,9 +2888,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.72"
+version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
+checksum = "f71489ff30030d2ae598524f61326b902466f72a0fb1a8564c001cc63425bcc7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2913,8 +2899,9 @@ dependencies = [
 
 [[package]]
 name = "systemd"
-version = "0.8.2"
-source = "git+https://github.com/stackabletech/rust-systemd.git?branch=stackable_patches_v0.8.2#c0361c69362ebf9c073493bc2e4e82d0bd5552b3"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe03808323f01aefb68591de263b994ffeb671af1fe040858dc820ed7b681996"
 dependencies = [
  "cstr-argument",
  "foreign-types 0.5.0",
@@ -2954,7 +2941,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "rand 0.8.3",
+ "rand 0.8.4",
  "redox_syscall",
  "remove_dir_all",
  "winapi 0.3.9",
@@ -3022,9 +3009,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.2.26"
+version = "0.2.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372"
+checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242"
 dependencies = [
  "const_fn",
  "libc",
@@ -3047,9 +3034,9 @@ dependencies = [
 
 [[package]]
 name = "time-macros-impl"
-version = "0.1.1"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5c3be1edfad6027c69f5491cf4cb310d1a71ecd6af742788c6ff8bced86b8fa"
+checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f"
 dependencies = [
  "proc-macro-hack",
  "proc-macro2",
@@ -3104,7 +3091,7 @@ dependencies = [
  "bytes 1.0.1",
  "libc",
  "memchr",
- "mio 0.7.11",
+ "mio 0.7.13",
  "num_cpus",
  "once_cell",
  "pin-project-lite 0.2.6",
@@ -3263,15 +3250,15 @@ dependencies = [
 
 [[package]]
 name = "tower"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf0aa6dfc29148c3826708dabbfa83c121eeb84df4d1468220825e3a33651687"
+checksum = "f60422bc7fefa2f3ec70359b8ff1caff59d785877eb70595904605bcc412470f"
 dependencies = [
  "futures-core",
  "futures-util",
  "indexmap",
  "pin-project 1.0.7",
- "rand 0.8.3",
+ "rand 0.8.4",
  "slab",
  "tokio 1.7.1",
  "tokio-stream",
@@ -3364,7 +3351,7 @@ dependencies = [
  "httparse",
  "input_buffer",
  "log",
- "rand 0.8.3",
+ "rand 0.8.4",
  "sha-1 0.9.6",
  "url 2.2.2",
  "utf-8",
@@ -3420,9 +3407,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.17"
+version = "0.1.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
+checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9"
 dependencies = [
  "tinyvec",
 ]
@@ -3503,9 +3490,9 @@ dependencies = [
 
 [[package]]
 name = "vcpkg"
-version = "0.2.12"
+version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vec_map"
@@ -3683,10 +3670,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "wepoll-sys"
-version = "3.0.1"
+name = "wepoll-ffi"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fcb14dea929042224824779fbc82d9fab8d2e6d3cbc0ac404de8edf489e77ff"
+checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb"
 dependencies = [
  "cc",
 ]
@@ -3820,7 +3807,7 @@ dependencies = [
  "hex",
  "nix 0.20.0",
  "once_cell",
- "rand 0.8.3",
+ "rand 0.8.4",
  "scoped-tls",
  "serde",
  "serde_repr",
diff --git a/Cargo.toml b/Cargo.toml
index 7170182..c443d65 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -37,7 +37,7 @@ shellexpand = "2.1"
 stackable_config = { git = "https://github.com/stackabletech/common.git", branch = "main" }
 strum = { version = "0.21", features = ["derive"] }
 strum_macros = "0.21"
-systemd = { git = "https://github.com/stackabletech/rust-systemd.git", branch = "stackable_patches_v0.8.2", default-features = false, features = ["journal"] }
+systemd = { version = "0.9", default-features = false, features = ["journal"] }
 tar = "0.4"
 thiserror = "1.0"
 tokio = { version = "1.7", features = ["macros", "rt-multi-thread", "time"] }

From e6b745091cfe760c1f4ba1176f688bed0fc47f07 Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Tue, 22 Jun 2021 10:40:43 +0200
Subject: [PATCH 3/7] Pull request ID fixed

---
 CHANGELOG.adoc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
index 8bd68fb..a835946 100644
--- a/CHANGELOG.adoc
+++ b/CHANGELOG.adoc
@@ -3,14 +3,14 @@
 == 0.4.0 - unreleased
 
 :188: https://github.com/stackabletech/agent/pull/188[#188]
-:204: https://github.com/stackabletech/agent/pull/204[#204]
+:205: https://github.com/stackabletech/agent/pull/205[#205]
 
 === Added
 * Annotation `featureLogs` added to the pods to indicate if logs can be
   retrieved with `kubectl logs` ({188}).
 
 === Fixed
-* Pods with restart policy `Never` handled correctly ({204}).
+* Pods with restart policy `Never` handled correctly ({205}).
 
 == 0.3.0 - 2021-05-27
 

From a2a41e19e40fd6283c9d84d46df1fac8ef829367 Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Tue, 22 Jun 2021 16:10:59 +0200
Subject: [PATCH 4/7] Pod types added to the navigation of the documentation

---
 docs/modules/ROOT/nav.adoc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
index d8136dc..f08faba 100644
--- a/docs/modules/ROOT/nav.adoc
+++ b/docs/modules/ROOT/nav.adoc
@@ -3,5 +3,8 @@
 ** xref:installation/binaries.adoc[]
 * xref:configuration.adoc[]
 * xref:limitations.adoc[]
+* Pod types
+** xref:pod_types/services.adoc[]
+** xref:pod_types/jobs.adoc[]
 * Monitoring
 ** xref:monitoring/logs.adoc[]

From f91bc22459610aad5b466293ffd2ed45e026171c Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Wed, 23 Jun 2021 10:02:26 +0200
Subject: [PATCH 5/7] Documentation improved

---
 docs/modules/ROOT/nav.adoc                            | 5 ++---
 docs/modules/ROOT/pages/{pod_types => }/jobs.adoc     | 4 ++--
 docs/modules/ROOT/pages/{pod_types => }/services.adoc | 8 +++-----
 3 files changed, 7 insertions(+), 10 deletions(-)
 rename docs/modules/ROOT/pages/{pod_types => }/jobs.adoc (87%)
 rename docs/modules/ROOT/pages/{pod_types => }/services.adoc (61%)

diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc
index f08faba..80a8c28 100644
--- a/docs/modules/ROOT/nav.adoc
+++ b/docs/modules/ROOT/nav.adoc
@@ -3,8 +3,7 @@
 ** xref:installation/binaries.adoc[]
 * xref:configuration.adoc[]
 * xref:limitations.adoc[]
-* Pod types
-** xref:pod_types/services.adoc[]
-** xref:pod_types/jobs.adoc[]
+* xref:services.adoc[]
+* xref:jobs.adoc[]
 * Monitoring
 ** xref:monitoring/logs.adoc[]
diff --git a/docs/modules/ROOT/pages/pod_types/jobs.adoc b/docs/modules/ROOT/pages/jobs.adoc
similarity index 87%
rename from docs/modules/ROOT/pages/pod_types/jobs.adoc
rename to docs/modules/ROOT/pages/jobs.adoc
index 0ed7914..a77786a 100644
--- a/docs/modules/ROOT/pages/pod_types/jobs.adoc
+++ b/docs/modules/ROOT/pages/jobs.adoc
@@ -1,7 +1,7 @@
 = Jobs
 
-Jobs which terminate after performing a task, must be created with
-`restartPolicy` set to `Never`:
+A job performs a task which terminates after some time. The
+`restartPolicy` must be explicitly set to `Never` or `OnFailure`:
 
     apiVersion: v1
     kind: Pod
diff --git a/docs/modules/ROOT/pages/pod_types/services.adoc b/docs/modules/ROOT/pages/services.adoc
similarity index 61%
rename from docs/modules/ROOT/pages/pod_types/services.adoc
rename to docs/modules/ROOT/pages/services.adoc
index 1887ae5..d743877 100644
--- a/docs/modules/ROOT/pages/pod_types/services.adoc
+++ b/docs/modules/ROOT/pages/services.adoc
@@ -1,10 +1,8 @@
 = Services
 
-Services do not terminate on their own. They run until the pod is
-deleted. The `restartPolicy` must be set to `Always` or `OnFailure`. As
-services cannot terminate successfully, there is no difference between
-these two policies. `restartPolicy` defaults to `Always` and can
-therefore be omitted.
+A pod which provides a service should never terminate on its own, so the
+`restartPolicy` must be set to `Always`. As `restartPolicy` defaults to
+`Always`, it can also be omitted.
 
     apiVersion: v1
     kind: Pod

From 452f4d104d45fc439d34de8e12094c9983fc4f5c Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Wed, 23 Jun 2021 10:13:06 +0200
Subject: [PATCH 6/7] Waiting on startup enabled only for restart policy
 `Always`

---
 src/provider/states/pod/starting.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/provider/states/pod/starting.rs b/src/provider/states/pod/starting.rs
index cf22e61..b9f7253 100644
--- a/src/provider/states/pod/starting.rs
+++ b/src/provider/states/pod/starting.rs
@@ -88,7 +88,7 @@ async fn start_service_units(
             info!("Enabling systemd unit [{}]", service_unit);
             systemd_manager.enable(service_unit).await?;
 
-            if restart_policy(pod) != RestartPolicy::Never {
+            if restart_policy(pod) == RestartPolicy::Always {
                 // TODO: does this need to be configurable, or ar we happy with a hard coded value
                 //  for now. I've briefly looked at the podspec and couldn't identify a good field
                 //  to use for this - also, currently this starts containers (= systemd units) in

From 026d008f27f53e738d95a33705f7469847dc3aa6 Mon Sep 17 00:00:00 2001
From: Siegfried Weber <mail@siegfriedweber.net>
Date: Wed, 23 Jun 2021 15:38:07 +0200
Subject: [PATCH 7/7] Restart setting in systemd units removed

---
 CHANGELOG.adoc                             |  5 ++++
 src/provider/systemdmanager/systemdunit.rs | 35 ----------------------
 2 files changed, 5 insertions(+), 35 deletions(-)

diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc
index a835946..90ea8f5 100644
--- a/CHANGELOG.adoc
+++ b/CHANGELOG.adoc
@@ -9,6 +9,11 @@
 * Annotation `featureLogs` added to the pods to indicate if logs can be
   retrieved with `kubectl logs` ({188}).
 
+=== Changed
+* Restart setting in systemd units removed because the agent already
+  monitors the units and restarts them according to the restart policy
+  in the pod spec ({205}).
+
 === Fixed
 * Pods with restart policy `Never` handled correctly ({205}).
 
diff --git a/src/provider/systemdmanager/systemdunit.rs b/src/provider/systemdmanager/systemdunit.rs
index 9b32d7f..a82f781 100644
--- a/src/provider/systemdmanager/systemdunit.rs
+++ b/src/provider/systemdmanager/systemdunit.rs
@@ -5,7 +5,6 @@ use kubelet::container::Container;
 use kubelet::pod::Pod;
 
 use crate::provider::error::StackableError;
-
 use crate::provider::error::StackableError::PodValidationError;
 use crate::provider::states::pod::creating_config::CreatingConfig;
 use crate::provider::states::pod::PodState;
@@ -18,15 +17,6 @@ use std::fmt::{Display, Formatter};
 use std::iter;
 use strum::{Display, EnumIter, IntoEnumIterator};
 
-lazy_static! {
-    // This is used to map from Kubernetes restart lingo to systemd restart terms
-    static ref RESTART_POLICY_MAP: HashMap<&'static str, &'static str> = [
-            ("Always", "always"),
-            ("OnFailure", "on-failure"),
-            ("Never", "no")
-        ].iter().cloned().collect();
-}
-
 /// The default timeout for stopping a service, after this has passed systemd will terminate
 /// the process
 const DEFAULT_TERMINATION_TIMEOUT_SECS: i64 = 30;
@@ -205,26 +195,6 @@ impl SystemDUnit {
             }
         };
 
-        // Get restart policy from pod, if none is specified default to "Never"
-        let restart_policy = pod_spec.restart_policy.as_deref().unwrap_or("Never");
-
-        // Lookup the equivalent systemd restart policy for the configured one
-        // If this lookup fails (which means a restart policy was specified which we do not know
-        // about) then we fail the entire service to avoid unpredictable behavior
-        let restart_policy = match RESTART_POLICY_MAP.get(restart_policy) {
-            Some(policy) => policy,
-            None => {
-                return Err(PodValidationError {
-                    msg: format!(
-                        "Unknown value [{}] for RestartPolicy in pod [{}]",
-                        restart_policy, unit.name
-                    ),
-                })
-            }
-        };
-
-        unit.add_property(Section::Service, "Restart", restart_policy);
-
         // If `terminationGracePeriodSeconds` was specified in the PodSpec set the value as
         // 'TimeOutStopSec` on the systemd unit
         // This means that the service will be killed after this period if it does not shutdown
@@ -512,14 +482,12 @@ mod test {
               name: stackable
             spec:
               containers: []
-              restartPolicy: Always
               securityContext:
                 windowsOptions:
                   runAsUserName: pod-user",
         "stackable.service",
         indoc! {"
             [Service]
-            Restart=always
             TimeoutStopSec=30
             User=pod-user"}
     )]
@@ -557,7 +525,6 @@ mod test {
             [Service]
             Environment="LOG_DIR=/var/log/default-stackable" "LOG_LEVEL=INFO"
             ExecStart=start.sh arg /etc/default-stackable
-            Restart=no
             StandardError=journal
             StandardOutput=journal
             TimeoutStopSec=30
@@ -591,7 +558,6 @@ mod test {
 
             [Service]
             ExecStart=start.sh
-            Restart=no
             StandardError=journal
             StandardOutput=journal
             TimeoutStopSec=30
@@ -612,7 +578,6 @@ mod test {
         "stackable.service",
         indoc! {"
             [Service]
-            Restart=no
             TimeoutStopSec=10"}
     )]