diff --git a/container/crio/handler.go b/container/crio/handler.go
index 13311c57..4d7efbf4 100644
--- a/container/crio/handler.go
+++ b/container/crio/handler.go
@@ -65,9 +65,6 @@ type crioContainerHandler struct {
 
 	ignoreMetrics container.MetricSet
 
-	// container restart count
-	restartCount int
-
 	reference info.ContainerReference
 
 	libcontainerHandler *containerlibcontainer.Handler
@@ -166,7 +163,10 @@ func newCrioContainerHandler(
 	// ignore err and get zero as default, this happens with sandboxes, not sure why...
 	// kube isn't sending restart count in labels for sandboxes.
 	restartCount, _ := strconv.Atoi(cInfo.Annotations["io.kubernetes.container.restartCount"])
-	handler.restartCount = restartCount
+	// Only adds restartcount label if it's greater than 0
+	if restartCount > 0 {
+		handler.labels["restartcount"] = strconv.Itoa(restartCount)
+	}
 
 	handler.ipAddress = cInfo.IP
 
@@ -210,10 +210,6 @@ func (self *crioContainerHandler) GetSpec() (info.ContainerSpec, error) {
 	spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem)
 
 	spec.Labels = self.labels
-	// Only adds restartcount label if it's greater than 0
-	if self.restartCount > 0 {
-		spec.Labels["restartcount"] = strconv.Itoa(self.restartCount)
-	}
 	spec.Envs = self.envs
 	spec.Image = self.image
 
diff --git a/container/docker/handler.go b/container/docker/handler.go
index 22b9455f..64726053 100644
--- a/container/docker/handler.go
+++ b/container/docker/handler.go
@@ -91,9 +91,6 @@ type dockerContainerHandler struct {
 	// zfsParent is the parent for docker zfs
 	zfsParent string
 
-	// container restart count
-	restartCount int
-
 	// Reference to the container
 	reference info.ContainerReference
 
@@ -226,7 +223,10 @@ func newDockerContainerHandler(
 	}
 	handler.image = ctnr.Config.Image
 	handler.networkMode = ctnr.HostConfig.NetworkMode
-	handler.restartCount = ctnr.RestartCount
+	// Only adds restartcount label if it's greater than 0
+	if ctnr.RestartCount > 0 {
+		handler.labels["restartcount"] = strconv.Itoa(ctnr.RestartCount)
+	}
 
 	// Obtain the IP address for the contianer.
 	// If the NetworkMode starts with 'container:' then we need to use the IP address of the container specified.
@@ -356,10 +356,6 @@ func (self *dockerContainerHandler) GetSpec() (info.ContainerSpec, error) {
 	spec, err := common.GetSpec(self.cgroupPaths, self.machineInfoFactory, self.needNet(), hasFilesystem)
 
 	spec.Labels = self.labels
-	// Only adds restartcount label if it's greater than 0
-	if self.restartCount > 0 {
-		spec.Labels["restartcount"] = strconv.Itoa(self.restartCount)
-	}
 	spec.Envs = self.envs
 	spec.Image = self.image
 	spec.CreationTime = self.creationTime