diff --git a/libkpod/sandbox/sandbox.go b/libkpod/sandbox/sandbox.go
index b19ff819..51097c2e 100644
--- a/libkpod/sandbox/sandbox.go
+++ b/libkpod/sandbox/sandbox.go
@@ -153,6 +153,7 @@ type Sandbox struct {
 	resolvPath     string
 	hostname       string
 	portMappings   []*hostport.PortMapping
+	stopped        bool
 }
 
 const (
@@ -381,6 +382,19 @@ func (s *Sandbox) NetNsCreate() error {
 	return nil
 }
 
+// SetStopped sets the sandbox state to stopped.
+// This should be set after a stop operation succeeds
+// so that subsequent stops can return fast.
+func (s *Sandbox) SetStopped() {
+	s.stopped = true
+}
+
+// Stopped returns whether the sandbox state has been
+// set to stopped.
+func (s *Sandbox) Stopped() bool {
+	return s.stopped
+}
+
 // NetNsJoin attempts to join the sandbox to an existing network namespace
 // This will fail if the sandbox is already part of a network namespace
 func (s *Sandbox) NetNsJoin(nspath, name string) error {
diff --git a/server/sandbox_stop.go b/server/sandbox_stop.go
index ecff1cfe..2332c6cc 100644
--- a/server/sandbox_stop.go
+++ b/server/sandbox_stop.go
@@ -37,6 +37,10 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
 		return resp, nil
 	}
 
+	if sb.Stopped() {
+		return &pb.StopPodSandboxResponse{}, nil
+	}
+
 	podInfraContainer := sb.InfraContainer()
 	netnsPath, err := podInfraContainer.NetNsPath()
 	if err != nil {
@@ -110,6 +114,7 @@ func (s *Server) StopPodSandbox(ctx context.Context, req *pb.StopPodSandboxReque
 		logrus.Warnf("failed to stop sandbox container in pod sandbox %s: %v", sb.ID(), err)
 	}
 
+	sb.SetStopped()
 	resp := &pb.StopPodSandboxResponse{}
 	logrus.Debugf("StopPodSandboxResponse: %+v", resp)
 	return resp, nil