crypto: ccp - Use a single queue for proper ordering of tfm requests

Move to a single queue to serialize requests within a tfm. When
testing using IPSec with a large number of network connections
the per cpu tfm queuing logic was not working properly.

Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index b3f22b07b5bd554daf3bc31d7d0a288575d8826c..010fded5d46ba32184bc031333a4333a820e8781 100644 (file)
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
 static LIST_HEAD(hash_algs);
 static LIST_HEAD(cipher_algs);
 
-/* For any tfm, requests for that tfm on the same CPU must be returned
- * in the order received.  With multiple queues available, the CCP can
- * process more than one cmd at a time.  Therefore we must maintain
- * a cmd list to insure the proper ordering of requests on a given tfm/cpu
- * combination.
+/* For any tfm, requests for that tfm must be returned on the order
+ * received.  With multiple queues available, the CCP can process more
+ * than one cmd at a time.  Therefore we must maintain a cmd list to insure
+ * the proper ordering of requests on a given tfm.
  */
-struct ccp_crypto_cpu_queue {
+struct ccp_crypto_queue {
        struct list_head cmds;
        struct list_head *backlog;
        unsigned int cmd_count;
 };
-#define CCP_CRYPTO_MAX_QLEN    50
+#define CCP_CRYPTO_MAX_QLEN    100
 
-struct ccp_crypto_percpu_queue {
-       struct ccp_crypto_cpu_queue __percpu *cpu_queue;
-};
-static struct ccp_crypto_percpu_queue req_queue;
+static struct ccp_crypto_queue req_queue;
+static spinlock_t req_queue_lock;
 
 struct ccp_crypto_cmd {
        struct list_head entry;
@@ -71,8 +68,6 @@ struct ccp_crypto_cmd {
 
        /* Used for held command processing to determine state */
        int ret;
-
-       int cpu;
 };
 
 struct ccp_crypto_cpu {
@@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err)
        return true;
 }
 
-/*
- * ccp_crypto_cmd_complete must be called while running on the appropriate
- * cpu and the caller must have done a get_cpu to disable preemption
- */
 static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
        struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)
 {
-       struct ccp_crypto_cpu_queue *cpu_queue;
        struct ccp_crypto_cmd *held = NULL, *tmp;
+       unsigned long flags;
 
        *backlog = NULL;
 
-       cpu_queue = this_cpu_ptr(req_queue.cpu_queue);
+       spin_lock_irqsave(&req_queue_lock, flags);
 
        /* Held cmds will be after the current cmd in the queue so start
         * searching for a cmd with a matching tfm for submission.
         */
        tmp = crypto_cmd;
-       list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) {
+       list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {
                if (crypto_cmd->tfm != tmp->tfm)
                        continue;
                held = tmp;
@@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
         *   Because cmds can be executed from any point in the cmd list
         *   special precautions have to be taken when handling the backlog.
         */
-       if (cpu_queue->backlog != &cpu_queue->cmds) {
+       if (req_queue.backlog != &req_queue.cmds) {
                /* Skip over this cmd if it is the next backlog cmd */
-               if (cpu_queue->backlog == &crypto_cmd->entry)
-                       cpu_queue->backlog = crypto_cmd->entry.next;
+               if (req_queue.backlog == &crypto_cmd->entry)
+                       req_queue.backlog = crypto_cmd->entry.next;
 
-               *backlog = container_of(cpu_queue->backlog,
+               *backlog = container_of(req_queue.backlog,
                                        struct ccp_crypto_cmd, entry);
-               cpu_queue->backlog = cpu_queue->backlog->next;
+               req_queue.backlog = req_queue.backlog->next;
 
                /* Skip over this cmd if it is now the next backlog cmd */
-               if (cpu_queue->backlog == &crypto_cmd->entry)
-                       cpu_queue->backlog = crypto_cmd->entry.next;
+               if (req_queue.backlog == &crypto_cmd->entry)
+                       req_queue.backlog = crypto_cmd->entry.next;
        }
 
        /* Remove the cmd entry from the list of cmds */
-       cpu_queue->cmd_count--;
+       req_queue.cmd_count--;
        list_del(&crypto_cmd->entry);
 
+       spin_unlock_irqrestore(&req_queue_lock, flags);
+
        return held;
 }
 
-static void ccp_crypto_complete_on_cpu(struct work_struct *work)
+static void ccp_crypto_complete(void *data, int err)
 {
-       struct ccp_crypto_cpu *cpu_work =
-               container_of(work, struct ccp_crypto_cpu, work);
-       struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd;
+       struct ccp_crypto_cmd *crypto_cmd = data;
        struct ccp_crypto_cmd *held, *next, *backlog;
        struct crypto_async_request *req = crypto_cmd->req;
        struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
-       int cpu, ret;
-
-       cpu = get_cpu();
+       int ret;
 
-       if (cpu_work->err == -EINPROGRESS) {
+       if (err == -EINPROGRESS) {
                /* Only propogate the -EINPROGRESS if necessary */
                if (crypto_cmd->ret == -EBUSY) {
                        crypto_cmd->ret = -EINPROGRESS;
                        req->complete(req, -EINPROGRESS);
                }
 
-               goto e_cpu;
+               return;
        }
 
        /* Operation has completed - update the queue before invoking
@@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
                req->complete(req, -EINPROGRESS);
 
        /* Completion callbacks */
-       ret = cpu_work->err;
+       ret = err;
        if (ctx->complete)
                ret = ctx->complete(req, ret);
        req->complete(req, ret);
@@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
        }
 
        kfree(crypto_cmd);
-
-e_cpu:
-       put_cpu();
-
-       complete(&cpu_work->completion);
-}
-
-static void ccp_crypto_complete(void *data, int err)
-{
-       struct ccp_crypto_cmd *crypto_cmd = data;
-       struct ccp_crypto_cpu cpu_work;
-
-       INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu);
-       init_completion(&cpu_work.completion);
-       cpu_work.crypto_cmd = crypto_cmd;
-       cpu_work.err = err;
-
-       schedule_work_on(crypto_cmd->cpu, &cpu_work.work);
-
-       /* Keep the completion call synchronous */
-       wait_for_completion(&cpu_work.completion);
 }
 
 static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
 {
-       struct ccp_crypto_cpu_queue *cpu_queue;
        struct ccp_crypto_cmd *active = NULL, *tmp;
-       int cpu, ret;
-
-       cpu = get_cpu();
-       crypto_cmd->cpu = cpu;
+       unsigned long flags;
+       int ret;
 
-       cpu_queue = this_cpu_ptr(req_queue.cpu_queue);
+       spin_lock_irqsave(&req_queue_lock, flags);
 
        /* Check if the cmd can/should be queued */
-       if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {
+       if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
                ret = -EBUSY;
                if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))
-                       goto e_cpu;
+                       goto e_lock;
        }
 
        /* Look for an entry with the same tfm.  If there is a cmd
-        * with the same tfm in the list for this cpu then the current
-        * cmd cannot be submitted to the CCP yet.
+        * with the same tfm in the list then the current cmd cannot
+        * be submitted to the CCP yet.
         */
-       list_for_each_entry(tmp, &cpu_queue->cmds, entry) {
+       list_for_each_entry(tmp, &req_queue.cmds, entry) {
                if (crypto_cmd->tfm != tmp->tfm)
                        continue;
                active = tmp;
@@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
        if (!active) {
                ret = ccp_enqueue_cmd(crypto_cmd->cmd);
                if (!ccp_crypto_success(ret))
-                       goto e_cpu;
+                       goto e_lock;
        }
 
-       if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {
+       if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
                ret = -EBUSY;
-               if (cpu_queue->backlog == &cpu_queue->cmds)
-                       cpu_queue->backlog = &crypto_cmd->entry;
+               if (req_queue.backlog == &req_queue.cmds)
+                       req_queue.backlog = &crypto_cmd->entry;
        }
        crypto_cmd->ret = ret;
 
-       cpu_queue->cmd_count++;
-       list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds);
+       req_queue.cmd_count++;
+       list_add_tail(&crypto_cmd->entry, &req_queue.cmds);
 
-e_cpu:
-       put_cpu();
+e_lock:
+       spin_unlock_irqrestore(&req_queue_lock, flags);
 
        return ret;
 }
@@ -387,50 +352,18 @@ static void ccp_unregister_algs(void)
        }
 }
 
-static int ccp_init_queues(void)
-{
-       struct ccp_crypto_cpu_queue *cpu_queue;
-       int cpu;
-
-       req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue);
-       if (!req_queue.cpu_queue)
-               return -ENOMEM;
-
-       for_each_possible_cpu(cpu) {
-               cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
-               INIT_LIST_HEAD(&cpu_queue->cmds);
-               cpu_queue->backlog = &cpu_queue->cmds;
-               cpu_queue->cmd_count = 0;
-       }
-
-       return 0;
-}
-
-static void ccp_fini_queue(void)
-{
-       struct ccp_crypto_cpu_queue *cpu_queue;
-       int cpu;
-
-       for_each_possible_cpu(cpu) {
-               cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
-               BUG_ON(!list_empty(&cpu_queue->cmds));
-       }
-       free_percpu(req_queue.cpu_queue);
-}
-
 static int ccp_crypto_init(void)
 {
        int ret;
 
-       ret = ccp_init_queues();
-       if (ret)
-               return ret;
+       spin_lock_init(&req_queue_lock);
+       INIT_LIST_HEAD(&req_queue.cmds);
+       req_queue.backlog = &req_queue.cmds;
+       req_queue.cmd_count = 0;
 
        ret = ccp_register_algs();
-       if (ret) {
+       if (ret)
                ccp_unregister_algs();
-               ccp_fini_queue();
-       }
 
        return ret;
 }
@@ -438,7 +371,6 @@ static int ccp_crypto_init(void)
 static void ccp_crypto_exit(void)
 {
        ccp_unregister_algs();
-       ccp_fini_queue();
 }
 
 module_init(ccp_crypto_init);