]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - arch/sparc/crypto/aes_glue.c
sparc64: Set CRYPTO_TFM_REQ_MAY_SLEEP consistently in AES code.
[~shefty/rdma-dev.git] / arch / sparc / crypto / aes_glue.c
1 /* Glue code for AES encryption optimized for sparc64 crypto opcodes.
2  *
3  * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9  * interface for 64-bit kernels.
10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
12  *             Tadeusz Struk (tadeusz.struk@intel.com)
13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *    Copyright (c) 2010, Intel Corporation.
15  */
16
17 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
18
19 #include <linux/crypto.h>
20 #include <linux/init.h>
21 #include <linux/module.h>
22 #include <linux/mm.h>
23 #include <linux/types.h>
24 #include <crypto/algapi.h>
25 #include <crypto/aes.h>
26
27 #include <asm/fpumacro.h>
28 #include <asm/pstate.h>
29 #include <asm/elf.h>
30
31 #include "opcodes.h"
32
33 struct aes_ops {
34         void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
35         void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
36         void (*load_encrypt_keys)(const u64 *key);
37         void (*load_decrypt_keys)(const u64 *key);
38         void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
39                             unsigned int len);
40         void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
41                             unsigned int len);
42         void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
43                             unsigned int len, u64 *iv);
44         void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
45                             unsigned int len, u64 *iv);
46         void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
47                           unsigned int len, u64 *iv);
48 };
49
50 struct crypto_sparc64_aes_ctx {
51         struct aes_ops *ops;
52         u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
53         u32 key_length;
54         u32 expanded_key_length;
55 };
56
57 extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
58                                     u32 *output);
59 extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
60                                     u32 *output);
61 extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
62                                     u32 *output);
63
64 extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
65                                     u32 *output);
66 extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
67                                     u32 *output);
68 extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
69                                     u32 *output);
70
71 extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
72 extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
73 extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
74
75 extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
76 extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
77 extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
78
79 extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
80                                         u64 *output, unsigned int len);
81 extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
82                                         u64 *output, unsigned int len);
83 extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
84                                         u64 *output, unsigned int len);
85
86 extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
87                                         u64 *output, unsigned int len);
88 extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
89                                         u64 *output, unsigned int len);
90 extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
91                                         u64 *output, unsigned int len);
92
93 extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
94                                         u64 *output, unsigned int len,
95                                         u64 *iv);
96
97 extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
98                                         u64 *output, unsigned int len,
99                                         u64 *iv);
100
101 extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
102                                         u64 *output, unsigned int len,
103                                         u64 *iv);
104
105 extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
106                                         u64 *output, unsigned int len,
107                                         u64 *iv);
108
109 extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
110                                         u64 *output, unsigned int len,
111                                         u64 *iv);
112
113 extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
114                                         u64 *output, unsigned int len,
115                                         u64 *iv);
116
117 extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
118                                       u64 *output, unsigned int len,
119                                       u64 *iv);
120 extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
121                                       u64 *output, unsigned int len,
122                                       u64 *iv);
123 extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
124                                       u64 *output, unsigned int len,
125                                       u64 *iv);
126
127 struct aes_ops aes128_ops = {
128         .encrypt                = aes_sparc64_encrypt_128,
129         .decrypt                = aes_sparc64_decrypt_128,
130         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_128,
131         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_128,
132         .ecb_encrypt            = aes_sparc64_ecb_encrypt_128,
133         .ecb_decrypt            = aes_sparc64_ecb_decrypt_128,
134         .cbc_encrypt            = aes_sparc64_cbc_encrypt_128,
135         .cbc_decrypt            = aes_sparc64_cbc_decrypt_128,
136         .ctr_crypt              = aes_sparc64_ctr_crypt_128,
137 };
138
139 struct aes_ops aes192_ops = {
140         .encrypt                = aes_sparc64_encrypt_192,
141         .decrypt                = aes_sparc64_decrypt_192,
142         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_192,
143         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_192,
144         .ecb_encrypt            = aes_sparc64_ecb_encrypt_192,
145         .ecb_decrypt            = aes_sparc64_ecb_decrypt_192,
146         .cbc_encrypt            = aes_sparc64_cbc_encrypt_192,
147         .cbc_decrypt            = aes_sparc64_cbc_decrypt_192,
148         .ctr_crypt              = aes_sparc64_ctr_crypt_192,
149 };
150
151 struct aes_ops aes256_ops = {
152         .encrypt                = aes_sparc64_encrypt_256,
153         .decrypt                = aes_sparc64_decrypt_256,
154         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_256,
155         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_256,
156         .ecb_encrypt            = aes_sparc64_ecb_encrypt_256,
157         .ecb_decrypt            = aes_sparc64_ecb_decrypt_256,
158         .cbc_encrypt            = aes_sparc64_cbc_encrypt_256,
159         .cbc_decrypt            = aes_sparc64_cbc_decrypt_256,
160         .ctr_crypt              = aes_sparc64_ctr_crypt_256,
161 };
162
163 extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
164                                    unsigned int key_len);
165
166 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
167                        unsigned int key_len)
168 {
169         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
170         u32 *flags = &tfm->crt_flags;
171
172         switch (key_len) {
173         case AES_KEYSIZE_128:
174                 ctx->expanded_key_length = 0xb0;
175                 ctx->ops = &aes128_ops;
176                 break;
177
178         case AES_KEYSIZE_192:
179                 ctx->expanded_key_length = 0xd0;
180                 ctx->ops = &aes192_ops;
181                 break;
182
183         case AES_KEYSIZE_256:
184                 ctx->expanded_key_length = 0xf0;
185                 ctx->ops = &aes256_ops;
186                 break;
187
188         default:
189                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
190                 return -EINVAL;
191         }
192
193         aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
194         ctx->key_length = key_len;
195
196         return 0;
197 }
198
199 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
200 {
201         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
202
203         ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
204 }
205
206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
207 {
208         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
209
210         ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
211 }
212
213 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
214
215 static int ecb_encrypt(struct blkcipher_desc *desc,
216                        struct scatterlist *dst, struct scatterlist *src,
217                        unsigned int nbytes)
218 {
219         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
220         struct blkcipher_walk walk;
221         int err;
222
223         blkcipher_walk_init(&walk, dst, src, nbytes);
224         err = blkcipher_walk_virt(desc, &walk);
225         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
226
227         ctx->ops->load_encrypt_keys(&ctx->key[0]);
228         while ((nbytes = walk.nbytes)) {
229                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
230
231                 if (likely(block_len)) {
232                         ctx->ops->ecb_encrypt(&ctx->key[0],
233                                               (const u64 *)walk.src.virt.addr,
234                                               (u64 *) walk.dst.virt.addr,
235                                               block_len);
236                 }
237                 nbytes &= AES_BLOCK_SIZE - 1;
238                 err = blkcipher_walk_done(desc, &walk, nbytes);
239         }
240         fprs_write(0);
241         return err;
242 }
243
244 static int ecb_decrypt(struct blkcipher_desc *desc,
245                        struct scatterlist *dst, struct scatterlist *src,
246                        unsigned int nbytes)
247 {
248         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
249         struct blkcipher_walk walk;
250         u64 *key_end;
251         int err;
252
253         blkcipher_walk_init(&walk, dst, src, nbytes);
254         err = blkcipher_walk_virt(desc, &walk);
255         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
256
257         ctx->ops->load_decrypt_keys(&ctx->key[0]);
258         key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
259         while ((nbytes = walk.nbytes)) {
260                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
261
262                 if (likely(block_len)) {
263                         ctx->ops->ecb_decrypt(key_end,
264                                               (const u64 *) walk.src.virt.addr,
265                                               (u64 *) walk.dst.virt.addr, block_len);
266                 }
267                 nbytes &= AES_BLOCK_SIZE - 1;
268                 err = blkcipher_walk_done(desc, &walk, nbytes);
269         }
270         fprs_write(0);
271
272         return err;
273 }
274
275 static int cbc_encrypt(struct blkcipher_desc *desc,
276                        struct scatterlist *dst, struct scatterlist *src,
277                        unsigned int nbytes)
278 {
279         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
280         struct blkcipher_walk walk;
281         int err;
282
283         blkcipher_walk_init(&walk, dst, src, nbytes);
284         err = blkcipher_walk_virt(desc, &walk);
285         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
286
287         ctx->ops->load_encrypt_keys(&ctx->key[0]);
288         while ((nbytes = walk.nbytes)) {
289                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
290
291                 if (likely(block_len)) {
292                         ctx->ops->cbc_encrypt(&ctx->key[0],
293                                               (const u64 *)walk.src.virt.addr,
294                                               (u64 *) walk.dst.virt.addr,
295                                               block_len, (u64 *) walk.iv);
296                 }
297                 nbytes &= AES_BLOCK_SIZE - 1;
298                 err = blkcipher_walk_done(desc, &walk, nbytes);
299         }
300         fprs_write(0);
301         return err;
302 }
303
304 static int cbc_decrypt(struct blkcipher_desc *desc,
305                        struct scatterlist *dst, struct scatterlist *src,
306                        unsigned int nbytes)
307 {
308         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
309         struct blkcipher_walk walk;
310         u64 *key_end;
311         int err;
312
313         blkcipher_walk_init(&walk, dst, src, nbytes);
314         err = blkcipher_walk_virt(desc, &walk);
315         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
316
317         ctx->ops->load_decrypt_keys(&ctx->key[0]);
318         key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
319         while ((nbytes = walk.nbytes)) {
320                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
321
322                 if (likely(block_len)) {
323                         ctx->ops->cbc_decrypt(key_end,
324                                               (const u64 *) walk.src.virt.addr,
325                                               (u64 *) walk.dst.virt.addr,
326                                               block_len, (u64 *) walk.iv);
327                 }
328                 nbytes &= AES_BLOCK_SIZE - 1;
329                 err = blkcipher_walk_done(desc, &walk, nbytes);
330         }
331         fprs_write(0);
332
333         return err;
334 }
335
336 static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx,
337                             struct blkcipher_walk *walk)
338 {
339         u8 *ctrblk = walk->iv;
340         u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
341         u8 *src = walk->src.virt.addr;
342         u8 *dst = walk->dst.virt.addr;
343         unsigned int nbytes = walk->nbytes;
344
345         ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
346                               keystream, AES_BLOCK_SIZE);
347         crypto_xor((u8 *) keystream, src, nbytes);
348         memcpy(dst, keystream, nbytes);
349         crypto_inc(ctrblk, AES_BLOCK_SIZE);
350 }
351
352 static int ctr_crypt(struct blkcipher_desc *desc,
353                      struct scatterlist *dst, struct scatterlist *src,
354                      unsigned int nbytes)
355 {
356         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
357         struct blkcipher_walk walk;
358         int err;
359
360         blkcipher_walk_init(&walk, dst, src, nbytes);
361         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
362         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
363
364         ctx->ops->load_encrypt_keys(&ctx->key[0]);
365         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
366                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
367
368                 if (likely(block_len)) {
369                         ctx->ops->ctr_crypt(&ctx->key[0],
370                                             (const u64 *)walk.src.virt.addr,
371                                             (u64 *) walk.dst.virt.addr,
372                                             block_len, (u64 *) walk.iv);
373                 }
374                 nbytes &= AES_BLOCK_SIZE - 1;
375                 err = blkcipher_walk_done(desc, &walk, nbytes);
376         }
377         if (walk.nbytes) {
378                 ctr_crypt_final(ctx, &walk);
379                 err = blkcipher_walk_done(desc, &walk, 0);
380         }
381         fprs_write(0);
382         return err;
383 }
384
385 static struct crypto_alg algs[] = { {
386         .cra_name               = "aes",
387         .cra_driver_name        = "aes-sparc64",
388         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
389         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
390         .cra_blocksize          = AES_BLOCK_SIZE,
391         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
392         .cra_alignmask          = 3,
393         .cra_module             = THIS_MODULE,
394         .cra_u  = {
395                 .cipher = {
396                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
397                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
398                         .cia_setkey             = aes_set_key,
399                         .cia_encrypt            = aes_encrypt,
400                         .cia_decrypt            = aes_decrypt
401                 }
402         }
403 }, {
404         .cra_name               = "ecb(aes)",
405         .cra_driver_name        = "ecb-aes-sparc64",
406         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
407         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
408         .cra_blocksize          = AES_BLOCK_SIZE,
409         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
410         .cra_alignmask          = 7,
411         .cra_type               = &crypto_blkcipher_type,
412         .cra_module             = THIS_MODULE,
413         .cra_u = {
414                 .blkcipher = {
415                         .min_keysize    = AES_MIN_KEY_SIZE,
416                         .max_keysize    = AES_MAX_KEY_SIZE,
417                         .setkey         = aes_set_key,
418                         .encrypt        = ecb_encrypt,
419                         .decrypt        = ecb_decrypt,
420                 },
421         },
422 }, {
423         .cra_name               = "cbc(aes)",
424         .cra_driver_name        = "cbc-aes-sparc64",
425         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
426         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
427         .cra_blocksize          = AES_BLOCK_SIZE,
428         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
429         .cra_alignmask          = 7,
430         .cra_type               = &crypto_blkcipher_type,
431         .cra_module             = THIS_MODULE,
432         .cra_u = {
433                 .blkcipher = {
434                         .min_keysize    = AES_MIN_KEY_SIZE,
435                         .max_keysize    = AES_MAX_KEY_SIZE,
436                         .setkey         = aes_set_key,
437                         .encrypt        = cbc_encrypt,
438                         .decrypt        = cbc_decrypt,
439                 },
440         },
441 }, {
442         .cra_name               = "ctr(aes)",
443         .cra_driver_name        = "ctr-aes-sparc64",
444         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
445         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
446         .cra_blocksize          = 1,
447         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
448         .cra_alignmask          = 7,
449         .cra_type               = &crypto_blkcipher_type,
450         .cra_module             = THIS_MODULE,
451         .cra_u = {
452                 .blkcipher = {
453                         .min_keysize    = AES_MIN_KEY_SIZE,
454                         .max_keysize    = AES_MAX_KEY_SIZE,
455                         .setkey         = aes_set_key,
456                         .encrypt        = ctr_crypt,
457                         .decrypt        = ctr_crypt,
458                 },
459         },
460 } };
461
462 static bool __init sparc64_has_aes_opcode(void)
463 {
464         unsigned long cfr;
465
466         if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
467                 return false;
468
469         __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
470         if (!(cfr & CFR_AES))
471                 return false;
472
473         return true;
474 }
475
476 static int __init aes_sparc64_mod_init(void)
477 {
478         int i;
479
480         for (i = 0; i < ARRAY_SIZE(algs); i++)
481                 INIT_LIST_HEAD(&algs[i].cra_list);
482
483         if (sparc64_has_aes_opcode()) {
484                 pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
485                 return crypto_register_algs(algs, ARRAY_SIZE(algs));
486         }
487         pr_info("sparc64 aes opcodes not available.\n");
488         return -ENODEV;
489 }
490
491 static void __exit aes_sparc64_mod_fini(void)
492 {
493         crypto_unregister_algs(algs, ARRAY_SIZE(algs));
494 }
495
496 module_init(aes_sparc64_mod_init);
497 module_exit(aes_sparc64_mod_fini);
498
499 MODULE_LICENSE("GPL");
500 MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated");
501
502 MODULE_ALIAS("aes");
503
504 #include "crop_devid.c"