]> git.openfabrics.org - ~shefty/rdma-dev.git/blob - arch/sparc/crypto/aes_glue.c
d26e75126fb5940a93fd6d1c04c01f30647a6926
[~shefty/rdma-dev.git] / arch / sparc / crypto / aes_glue.c
1 /* Glue code for AES encryption optimized for sparc64 crypto opcodes.
2  *
3  * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
4  *
5  * Copyright (C) 2008, Intel Corp.
6  *    Author: Huang Ying <ying.huang@intel.com>
7  *
8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
9  * interface for 64-bit kernels.
10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
12  *             Tadeusz Struk (tadeusz.struk@intel.com)
13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *    Copyright (c) 2010, Intel Corporation.
15  */
16
17 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
18
19 #include <linux/crypto.h>
20 #include <linux/init.h>
21 #include <linux/module.h>
22 #include <linux/mm.h>
23 #include <linux/types.h>
24 #include <crypto/algapi.h>
25 #include <crypto/aes.h>
26
27 #include <asm/fpumacro.h>
28 #include <asm/pstate.h>
29 #include <asm/elf.h>
30
31 #include "opcodes.h"
32
33 struct aes_ops {
34         void (*encrypt)(const u64 *key, const u32 *input, u32 *output);
35         void (*decrypt)(const u64 *key, const u32 *input, u32 *output);
36         void (*load_encrypt_keys)(const u64 *key);
37         void (*load_decrypt_keys)(const u64 *key);
38         void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
39                             unsigned int len);
40         void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
41                             unsigned int len);
42         void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
43                             unsigned int len, u64 *iv);
44         void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
45                             unsigned int len, u64 *iv);
46         void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
47                           unsigned int len, u64 *iv);
48 };
49
50 struct crypto_sparc64_aes_ctx {
51         struct aes_ops *ops;
52         u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
53         u32 key_length;
54         u32 expanded_key_length;
55 };
56
57 extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input,
58                                     u32 *output);
59 extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input,
60                                     u32 *output);
61 extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input,
62                                     u32 *output);
63
64 extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input,
65                                     u32 *output);
66 extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input,
67                                     u32 *output);
68 extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input,
69                                     u32 *output);
70
71 extern void aes_sparc64_load_encrypt_keys_128(const u64 *key);
72 extern void aes_sparc64_load_encrypt_keys_192(const u64 *key);
73 extern void aes_sparc64_load_encrypt_keys_256(const u64 *key);
74
75 extern void aes_sparc64_load_decrypt_keys_128(const u64 *key);
76 extern void aes_sparc64_load_decrypt_keys_192(const u64 *key);
77 extern void aes_sparc64_load_decrypt_keys_256(const u64 *key);
78
79 extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input,
80                                         u64 *output, unsigned int len);
81 extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input,
82                                         u64 *output, unsigned int len);
83 extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input,
84                                         u64 *output, unsigned int len);
85
86 extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input,
87                                         u64 *output, unsigned int len);
88 extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input,
89                                         u64 *output, unsigned int len);
90 extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input,
91                                         u64 *output, unsigned int len);
92
93 extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input,
94                                         u64 *output, unsigned int len,
95                                         u64 *iv);
96
97 extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input,
98                                         u64 *output, unsigned int len,
99                                         u64 *iv);
100
101 extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input,
102                                         u64 *output, unsigned int len,
103                                         u64 *iv);
104
105 extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input,
106                                         u64 *output, unsigned int len,
107                                         u64 *iv);
108
109 extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input,
110                                         u64 *output, unsigned int len,
111                                         u64 *iv);
112
113 extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input,
114                                         u64 *output, unsigned int len,
115                                         u64 *iv);
116
117 extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input,
118                                       u64 *output, unsigned int len,
119                                       u64 *iv);
120 extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input,
121                                       u64 *output, unsigned int len,
122                                       u64 *iv);
123 extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input,
124                                       u64 *output, unsigned int len,
125                                       u64 *iv);
126
127 struct aes_ops aes128_ops = {
128         .encrypt                = aes_sparc64_encrypt_128,
129         .decrypt                = aes_sparc64_decrypt_128,
130         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_128,
131         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_128,
132         .ecb_encrypt            = aes_sparc64_ecb_encrypt_128,
133         .ecb_decrypt            = aes_sparc64_ecb_decrypt_128,
134         .cbc_encrypt            = aes_sparc64_cbc_encrypt_128,
135         .cbc_decrypt            = aes_sparc64_cbc_decrypt_128,
136         .ctr_crypt              = aes_sparc64_ctr_crypt_128,
137 };
138
139 struct aes_ops aes192_ops = {
140         .encrypt                = aes_sparc64_encrypt_192,
141         .decrypt                = aes_sparc64_decrypt_192,
142         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_192,
143         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_192,
144         .ecb_encrypt            = aes_sparc64_ecb_encrypt_192,
145         .ecb_decrypt            = aes_sparc64_ecb_decrypt_192,
146         .cbc_encrypt            = aes_sparc64_cbc_encrypt_192,
147         .cbc_decrypt            = aes_sparc64_cbc_decrypt_192,
148         .ctr_crypt              = aes_sparc64_ctr_crypt_192,
149 };
150
151 struct aes_ops aes256_ops = {
152         .encrypt                = aes_sparc64_encrypt_256,
153         .decrypt                = aes_sparc64_decrypt_256,
154         .load_encrypt_keys      = aes_sparc64_load_encrypt_keys_256,
155         .load_decrypt_keys      = aes_sparc64_load_decrypt_keys_256,
156         .ecb_encrypt            = aes_sparc64_ecb_encrypt_256,
157         .ecb_decrypt            = aes_sparc64_ecb_decrypt_256,
158         .cbc_encrypt            = aes_sparc64_cbc_encrypt_256,
159         .cbc_decrypt            = aes_sparc64_cbc_decrypt_256,
160         .ctr_crypt              = aes_sparc64_ctr_crypt_256,
161 };
162
163 extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
164                                    unsigned int key_len);
165
166 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
167                        unsigned int key_len)
168 {
169         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
170         u32 *flags = &tfm->crt_flags;
171
172         switch (key_len) {
173         case AES_KEYSIZE_128:
174                 ctx->expanded_key_length = 0xb0;
175                 ctx->ops = &aes128_ops;
176                 break;
177
178         case AES_KEYSIZE_192:
179                 ctx->expanded_key_length = 0xd0;
180                 ctx->ops = &aes192_ops;
181                 break;
182
183         case AES_KEYSIZE_256:
184                 ctx->expanded_key_length = 0xf0;
185                 ctx->ops = &aes256_ops;
186                 break;
187
188         default:
189                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
190                 return -EINVAL;
191         }
192
193         aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
194         ctx->key_length = key_len;
195
196         return 0;
197 }
198
199 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
200 {
201         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
202
203         ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
204 }
205
206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
207 {
208         struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm);
209
210         ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst);
211 }
212
213 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
214
215 static int ecb_encrypt(struct blkcipher_desc *desc,
216                        struct scatterlist *dst, struct scatterlist *src,
217                        unsigned int nbytes)
218 {
219         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
220         struct blkcipher_walk walk;
221         int err;
222
223         blkcipher_walk_init(&walk, dst, src, nbytes);
224         err = blkcipher_walk_virt(desc, &walk);
225
226         ctx->ops->load_encrypt_keys(&ctx->key[0]);
227         while ((nbytes = walk.nbytes)) {
228                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
229
230                 if (likely(block_len)) {
231                         ctx->ops->ecb_encrypt(&ctx->key[0],
232                                               (const u64 *)walk.src.virt.addr,
233                                               (u64 *) walk.dst.virt.addr,
234                                               block_len);
235                 }
236                 nbytes &= AES_BLOCK_SIZE - 1;
237                 err = blkcipher_walk_done(desc, &walk, nbytes);
238         }
239         fprs_write(0);
240         return err;
241 }
242
243 static int ecb_decrypt(struct blkcipher_desc *desc,
244                        struct scatterlist *dst, struct scatterlist *src,
245                        unsigned int nbytes)
246 {
247         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
248         struct blkcipher_walk walk;
249         u64 *key_end;
250         int err;
251
252         blkcipher_walk_init(&walk, dst, src, nbytes);
253         err = blkcipher_walk_virt(desc, &walk);
254
255         ctx->ops->load_decrypt_keys(&ctx->key[0]);
256         key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
257         while ((nbytes = walk.nbytes)) {
258                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
259
260                 if (likely(block_len)) {
261                         ctx->ops->ecb_decrypt(key_end,
262                                               (const u64 *) walk.src.virt.addr,
263                                               (u64 *) walk.dst.virt.addr, block_len);
264                 }
265                 nbytes &= AES_BLOCK_SIZE - 1;
266                 err = blkcipher_walk_done(desc, &walk, nbytes);
267         }
268         fprs_write(0);
269
270         return err;
271 }
272
273 static int cbc_encrypt(struct blkcipher_desc *desc,
274                        struct scatterlist *dst, struct scatterlist *src,
275                        unsigned int nbytes)
276 {
277         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
278         struct blkcipher_walk walk;
279         int err;
280
281         blkcipher_walk_init(&walk, dst, src, nbytes);
282         err = blkcipher_walk_virt(desc, &walk);
283
284         ctx->ops->load_encrypt_keys(&ctx->key[0]);
285         while ((nbytes = walk.nbytes)) {
286                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
287
288                 if (likely(block_len)) {
289                         ctx->ops->cbc_encrypt(&ctx->key[0],
290                                               (const u64 *)walk.src.virt.addr,
291                                               (u64 *) walk.dst.virt.addr,
292                                               block_len, (u64 *) walk.iv);
293                 }
294                 nbytes &= AES_BLOCK_SIZE - 1;
295                 err = blkcipher_walk_done(desc, &walk, nbytes);
296         }
297         fprs_write(0);
298         return err;
299 }
300
301 static int cbc_decrypt(struct blkcipher_desc *desc,
302                        struct scatterlist *dst, struct scatterlist *src,
303                        unsigned int nbytes)
304 {
305         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
306         struct blkcipher_walk walk;
307         u64 *key_end;
308         int err;
309
310         blkcipher_walk_init(&walk, dst, src, nbytes);
311         err = blkcipher_walk_virt(desc, &walk);
312
313         ctx->ops->load_decrypt_keys(&ctx->key[0]);
314         key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
315         while ((nbytes = walk.nbytes)) {
316                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
317
318                 if (likely(block_len)) {
319                         ctx->ops->cbc_decrypt(key_end,
320                                               (const u64 *) walk.src.virt.addr,
321                                               (u64 *) walk.dst.virt.addr,
322                                               block_len, (u64 *) walk.iv);
323                 }
324                 nbytes &= AES_BLOCK_SIZE - 1;
325                 err = blkcipher_walk_done(desc, &walk, nbytes);
326         }
327         fprs_write(0);
328
329         return err;
330 }
331
332 static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx,
333                             struct blkcipher_walk *walk)
334 {
335         u8 *ctrblk = walk->iv;
336         u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
337         u8 *src = walk->src.virt.addr;
338         u8 *dst = walk->dst.virt.addr;
339         unsigned int nbytes = walk->nbytes;
340
341         ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
342                               keystream, AES_BLOCK_SIZE);
343         crypto_xor((u8 *) keystream, src, nbytes);
344         memcpy(dst, keystream, nbytes);
345         crypto_inc(ctrblk, AES_BLOCK_SIZE);
346 }
347
348 static int ctr_crypt(struct blkcipher_desc *desc,
349                      struct scatterlist *dst, struct scatterlist *src,
350                      unsigned int nbytes)
351 {
352         struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
353         struct blkcipher_walk walk;
354         int err;
355
356         blkcipher_walk_init(&walk, dst, src, nbytes);
357         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
358         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
359
360         ctx->ops->load_encrypt_keys(&ctx->key[0]);
361         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
362                 unsigned int block_len = nbytes & AES_BLOCK_MASK;
363
364                 if (likely(block_len)) {
365                         ctx->ops->ctr_crypt(&ctx->key[0],
366                                             (const u64 *)walk.src.virt.addr,
367                                             (u64 *) walk.dst.virt.addr,
368                                             block_len, (u64 *) walk.iv);
369                 }
370                 nbytes &= AES_BLOCK_SIZE - 1;
371                 err = blkcipher_walk_done(desc, &walk, nbytes);
372         }
373         if (walk.nbytes) {
374                 ctr_crypt_final(ctx, &walk);
375                 err = blkcipher_walk_done(desc, &walk, 0);
376         }
377         fprs_write(0);
378         return err;
379 }
380
381 static struct crypto_alg algs[] = { {
382         .cra_name               = "aes",
383         .cra_driver_name        = "aes-sparc64",
384         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
385         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
386         .cra_blocksize          = AES_BLOCK_SIZE,
387         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
388         .cra_alignmask          = 3,
389         .cra_module             = THIS_MODULE,
390         .cra_u  = {
391                 .cipher = {
392                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
393                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
394                         .cia_setkey             = aes_set_key,
395                         .cia_encrypt            = aes_encrypt,
396                         .cia_decrypt            = aes_decrypt
397                 }
398         }
399 }, {
400         .cra_name               = "ecb(aes)",
401         .cra_driver_name        = "ecb-aes-sparc64",
402         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
403         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
404         .cra_blocksize          = AES_BLOCK_SIZE,
405         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
406         .cra_alignmask          = 7,
407         .cra_type               = &crypto_blkcipher_type,
408         .cra_module             = THIS_MODULE,
409         .cra_u = {
410                 .blkcipher = {
411                         .min_keysize    = AES_MIN_KEY_SIZE,
412                         .max_keysize    = AES_MAX_KEY_SIZE,
413                         .setkey         = aes_set_key,
414                         .encrypt        = ecb_encrypt,
415                         .decrypt        = ecb_decrypt,
416                 },
417         },
418 }, {
419         .cra_name               = "cbc(aes)",
420         .cra_driver_name        = "cbc-aes-sparc64",
421         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
422         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
423         .cra_blocksize          = AES_BLOCK_SIZE,
424         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
425         .cra_alignmask          = 7,
426         .cra_type               = &crypto_blkcipher_type,
427         .cra_module             = THIS_MODULE,
428         .cra_u = {
429                 .blkcipher = {
430                         .min_keysize    = AES_MIN_KEY_SIZE,
431                         .max_keysize    = AES_MAX_KEY_SIZE,
432                         .setkey         = aes_set_key,
433                         .encrypt        = cbc_encrypt,
434                         .decrypt        = cbc_decrypt,
435                 },
436         },
437 }, {
438         .cra_name               = "ctr(aes)",
439         .cra_driver_name        = "ctr-aes-sparc64",
440         .cra_priority           = SPARC_CR_OPCODE_PRIORITY,
441         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
442         .cra_blocksize          = 1,
443         .cra_ctxsize            = sizeof(struct crypto_sparc64_aes_ctx),
444         .cra_alignmask          = 7,
445         .cra_type               = &crypto_blkcipher_type,
446         .cra_module             = THIS_MODULE,
447         .cra_u = {
448                 .blkcipher = {
449                         .min_keysize    = AES_MIN_KEY_SIZE,
450                         .max_keysize    = AES_MAX_KEY_SIZE,
451                         .setkey         = aes_set_key,
452                         .encrypt        = ctr_crypt,
453                         .decrypt        = ctr_crypt,
454                 },
455         },
456 } };
457
458 static bool __init sparc64_has_aes_opcode(void)
459 {
460         unsigned long cfr;
461
462         if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
463                 return false;
464
465         __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
466         if (!(cfr & CFR_AES))
467                 return false;
468
469         return true;
470 }
471
472 static int __init aes_sparc64_mod_init(void)
473 {
474         int i;
475
476         for (i = 0; i < ARRAY_SIZE(algs); i++)
477                 INIT_LIST_HEAD(&algs[i].cra_list);
478
479         if (sparc64_has_aes_opcode()) {
480                 pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
481                 return crypto_register_algs(algs, ARRAY_SIZE(algs));
482         }
483         pr_info("sparc64 aes opcodes not available.\n");
484         return -ENODEV;
485 }
486
487 static void __exit aes_sparc64_mod_fini(void)
488 {
489         crypto_unregister_algs(algs, ARRAY_SIZE(algs));
490 }
491
492 module_init(aes_sparc64_mod_init);
493 module_exit(aes_sparc64_mod_fini);
494
495 MODULE_LICENSE("GPL");
496 MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated");
497
498 MODULE_ALIAS("aes");
499
500 #include "crop_devid.c"