]> git.openfabrics.org - ~shefty/rdma-dev.git/commitdiff
Merge git://github.com/herbertx/crypto
authorLinus Torvalds <torvalds@linux-foundation.org>
Tue, 1 Nov 2011 16:24:41 +0000 (09:24 -0700)
committerLinus Torvalds <torvalds@linux-foundation.org>
Tue, 1 Nov 2011 16:24:41 +0000 (09:24 -0700)
* git://github.com/herbertx/crypto: (48 commits)
  crypto: user - Depend on NET instead of selecting it
  crypto: user - Add dependency on NET
  crypto: talitos - handle descriptor not found in error path
  crypto: user - Initialise match in crypto_alg_match
  crypto: testmgr - add twofish tests
  crypto: testmgr - add blowfish test-vectors
  crypto: Make hifn_795x build depend on !ARCH_DMA_ADDR_T_64BIT
  crypto: twofish-x86_64-3way - fix ctr blocksize to 1
  crypto: blowfish-x86_64 - fix ctr blocksize to 1
  crypto: whirlpool - count rounds from 0
  crypto: Add userspace report for compress type algorithms
  crypto: Add userspace report for cipher type algorithms
  crypto: Add userspace report for rng type algorithms
  crypto: Add userspace report for pcompress type algorithms
  crypto: Add userspace report for nivaead type algorithms
  crypto: Add userspace report for aead type algorithms
  crypto: Add userspace report for givcipher type algorithms
  crypto: Add userspace report for ablkcipher type algorithms
  crypto: Add userspace report for blkcipher type algorithms
  crypto: Add userspace report for ahash type algorithms
  ...

46 files changed:
Documentation/devicetree/bindings/crypto/picochip-spacc.txt [new file with mode: 0644]
arch/x86/crypto/Makefile
arch/x86/crypto/aes_glue.c
arch/x86/crypto/blowfish-x86_64-asm_64.S [new file with mode: 0644]
arch/x86/crypto/blowfish_glue.c [new file with mode: 0644]
arch/x86/crypto/sha1_ssse3_asm.S [new file with mode: 0644]
arch/x86/crypto/sha1_ssse3_glue.c [new file with mode: 0644]
arch/x86/crypto/twofish-i586-asm_32.S
arch/x86/crypto/twofish-x86_64-asm_64-3way.S [new file with mode: 0644]
arch/x86/crypto/twofish-x86_64-asm_64.S
arch/x86/crypto/twofish_glue.c
arch/x86/crypto/twofish_glue_3way.c [new file with mode: 0644]
arch/x86/include/asm/cpufeature.h
crypto/Kconfig
crypto/Makefile
crypto/ablkcipher.c
crypto/aead.c
crypto/ahash.c
crypto/algapi.c
crypto/blkcipher.c
crypto/blowfish.c [deleted file]
crypto/blowfish_common.c [new file with mode: 0644]
crypto/blowfish_generic.c [new file with mode: 0644]
crypto/cryptd.c
crypto/crypto_user.c [new file with mode: 0644]
crypto/internal.h
crypto/pcompress.c
crypto/rng.c
crypto/sha1_generic.c
crypto/shash.c
crypto/tcrypt.c
crypto/testmgr.c
crypto/testmgr.h
crypto/wp512.c
drivers/crypto/Kconfig
drivers/crypto/hifn_795x.c
drivers/crypto/n2_core.c
drivers/crypto/padlock-aes.c
drivers/crypto/picoxcell_crypto.c
drivers/crypto/talitos.c
include/crypto/algapi.h
include/crypto/blowfish.h [new file with mode: 0644]
include/crypto/sha.h
include/linux/crypto.h
include/linux/cryptouser.h [new file with mode: 0644]
include/linux/netlink.h

diff --git a/Documentation/devicetree/bindings/crypto/picochip-spacc.txt b/Documentation/devicetree/bindings/crypto/picochip-spacc.txt
new file mode 100644 (file)
index 0000000..d8609ec
--- /dev/null
@@ -0,0 +1,23 @@
+Picochip picoXcell SPAcc (Security Protocol Accelerator) bindings
+
+Picochip picoXcell devices contain crypto offload engines that may be used for
+IPSEC and femtocell layer 2 ciphering.
+
+Required properties:
+  - compatible : "picochip,spacc-ipsec" for the IPSEC offload engine
+    "picochip,spacc-l2" for the femtocell layer 2 ciphering engine.
+  - reg : Offset and length of the register set for this device
+  - interrupt-parent : The interrupt controller that controls the SPAcc
+    interrupt.
+  - interrupts : The interrupt line from the SPAcc.
+  - ref-clock : The input clock that drives the SPAcc.
+
+Example SPAcc node:
+
+spacc@10000 {
+       compatible = "picochip,spacc-ipsec";
+       reg = <0x100000 0x10000>;
+       interrupt-parent = <&vic0>;
+       interrupts = <24>;
+       ref-clock = <&ipsec_clk>, "ref";
+};
index c04f1b7a91397e6901ade2e09e498066b5697df2..3537d4b91f7407cd3e01c99ffd035a8beb2d629c 100644 (file)
@@ -7,21 +7,33 @@ obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o
 obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o
 
 obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
+obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
 obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
+obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o
 obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
 obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
 obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
 
 obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
+obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
 
 aes-i586-y := aes-i586-asm_32.o aes_glue.o
 twofish-i586-y := twofish-i586-asm_32.o twofish_glue.o
 salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o
 
 aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o
+blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
 twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
+twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o
 salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
 
 aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
 
 ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
+
+# enable AVX support only when $(AS) can actually assemble the instructions
+ifeq ($(call as-instr,vpxor %xmm0$(comma)%xmm1$(comma)%xmm2,yes,no),yes)
+AFLAGS_sha1_ssse3_asm.o += -DSHA1_ENABLE_AVX_SUPPORT
+CFLAGS_sha1_ssse3_glue.o += -DSHA1_ENABLE_AVX_SUPPORT
+endif
+sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
index 49ae9fe32b22079dc22c8d0bf121410285010d3d..b0b6950cc8c8850d68366b869ca39e4250827013 100644 (file)
@@ -4,6 +4,7 @@
  */
 
 #include <crypto/aes.h>
+#include <asm/aes.h>
 
 asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
 asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S
new file mode 100644 (file)
index 0000000..391d245
--- /dev/null
@@ -0,0 +1,390 @@
+/*
+ * Blowfish Cipher Algorithm (x86_64)
+ *
+ * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+.file "blowfish-x86_64-asm.S"
+.text
+
+/* structure of crypto context */
+#define p      0
+#define s0     ((16 + 2) * 4)
+#define s1     ((16 + 2 + (1 * 256)) * 4)
+#define s2     ((16 + 2 + (2 * 256)) * 4)
+#define s3     ((16 + 2 + (3 * 256)) * 4)
+
+/* register macros */
+#define CTX %rdi
+#define RIO %rsi
+
+#define RX0 %rax
+#define RX1 %rbx
+#define RX2 %rcx
+#define RX3 %rdx
+
+#define RX0d %eax
+#define RX1d %ebx
+#define RX2d %ecx
+#define RX3d %edx
+
+#define RX0bl %al
+#define RX1bl %bl
+#define RX2bl %cl
+#define RX3bl %dl
+
+#define RX0bh %ah
+#define RX1bh %bh
+#define RX2bh %ch
+#define RX3bh %dh
+
+#define RT0 %rbp
+#define RT1 %rsi
+#define RT2 %r8
+#define RT3 %r9
+
+#define RT0d %ebp
+#define RT1d %esi
+#define RT2d %r8d
+#define RT3d %r9d
+
+#define RKEY %r10
+
+/***********************************************************************
+ * 1-way blowfish
+ ***********************************************************************/
+#define F() \
+       rorq $16,               RX0; \
+       movzbl RX0bh,           RT0d; \
+       movzbl RX0bl,           RT1d; \
+       rolq $16,               RX0; \
+       movl s0(CTX,RT0,4),     RT0d; \
+       addl s1(CTX,RT1,4),     RT0d; \
+       movzbl RX0bh,           RT1d; \
+       movzbl RX0bl,           RT2d; \
+       rolq $32,               RX0; \
+       xorl s2(CTX,RT1,4),     RT0d; \
+       addl s3(CTX,RT2,4),     RT0d; \
+       xorq RT0,               RX0;
+
+#define add_roundkey_enc(n) \
+       xorq p+4*(n)(CTX),      RX0;
+
+#define round_enc(n) \
+       add_roundkey_enc(n); \
+       \
+       F(); \
+       F();
+
+#define add_roundkey_dec(n) \
+       movq p+4*(n-1)(CTX),    RT0; \
+       rorq $32,               RT0; \
+       xorq RT0,               RX0;
+
+#define round_dec(n) \
+       add_roundkey_dec(n); \
+       \
+       F(); \
+       F(); \
+
+#define read_block() \
+       movq (RIO),             RX0; \
+       rorq $32,               RX0; \
+       bswapq                  RX0;
+
+#define write_block() \
+       bswapq                  RX0; \
+       movq RX0,               (RIO);
+
+#define xor_block() \
+       bswapq                  RX0; \
+       xorq RX0,               (RIO);
+
+.align 8
+.global __blowfish_enc_blk
+.type   __blowfish_enc_blk,@function;
+
+__blowfish_enc_blk:
+       /* input:
+        *      %rdi: ctx, CTX
+        *      %rsi: dst
+        *      %rdx: src
+        *      %rcx: bool, if true: xor output
+        */
+       movq %rbp, %r11;
+
+       movq %rsi, %r10;
+       movq %rdx, RIO;
+
+       read_block();
+
+       round_enc(0);
+       round_enc(2);
+       round_enc(4);
+       round_enc(6);
+       round_enc(8);
+       round_enc(10);
+       round_enc(12);
+       round_enc(14);
+       add_roundkey_enc(16);
+
+       movq %r11, %rbp;
+
+       movq %r10, RIO;
+       test %cl, %cl;
+       jnz __enc_xor;
+
+       write_block();
+       ret;
+__enc_xor:
+       xor_block();
+       ret;
+
+.align 8
+.global blowfish_dec_blk
+.type   blowfish_dec_blk,@function;
+
+blowfish_dec_blk:
+       /* input:
+        *      %rdi: ctx, CTX
+        *      %rsi: dst
+        *      %rdx: src
+        */
+       movq %rbp, %r11;
+
+       movq %rsi, %r10;
+       movq %rdx, RIO;
+
+       read_block();
+
+       round_dec(17);
+       round_dec(15);
+       round_dec(13);
+       round_dec(11);
+       round_dec(9);
+       round_dec(7);
+       round_dec(5);
+       round_dec(3);
+       add_roundkey_dec(1);
+
+       movq %r10, RIO;
+       write_block();
+
+       movq %r11, %rbp;
+
+       ret;
+
+/**********************************************************************
+  4-way blowfish, four blocks parallel
+ **********************************************************************/
+
+/* F() for 4-way. Slower when used alone/1-way, but faster when used
+ * parallel/4-way (tested on AMD Phenom II & Intel Xeon E7330).
+ */
+#define F4(x) \
+       movzbl x ## bh,         RT1d; \
+       movzbl x ## bl,         RT3d; \
+       rorq $16,               x; \
+       movzbl x ## bh,         RT0d; \
+       movzbl x ## bl,         RT2d; \
+       rorq $16,               x; \
+       movl s0(CTX,RT0,4),     RT0d; \
+       addl s1(CTX,RT2,4),     RT0d; \
+       xorl s2(CTX,RT1,4),     RT0d; \
+       addl s3(CTX,RT3,4),     RT0d; \
+       xorq RT0,               x;
+
+#define add_preloaded_roundkey4() \
+       xorq RKEY,              RX0; \
+       xorq RKEY,              RX1; \
+       xorq RKEY,              RX2; \
+       xorq RKEY,              RX3;
+
+#define preload_roundkey_enc(n) \
+       movq p+4*(n)(CTX),      RKEY;
+
+#define add_roundkey_enc4(n) \
+       add_preloaded_roundkey4(); \
+       preload_roundkey_enc(n + 2);
+
+#define round_enc4(n) \
+       add_roundkey_enc4(n); \
+       \
+       F4(RX0); \
+       F4(RX1); \
+       F4(RX2); \
+       F4(RX3); \
+       \
+       F4(RX0); \
+       F4(RX1); \
+       F4(RX2); \
+       F4(RX3);
+
+#define preload_roundkey_dec(n) \
+       movq p+4*((n)-1)(CTX),  RKEY; \
+       rorq $32,               RKEY;
+
+#define add_roundkey_dec4(n) \
+       add_preloaded_roundkey4(); \
+       preload_roundkey_dec(n - 2);
+
+#define round_dec4(n) \
+       add_roundkey_dec4(n); \
+       \
+       F4(RX0); \
+       F4(RX1); \
+       F4(RX2); \
+       F4(RX3); \
+       \
+       F4(RX0); \
+       F4(RX1); \
+       F4(RX2); \
+       F4(RX3);
+
+#define read_block4() \
+       movq (RIO),             RX0; \
+       rorq $32,               RX0; \
+       bswapq                  RX0; \
+       \
+       movq 8(RIO),            RX1; \
+       rorq $32,               RX1; \
+       bswapq                  RX1; \
+       \
+       movq 16(RIO),           RX2; \
+       rorq $32,               RX2; \
+       bswapq                  RX2; \
+       \
+       movq 24(RIO),           RX3; \
+       rorq $32,               RX3; \
+       bswapq                  RX3;
+
+#define write_block4() \
+       bswapq                  RX0; \
+       movq RX0,               (RIO); \
+       \
+       bswapq                  RX1; \
+       movq RX1,               8(RIO); \
+       \
+       bswapq                  RX2; \
+       movq RX2,               16(RIO); \
+       \
+       bswapq                  RX3; \
+       movq RX3,               24(RIO);
+
+#define xor_block4() \
+       bswapq                  RX0; \
+       xorq RX0,               (RIO); \
+       \
+       bswapq                  RX1; \
+       xorq RX1,               8(RIO); \
+       \
+       bswapq                  RX2; \
+       xorq RX2,               16(RIO); \
+       \
+       bswapq                  RX3; \
+       xorq RX3,               24(RIO);
+
+.align 8
+.global __blowfish_enc_blk_4way
+.type   __blowfish_enc_blk_4way,@function;
+
+__blowfish_enc_blk_4way:
+       /* input:
+        *      %rdi: ctx, CTX
+        *      %rsi: dst
+        *      %rdx: src
+        *      %rcx: bool, if true: xor output
+        */
+       pushq %rbp;
+       pushq %rbx;
+       pushq %rcx;
+
+       preload_roundkey_enc(0);
+
+       movq %rsi, %r11;
+       movq %rdx, RIO;
+
+       read_block4();
+
+       round_enc4(0);
+       round_enc4(2);
+       round_enc4(4);
+       round_enc4(6);
+       round_enc4(8);
+       round_enc4(10);
+       round_enc4(12);
+       round_enc4(14);
+       add_preloaded_roundkey4();
+
+       popq %rbp;
+       movq %r11, RIO;
+
+       test %bpl, %bpl;
+       jnz __enc_xor4;
+
+       write_block4();
+
+       popq %rbx;
+       popq %rbp;
+       ret;
+
+__enc_xor4:
+       xor_block4();
+
+       popq %rbx;
+       popq %rbp;
+       ret;
+
+.align 8
+.global blowfish_dec_blk_4way
+.type   blowfish_dec_blk_4way,@function;
+
+blowfish_dec_blk_4way:
+       /* input:
+        *      %rdi: ctx, CTX
+        *      %rsi: dst
+        *      %rdx: src
+        */
+       pushq %rbp;
+       pushq %rbx;
+       preload_roundkey_dec(17);
+
+       movq %rsi, %r11;
+       movq %rdx, RIO;
+
+       read_block4();
+
+       round_dec4(17);
+       round_dec4(15);
+       round_dec4(13);
+       round_dec4(11);
+       round_dec4(9);
+       round_dec4(7);
+       round_dec4(5);
+       round_dec4(3);
+       add_preloaded_roundkey4();
+
+       movq %r11, RIO;
+       write_block4();
+
+       popq %rbx;
+       popq %rbp;
+
+       ret;
+
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
new file mode 100644 (file)
index 0000000..b05aa16
--- /dev/null
@@ -0,0 +1,492 @@
+/*
+ * Glue Code for assembler optimized version of Blowfish
+ *
+ * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
+ *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
+ * CTR part based on code (crypto/ctr.c) by:
+ *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+#include <crypto/blowfish.h>
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <crypto/algapi.h>
+
+/* regular block cipher functions */
+asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
+                                  bool xor);
+asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
+
+/* 4-way parallel cipher functions */
+asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
+                                       const u8 *src, bool xor);
+asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
+                                     const u8 *src);
+
+static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
+{
+       __blowfish_enc_blk(ctx, dst, src, false);
+}
+
+static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
+                                       const u8 *src)
+{
+       __blowfish_enc_blk(ctx, dst, src, true);
+}
+
+static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
+                                        const u8 *src)
+{
+       __blowfish_enc_blk_4way(ctx, dst, src, false);
+}
+
+static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
+                                     const u8 *src)
+{
+       __blowfish_enc_blk_4way(ctx, dst, src, true);
+}
+
+static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+       blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
+}
+
+static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+       blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
+}
+
+static struct crypto_alg bf_alg = {
+       .cra_name               =       "blowfish",
+       .cra_driver_name        =       "blowfish-asm",
+       .cra_priority           =       200,
+       .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
+       .cra_blocksize          =       BF_BLOCK_SIZE,
+       .cra_ctxsize            =       sizeof(struct bf_ctx),
+       .cra_alignmask          =       3,
+       .cra_module             =       THIS_MODULE,
+       .cra_list               =       LIST_HEAD_INIT(bf_alg.cra_list),
+       .cra_u                  =       {
+               .cipher = {
+                       .cia_min_keysize        =       BF_MIN_KEY_SIZE,
+                       .cia_max_keysize        =       BF_MAX_KEY_SIZE,
+                       .cia_setkey             =       blowfish_setkey,
+                       .cia_encrypt            =       blowfish_encrypt,
+                       .cia_decrypt            =       blowfish_decrypt,
+               }
+       }
+};
+
+static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
+                    void (*fn)(struct bf_ctx *, u8 *, const u8 *),
+                    void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
+{
+       struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = BF_BLOCK_SIZE;
+       unsigned int nbytes;
+       int err;
+
+       err = blkcipher_walk_virt(desc, walk);
+
+       while ((nbytes = walk->nbytes)) {
+               u8 *wsrc = walk->src.virt.addr;
+               u8 *wdst = walk->dst.virt.addr;
+
+               /* Process four block batch */
+               if (nbytes >= bsize * 4) {
+                       do {
+                               fn_4way(ctx, wdst, wsrc);
+
+                               wsrc += bsize * 4;
+                               wdst += bsize * 4;
+                               nbytes -= bsize * 4;
+                       } while (nbytes >= bsize * 4);
+
+                       if (nbytes < bsize)
+                               goto done;
+               }
+
+               /* Handle leftovers */
+               do {
+                       fn(ctx, wdst, wsrc);
+
+                       wsrc += bsize;
+                       wdst += bsize;
+                       nbytes -= bsize;
+               } while (nbytes >= bsize);
+
+done:
+               err = blkcipher_walk_done(desc, walk, nbytes);
+       }
+
+       return err;
+}
+
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);
+}
+
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);
+}
+
+static struct crypto_alg blk_ecb_alg = {
+       .cra_name               = "ecb(blowfish)",
+       .cra_driver_name        = "ecb-blowfish-asm",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+       .cra_blocksize          = BF_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct bf_ctx),
+       .cra_alignmask          = 0,
+       .cra_type               = &crypto_blkcipher_type,
+       .cra_module             = THIS_MODULE,
+       .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
+       .cra_u = {
+               .blkcipher = {
+                       .min_keysize    = BF_MIN_KEY_SIZE,
+                       .max_keysize    = BF_MAX_KEY_SIZE,
+                       .setkey         = blowfish_setkey,
+                       .encrypt        = ecb_encrypt,
+                       .decrypt        = ecb_decrypt,
+               },
+       },
+};
+
+static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
+                                 struct blkcipher_walk *walk)
+{
+       struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = BF_BLOCK_SIZE;
+       unsigned int nbytes = walk->nbytes;
+       u64 *src = (u64 *)walk->src.virt.addr;
+       u64 *dst = (u64 *)walk->dst.virt.addr;
+       u64 *iv = (u64 *)walk->iv;
+
+       do {
+               *dst = *src ^ *iv;
+               blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
+               iv = dst;
+
+               src += 1;
+               dst += 1;
+               nbytes -= bsize;
+       } while (nbytes >= bsize);
+
+       *(u64 *)walk->iv = *iv;
+       return nbytes;
+}
+
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+       int err;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       err = blkcipher_walk_virt(desc, &walk);
+
+       while ((nbytes = walk.nbytes)) {
+               nbytes = __cbc_encrypt(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, nbytes);
+       }
+
+       return err;
+}
+
+static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
+                                 struct blkcipher_walk *walk)
+{
+       struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = BF_BLOCK_SIZE;
+       unsigned int nbytes = walk->nbytes;
+       u64 *src = (u64 *)walk->src.virt.addr;
+       u64 *dst = (u64 *)walk->dst.virt.addr;
+       u64 ivs[4 - 1];
+       u64 last_iv;
+
+       /* Start of the last block. */
+       src += nbytes / bsize - 1;
+       dst += nbytes / bsize - 1;
+
+       last_iv = *src;
+
+       /* Process four block batch */
+       if (nbytes >= bsize * 4) {
+               do {
+                       nbytes -= bsize * 4 - bsize;
+                       src -= 4 - 1;
+                       dst -= 4 - 1;
+
+                       ivs[0] = src[0];
+                       ivs[1] = src[1];
+                       ivs[2] = src[2];
+
+                       blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
+
+                       dst[1] ^= ivs[0];
+                       dst[2] ^= ivs[1];
+                       dst[3] ^= ivs[2];
+
+                       nbytes -= bsize;
+                       if (nbytes < bsize)
+                               goto done;
+
+                       *dst ^= *(src - 1);
+                       src -= 1;
+                       dst -= 1;
+               } while (nbytes >= bsize * 4);
+
+               if (nbytes < bsize)
+                       goto done;
+       }
+
+       /* Handle leftovers */
+       for (;;) {
+               blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
+
+               nbytes -= bsize;
+               if (nbytes < bsize)
+                       break;
+
+               *dst ^= *(src - 1);
+               src -= 1;
+               dst -= 1;
+       }
+
+done:
+       *dst ^= *(u64 *)walk->iv;
+       *(u64 *)walk->iv = last_iv;
+
+       return nbytes;
+}
+
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+       int err;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       err = blkcipher_walk_virt(desc, &walk);
+
+       while ((nbytes = walk.nbytes)) {
+               nbytes = __cbc_decrypt(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, nbytes);
+       }
+
+       return err;
+}
+
+static struct crypto_alg blk_cbc_alg = {
+       .cra_name               = "cbc(blowfish)",
+       .cra_driver_name        = "cbc-blowfish-asm",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+       .cra_blocksize          = BF_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct bf_ctx),
+       .cra_alignmask          = 0,
+       .cra_type               = &crypto_blkcipher_type,
+       .cra_module             = THIS_MODULE,
+       .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
+       .cra_u = {
+               .blkcipher = {
+                       .min_keysize    = BF_MIN_KEY_SIZE,
+                       .max_keysize    = BF_MAX_KEY_SIZE,
+                       .ivsize         = BF_BLOCK_SIZE,
+                       .setkey         = blowfish_setkey,
+                       .encrypt        = cbc_encrypt,
+                       .decrypt        = cbc_decrypt,
+               },
+       },
+};
+
+static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
+{
+       u8 *ctrblk = walk->iv;
+       u8 keystream[BF_BLOCK_SIZE];
+       u8 *src = walk->src.virt.addr;
+       u8 *dst = walk->dst.virt.addr;
+       unsigned int nbytes = walk->nbytes;
+
+       blowfish_enc_blk(ctx, keystream, ctrblk);
+       crypto_xor(keystream, src, nbytes);
+       memcpy(dst, keystream, nbytes);
+
+       crypto_inc(ctrblk, BF_BLOCK_SIZE);
+}
+
+static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
+                               struct blkcipher_walk *walk)
+{
+       struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = BF_BLOCK_SIZE;
+       unsigned int nbytes = walk->nbytes;
+       u64 *src = (u64 *)walk->src.virt.addr;
+       u64 *dst = (u64 *)walk->dst.virt.addr;
+       u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
+       __be64 ctrblocks[4];
+
+       /* Process four block batch */
+       if (nbytes >= bsize * 4) {
+               do {
+                       if (dst != src) {
+                               dst[0] = src[0];
+                               dst[1] = src[1];
+                               dst[2] = src[2];
+                               dst[3] = src[3];
+                       }
+
+                       /* create ctrblks for parallel encrypt */
+                       ctrblocks[0] = cpu_to_be64(ctrblk++);
+                       ctrblocks[1] = cpu_to_be64(ctrblk++);
+                       ctrblocks[2] = cpu_to_be64(ctrblk++);
+                       ctrblocks[3] = cpu_to_be64(ctrblk++);
+
+                       blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
+                                                 (u8 *)ctrblocks);
+
+                       src += 4;
+                       dst += 4;
+               } while ((nbytes -= bsize * 4) >= bsize * 4);
+
+               if (nbytes < bsize)
+                       goto done;
+       }
+
+       /* Handle leftovers */
+       do {
+               if (dst != src)
+                       *dst = *src;
+
+               ctrblocks[0] = cpu_to_be64(ctrblk++);
+
+               blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
+
+               src += 1;
+               dst += 1;
+       } while ((nbytes -= bsize) >= bsize);
+
+done:
+       *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
+       return nbytes;
+}
+
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                    struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+       int err;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);
+
+       while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
+               nbytes = __ctr_crypt(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, nbytes);
+       }
+
+       if (walk.nbytes) {
+               ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);
+               err = blkcipher_walk_done(desc, &walk, 0);
+       }
+
+       return err;
+}
+
+static struct crypto_alg blk_ctr_alg = {
+       .cra_name               = "ctr(blowfish)",
+       .cra_driver_name        = "ctr-blowfish-asm",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+       .cra_blocksize          = 1,
+       .cra_ctxsize            = sizeof(struct bf_ctx),
+       .cra_alignmask          = 0,
+       .cra_type               = &crypto_blkcipher_type,
+       .cra_module             = THIS_MODULE,
+       .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
+       .cra_u = {
+               .blkcipher = {
+                       .min_keysize    = BF_MIN_KEY_SIZE,
+                       .max_keysize    = BF_MAX_KEY_SIZE,
+                       .ivsize         = BF_BLOCK_SIZE,
+                       .setkey         = blowfish_setkey,
+                       .encrypt        = ctr_crypt,
+                       .decrypt        = ctr_crypt,
+               },
+       },
+};
+
+static int __init init(void)
+{
+       int err;
+
+       err = crypto_register_alg(&bf_alg);
+       if (err)
+               goto bf_err;
+       err = crypto_register_alg(&blk_ecb_alg);
+       if (err)
+               goto ecb_err;
+       err = crypto_register_alg(&blk_cbc_alg);
+       if (err)
+               goto cbc_err;
+       err = crypto_register_alg(&blk_ctr_alg);
+       if (err)
+               goto ctr_err;
+
+       return 0;
+
+ctr_err:
+       crypto_unregister_alg(&blk_cbc_alg);
+cbc_err:
+       crypto_unregister_alg(&blk_ecb_alg);
+ecb_err:
+       crypto_unregister_alg(&bf_alg);
+bf_err:
+       return err;
+}
+
+static void __exit fini(void)
+{
+       crypto_unregister_alg(&blk_ctr_alg);
+       crypto_unregister_alg(&blk_cbc_alg);
+       crypto_unregister_alg(&blk_ecb_alg);
+       crypto_unregister_alg(&bf_alg);
+}
+
+module_init(init);
+module_exit(fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
+MODULE_ALIAS("blowfish");
+MODULE_ALIAS("blowfish-asm");
diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
new file mode 100644 (file)
index 0000000..b2c2f57
--- /dev/null
@@ -0,0 +1,558 @@
+/*
+ * This is a SIMD SHA-1 implementation. It requires the Intel(R) Supplemental
+ * SSE3 instruction set extensions introduced in Intel Core Microarchitecture
+ * processors. CPUs supporting Intel(R) AVX extensions will get an additional
+ * boost.
+ *
+ * This work was inspired by the vectorized implementation of Dean Gaudet.
+ * Additional information on it can be found at:
+ *    http://www.arctic.org/~dean/crypto/sha1.html
+ *
+ * It was improved upon with more efficient vectorization of the message
+ * scheduling. This implementation has also been optimized for all current and
+ * several future generations of Intel CPUs.
+ *
+ * See this article for more information about the implementation details:
+ *   http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/
+ *
+ * Copyright (C) 2010, Intel Corp.
+ *   Authors: Maxim Locktyukhin <maxim.locktyukhin@intel.com>
+ *            Ronen Zohar <ronen.zohar@intel.com>
+ *
+ * Converted to AT&T syntax and adapted for inclusion in the Linux kernel:
+ *   Author: Mathias Krause <minipli@googlemail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#define CTX    %rdi    // arg1
+#define BUF    %rsi    // arg2
+#define CNT    %rdx    // arg3
+
+#define REG_A  %ecx
+#define REG_B  %esi
+#define REG_C  %edi
+#define REG_D  %ebp
+#define REG_E  %edx
+
+#define REG_T1 %eax
+#define REG_T2 %ebx
+
+#define K_BASE         %r8
+#define HASH_PTR       %r9
+#define BUFFER_PTR     %r10
+#define BUFFER_END     %r11
+
+#define W_TMP1 %xmm0
+#define W_TMP2 %xmm9
+
+#define W0     %xmm1
+#define W4     %xmm2
+#define W8     %xmm3
+#define W12    %xmm4
+#define W16    %xmm5
+#define W20    %xmm6
+#define W24    %xmm7
+#define W28    %xmm8
+
+#define XMM_SHUFB_BSWAP        %xmm10
+
+/* we keep window of 64 w[i]+K pre-calculated values in a circular buffer */
+#define WK(t)  (((t) & 15) * 4)(%rsp)
+#define W_PRECALC_AHEAD        16
+
+/*
+ * This macro implements the SHA-1 function's body for single 64-byte block
+ * param: function's name
+ */
+.macro SHA1_VECTOR_ASM  name
+       .global \name
+       .type   \name, @function
+       .align 32
+\name:
+       push    %rbx
+       push    %rbp
+       push    %r12
+
+       mov     %rsp, %r12
+       sub     $64, %rsp               # allocate workspace
+       and     $~15, %rsp              # align stack
+
+       mov     CTX, HASH_PTR
+       mov     BUF, BUFFER_PTR
+
+       shl     $6, CNT                 # multiply by 64
+       add     BUF, CNT
+       mov     CNT, BUFFER_END
+
+       lea     K_XMM_AR(%rip), K_BASE
+       xmm_mov BSWAP_SHUFB_CTL(%rip), XMM_SHUFB_BSWAP
+
+       SHA1_PIPELINED_MAIN_BODY
+
+       # cleanup workspace
+       mov     $8, %ecx
+       mov     %rsp, %rdi
+       xor     %rax, %rax
+       rep stosq
+
+       mov     %r12, %rsp              # deallocate workspace
+
+       pop     %r12
+       pop     %rbp
+       pop     %rbx
+       ret
+
+       .size   \name, .-\name
+.endm
+
+/*
+ * This macro implements 80 rounds of SHA-1 for one 64-byte block
+ */
+.macro SHA1_PIPELINED_MAIN_BODY
+       INIT_REGALLOC
+
+       mov       (HASH_PTR), A
+       mov      4(HASH_PTR), B
+       mov      8(HASH_PTR), C
+       mov     12(HASH_PTR), D
+       mov     16(HASH_PTR), E
+
+  .set i, 0
+  .rept W_PRECALC_AHEAD
+       W_PRECALC i
+    .set i, (i+1)
+  .endr
+
+.align 4
+1:
+       RR F1,A,B,C,D,E,0
+       RR F1,D,E,A,B,C,2
+       RR F1,B,C,D,E,A,4
+       RR F1,E,A,B,C,D,6
+       RR F1,C,D,E,A,B,8
+
+       RR F1,A,B,C,D,E,10
+       RR F1,D,E,A,B,C,12
+       RR F1,B,C,D,E,A,14
+       RR F1,E,A,B,C,D,16
+       RR F1,C,D,E,A,B,18
+
+       RR F2,A,B,C,D,E,20
+       RR F2,D,E,A,B,C,22
+       RR F2,B,C,D,E,A,24
+       RR F2,E,A,B,C,D,26
+       RR F2,C,D,E,A,B,28
+
+       RR F2,A,B,C,D,E,30
+       RR F2,D,E,A,B,C,32
+       RR F2,B,C,D,E,A,34
+       RR F2,E,A,B,C,D,36
+       RR F2,C,D,E,A,B,38
+
+       RR F3,A,B,C,D,E,40
+       RR F3,D,E,A,B,C,42
+       RR F3,B,C,D,E,A,44
+       RR F3,E,A,B,C,D,46
+       RR F3,C,D,E,A,B,48
+
+       RR F3,A,B,C,D,E,50
+       RR F3,D,E,A,B,C,52
+       RR F3,B,C,D,E,A,54
+       RR F3,E,A,B,C,D,56
+       RR F3,C,D,E,A,B,58
+
+       add     $64, BUFFER_PTR         # move to the next 64-byte block
+       cmp     BUFFER_END, BUFFER_PTR  # if the current is the last one use
+       cmovae  K_BASE, BUFFER_PTR      # dummy source to avoid buffer overrun
+
+       RR F4,A,B,C,D,E,60
+       RR F4,D,E,A,B,C,62
+       RR F4,B,C,D,E,A,64
+       RR F4,E,A,B,C,D,66
+       RR F4,C,D,E,A,B,68
+
+       RR F4,A,B,C,D,E,70
+       RR F4,D,E,A,B,C,72
+       RR F4,B,C,D,E,A,74
+       RR F4,E,A,B,C,D,76
+       RR F4,C,D,E,A,B,78
+
+       UPDATE_HASH   (HASH_PTR), A
+       UPDATE_HASH  4(HASH_PTR), B
+       UPDATE_HASH  8(HASH_PTR), C
+       UPDATE_HASH 12(HASH_PTR), D
+       UPDATE_HASH 16(HASH_PTR), E
+
+       RESTORE_RENAMED_REGS
+       cmp     K_BASE, BUFFER_PTR      # K_BASE means, we reached the end
+       jne     1b
+.endm
+
+.macro INIT_REGALLOC
+  .set A, REG_A
+  .set B, REG_B
+  .set C, REG_C
+  .set D, REG_D
+  .set E, REG_E
+  .set T1, REG_T1
+  .set T2, REG_T2
+.endm
+
+.macro RESTORE_RENAMED_REGS
+       # order is important (REG_C is where it should be)
+       mov     B, REG_B
+       mov     D, REG_D
+       mov     A, REG_A
+       mov     E, REG_E
+.endm
+
+.macro SWAP_REG_NAMES  a, b
+  .set _T, \a
+  .set \a, \b
+  .set \b, _T
+.endm
+
+.macro F1  b, c, d
+       mov     \c, T1
+       SWAP_REG_NAMES \c, T1
+       xor     \d, T1
+       and     \b, T1
+       xor     \d, T1
+.endm
+
+.macro F2  b, c, d
+       mov     \d, T1
+       SWAP_REG_NAMES \d, T1
+       xor     \c, T1
+       xor     \b, T1
+.endm
+
+.macro F3  b, c ,d
+       mov     \c, T1
+       SWAP_REG_NAMES \c, T1
+       mov     \b, T2
+       or      \b, T1
+       and     \c, T2
+       and     \d, T1
+       or      T2, T1
+.endm
+
+.macro F4  b, c, d
+       F2 \b, \c, \d
+.endm
+
+.macro UPDATE_HASH  hash, val
+       add     \hash, \val
+       mov     \val, \hash
+.endm
+
+/*
+ * RR does two rounds of SHA-1 back to back with W[] pre-calc
+ *   t1 = F(b, c, d);   e += w(i)
+ *   e += t1;           b <<= 30;   d  += w(i+1);
+ *   t1 = F(a, b, c);
+ *   d += t1;           a <<= 5;
+ *   e += a;
+ *   t1 = e;            a >>= 7;
+ *   t1 <<= 5;
+ *   d += t1;
+ */
+.macro RR  F, a, b, c, d, e, round
+       add     WK(\round), \e
+       \F   \b, \c, \d         # t1 = F(b, c, d);
+       W_PRECALC (\round + W_PRECALC_AHEAD)
+       rol     $30, \b
+       add     T1, \e
+       add     WK(\round + 1), \d
+
+       \F   \a, \b, \c
+       W_PRECALC (\round + W_PRECALC_AHEAD + 1)
+       rol     $5, \a
+       add     \a, \e
+       add     T1, \d
+       ror     $7, \a          # (a <<r 5) >>r 7) => a <<r 30)
+
+       mov     \e, T1
+       SWAP_REG_NAMES \e, T1
+
+       rol     $5, T1
+       add     T1, \d
+
+       # write:  \a, \b
+       # rotate: \a<=\d, \b<=\e, \c<=\a, \d<=\b, \e<=\c
+.endm
+
+.macro W_PRECALC  r
+  .set i, \r
+
+  .if (i < 20)
+    .set K_XMM, 0
+  .elseif (i < 40)
+    .set K_XMM, 16
+  .elseif (i < 60)
+    .set K_XMM, 32
+  .elseif (i < 80)
+    .set K_XMM, 48
+  .endif
+
+  .if ((i < 16) || ((i >= 80) && (i < (80 + W_PRECALC_AHEAD))))
+    .set i, ((\r) % 80)            # pre-compute for the next iteration
+    .if (i == 0)
+       W_PRECALC_RESET
+    .endif
+       W_PRECALC_00_15
+  .elseif (i<32)
+       W_PRECALC_16_31
+  .elseif (i < 80)   // rounds 32-79
+       W_PRECALC_32_79
+  .endif
+.endm
+
+.macro W_PRECALC_RESET
+  .set W,          W0
+  .set W_minus_04, W4
+  .set W_minus_08, W8
+  .set W_minus_12, W12
+  .set W_minus_16, W16
+  .set W_minus_20, W20
+  .set W_minus_24, W24
+  .set W_minus_28, W28
+  .set W_minus_32, W
+.endm
+
+.macro W_PRECALC_ROTATE
+  .set W_minus_32, W_minus_28
+  .set W_minus_28, W_minus_24
+  .set W_minus_24, W_minus_20
+  .set W_minus_20, W_minus_16
+  .set W_minus_16, W_minus_12
+  .set W_minus_12, W_minus_08
+  .set W_minus_08, W_minus_04
+  .set W_minus_04, W
+  .set W,          W_minus_32
+.endm
+
+.macro W_PRECALC_SSSE3
+
+.macro W_PRECALC_00_15
+       W_PRECALC_00_15_SSSE3
+.endm
+.macro W_PRECALC_16_31
+       W_PRECALC_16_31_SSSE3
+.endm
+.macro W_PRECALC_32_79
+       W_PRECALC_32_79_SSSE3
+.endm
+
+/* message scheduling pre-compute for rounds 0-15 */
+.macro W_PRECALC_00_15_SSSE3
+  .if ((i & 3) == 0)
+       movdqu  (i*4)(BUFFER_PTR), W_TMP1
+  .elseif ((i & 3) == 1)
+       pshufb  XMM_SHUFB_BSWAP, W_TMP1
+       movdqa  W_TMP1, W
+  .elseif ((i & 3) == 2)
+       paddd   (K_BASE), W_TMP1
+  .elseif ((i & 3) == 3)
+       movdqa  W_TMP1, WK(i&~3)
+       W_PRECALC_ROTATE
+  .endif
+.endm
+
+/* message scheduling pre-compute for rounds 16-31
+ *
+ * - calculating last 32 w[i] values in 8 XMM registers
+ * - pre-calculate K+w[i] values and store to mem, for later load by ALU add
+ *   instruction
+ *
+ * some "heavy-lifting" vectorization for rounds 16-31 due to w[i]->w[i-3]
+ * dependency, but improves for 32-79
+ */
+.macro W_PRECALC_16_31_SSSE3
+  # blended scheduling of vector and scalar instruction streams, one 4-wide
+  # vector iteration / 4 scalar rounds
+  .if ((i & 3) == 0)
+       movdqa  W_minus_12, W
+       palignr $8, W_minus_16, W       # w[i-14]
+       movdqa  W_minus_04, W_TMP1
+       psrldq  $4, W_TMP1              # w[i-3]
+       pxor    W_minus_08, W
+  .elseif ((i & 3) == 1)
+       pxor    W_minus_16, W_TMP1
+       pxor    W_TMP1, W
+       movdqa  W, W_TMP2
+       movdqa  W, W_TMP1
+       pslldq  $12, W_TMP2
+  .elseif ((i & 3) == 2)
+       psrld   $31, W
+       pslld   $1, W_TMP1
+       por     W, W_TMP1
+       movdqa  W_TMP2, W
+       psrld   $30, W_TMP2
+       pslld   $2, W
+  .elseif ((i & 3) == 3)
+       pxor    W, W_TMP1
+       pxor    W_TMP2, W_TMP1
+       movdqa  W_TMP1, W
+       paddd   K_XMM(K_BASE), W_TMP1
+       movdqa  W_TMP1, WK(i&~3)
+       W_PRECALC_ROTATE
+  .endif
+.endm
+
+/* message scheduling pre-compute for rounds 32-79
+ *
+ * in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
+ * instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
+ * allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
+ */
+.macro W_PRECALC_32_79_SSSE3
+  .if ((i & 3) == 0)
+       movdqa  W_minus_04, W_TMP1
+       pxor    W_minus_28, W           # W is W_minus_32 before xor
+       palignr $8, W_minus_08, W_TMP1
+  .elseif ((i & 3) == 1)
+       pxor    W_minus_16, W
+       pxor    W_TMP1, W
+       movdqa  W, W_TMP1
+  .elseif ((i & 3) == 2)
+       psrld   $30, W
+       pslld   $2, W_TMP1
+       por     W, W_TMP1
+  .elseif ((i & 3) == 3)
+       movdqa  W_TMP1, W
+       paddd   K_XMM(K_BASE), W_TMP1
+       movdqa  W_TMP1, WK(i&~3)
+       W_PRECALC_ROTATE
+  .endif
+.endm
+
+.endm          // W_PRECALC_SSSE3
+
+
+#define K1     0x5a827999
+#define K2     0x6ed9eba1
+#define K3     0x8f1bbcdc
+#define K4     0xca62c1d6
+
+.section .rodata
+.align 16
+
+K_XMM_AR:
+       .long K1, K1, K1, K1
+       .long K2, K2, K2, K2
+       .long K3, K3, K3, K3
+       .long K4, K4, K4, K4
+
+BSWAP_SHUFB_CTL:
+       .long 0x00010203
+       .long 0x04050607
+       .long 0x08090a0b
+       .long 0x0c0d0e0f
+
+
+.section .text
+
+W_PRECALC_SSSE3
+.macro xmm_mov a, b
+       movdqu  \a,\b
+.endm
+
+/* SSSE3 optimized implementation:
+ *  extern "C" void sha1_transform_ssse3(u32 *digest, const char *data, u32 *ws,
+ *                                       unsigned int rounds);
+ */
+SHA1_VECTOR_ASM     sha1_transform_ssse3
+
+#ifdef SHA1_ENABLE_AVX_SUPPORT
+
+.macro W_PRECALC_AVX
+
+.purgem W_PRECALC_00_15
+.macro  W_PRECALC_00_15
+    W_PRECALC_00_15_AVX
+.endm
+.purgem W_PRECALC_16_31
+.macro  W_PRECALC_16_31
+    W_PRECALC_16_31_AVX
+.endm
+.purgem W_PRECALC_32_79
+.macro  W_PRECALC_32_79
+    W_PRECALC_32_79_AVX
+.endm
+
+.macro W_PRECALC_00_15_AVX
+  .if ((i & 3) == 0)
+       vmovdqu (i*4)(BUFFER_PTR), W_TMP1
+  .elseif ((i & 3) == 1)
+       vpshufb XMM_SHUFB_BSWAP, W_TMP1, W
+  .elseif ((i & 3) == 2)
+       vpaddd  (K_BASE), W, W_TMP1
+  .elseif ((i & 3) == 3)
+       vmovdqa W_TMP1, WK(i&~3)
+       W_PRECALC_ROTATE
+  .endif
+.endm
+
+.macro W_PRECALC_16_31_AVX
+  .if ((i & 3) == 0)
+       vpalignr $8, W_minus_16, W_minus_12, W  # w[i-14]
+       vpsrldq $4, W_minus_04, W_TMP1          # w[i-3]
+       vpxor   W_minus_08, W, W
+       vpxor   W_minus_16, W_TMP1, W_TMP1
+  .elseif ((i & 3) == 1)
+       vpxor   W_TMP1, W, W
+       vpslldq $12, W, W_TMP2
+       vpslld  $1, W, W_TMP1
+  .elseif ((i & 3) == 2)
+       vpsrld  $31, W, W
+       vpor    W, W_TMP1, W_TMP1
+       vpslld  $2, W_TMP2, W
+       vpsrld  $30, W_TMP2, W_TMP2
+  .elseif ((i & 3) == 3)
+       vpxor   W, W_TMP1, W_TMP1
+       vpxor   W_TMP2, W_TMP1, W
+       vpaddd  K_XMM(K_BASE), W, W_TMP1
+       vmovdqu W_TMP1, WK(i&~3)
+       W_PRECALC_ROTATE
+  .endif
+.endm
+
+.macro W_PRECALC_32_79_AVX
+  .if ((i & 3) == 0)
+       vpalignr $8, W_minus_08, W_minus_04, W_TMP1
+       vpxor   W_minus_28, W, W                # W is W_minus_32 before xor
+  .elseif ((i & 3) == 1)
+       vpxor   W_minus_16, W_TMP1, W_TMP1
+       vpxor   W_TMP1, W, W
+  .elseif ((i & 3) == 2)
+       vpslld  $2, W, W_TMP1
+       vpsrld  $30, W, W
+       vpor    W, W_TMP1, W
+  .elseif ((i & 3) == 3)
+       vpaddd  K_XMM(K_BASE), W, W_TMP1
+       vmovdqu W_TMP1, WK(i&~3)
+       W_PRECALC_ROTATE
+  .endif
+.endm
+
+.endm    // W_PRECALC_AVX
+
+W_PRECALC_AVX
+.purgem xmm_mov
+.macro xmm_mov a, b
+       vmovdqu \a,\b
+.endm
+
+
+/* AVX optimized implementation:
+ *  extern "C" void sha1_transform_avx(u32 *digest, const char *data, u32 *ws,
+ *                                     unsigned int rounds);
+ */
+SHA1_VECTOR_ASM     sha1_transform_avx
+
+#endif
diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c
new file mode 100644 (file)
index 0000000..f916499
--- /dev/null
@@ -0,0 +1,240 @@
+/*
+ * Cryptographic API.
+ *
+ * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
+ * Supplemental SSE3 instructions.
+ *
+ * This file is based on sha1_generic.c
+ *
+ * Copyright (c) Alan Smithee.
+ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
+ * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
+ * Copyright (c) Mathias Krause <minipli@googlemail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#define pr_fmt(fmt)    KBUILD_MODNAME ": " fmt
+
+#include <crypto/internal/hash.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/cryptohash.h>
+#include <linux/types.h>
+#include <crypto/sha.h>
+#include <asm/byteorder.h>
+#include <asm/i387.h>
+#include <asm/xcr.h>
+#include <asm/xsave.h>
+
+
+asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
+                                    unsigned int rounds);
+#ifdef SHA1_ENABLE_AVX_SUPPORT
+asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
+                                  unsigned int rounds);
+#endif
+
+static asmlinkage void (*sha1_transform_asm)(u32 *, const char *, unsigned int);
+
+
+static int sha1_ssse3_init(struct shash_desc *desc)
+{
+       struct sha1_state *sctx = shash_desc_ctx(desc);
+
+       *sctx = (struct sha1_state){
+               .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
+       };
+
+       return 0;
+}
+
+static int __sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
+                              unsigned int len, unsigned int partial)
+{
+       struct sha1_state *sctx = shash_desc_ctx(desc);
+       unsigned int done = 0;
+
+       sctx->count += len;
+
+       if (partial) {
+               done = SHA1_BLOCK_SIZE - partial;
+               memcpy(sctx->buffer + partial, data, done);
+               sha1_transform_asm(sctx->state, sctx->buffer, 1);
+       }
+
+       if (len - done >= SHA1_BLOCK_SIZE) {
+               const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE;
+
+               sha1_transform_asm(sctx->state, data + done, rounds);
+               done += rounds * SHA1_BLOCK_SIZE;
+       }
+
+       memcpy(sctx->buffer, data + done, len - done);
+
+       return 0;
+}
+
+static int sha1_ssse3_update(struct shash_desc *desc, const u8 *data,
+                            unsigned int len)
+{
+       struct sha1_state *sctx = shash_desc_ctx(desc);
+       unsigned int partial = sctx->count % SHA1_BLOCK_SIZE;
+       int res;
+
+       /* Handle the fast case right here */
+       if (partial + len < SHA1_BLOCK_SIZE) {
+               sctx->count += len;
+               memcpy(sctx->buffer + partial, data, len);
+
+               return 0;
+       }
+
+       if (!irq_fpu_usable()) {
+               res = crypto_sha1_update(desc, data, len);
+       } else {
+               kernel_fpu_begin();
+               res = __sha1_ssse3_update(desc, data, len, partial);
+               kernel_fpu_end();
+       }
+
+       return res;
+}
+
+
+/* Add padding and return the message digest. */
+static int sha1_ssse3_final(struct shash_desc *desc, u8 *out)
+{
+       struct sha1_state *sctx = shash_desc_ctx(desc);
+       unsigned int i, index, padlen;
+       __be32 *dst = (__be32 *)out;
+       __be64 bits;
+       static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, };
+
+       bits = cpu_to_be64(sctx->count << 3);
+
+       /* Pad out to 56 mod 64 and append length */
+       index = sctx->count % SHA1_BLOCK_SIZE;
+       padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index);
+       if (!irq_fpu_usable()) {
+               crypto_sha1_update(desc, padding, padlen);
+               crypto_sha1_update(desc, (const u8 *)&bits, sizeof(bits));
+       } else {
+               kernel_fpu_begin();
+               /* We need to fill a whole block for __sha1_ssse3_update() */
+               if (padlen <= 56) {
+                       sctx->count += padlen;
+                       memcpy(sctx->buffer + index, padding, padlen);
+               } else {
+                       __sha1_ssse3_update(desc, padding, padlen, index);
+               }
+               __sha1_ssse3_update(desc, (const u8 *)&bits, sizeof(bits), 56);
+               kernel_fpu_end();
+       }
+
+       /* Store state in digest */
+       for (i = 0; i < 5; i++)
+               dst[i] = cpu_to_be32(sctx->state[i]);
+
+       /* Wipe context */
+       memset(sctx, 0, sizeof(*sctx));
+
+       return 0;
+}
+
+static int sha1_ssse3_export(struct shash_desc *desc, void *out)
+{
+       struct sha1_state *sctx = shash_desc_ctx(desc);
+
+       memcpy(out, sctx, sizeof(*sctx));
+
+       return 0;
+}
+
+static int sha1_ssse3_import(struct shash_desc *desc, const void *in)
+{
+       struct sha1_state *sctx = shash_desc_ctx(desc);
+
+       memcpy(sctx, in, sizeof(*sctx));
+
+       return 0;
+}
+
+static struct shash_alg alg = {
+       .digestsize     =       SHA1_DIGEST_SIZE,
+       .init           =       sha1_ssse3_init,
+       .update         =       sha1_ssse3_update,
+       .final          =       sha1_ssse3_final,
+       .export         =       sha1_ssse3_export,
+       .import         =       sha1_ssse3_import,
+       .descsize       =       sizeof(struct sha1_state),
+       .statesize      =       sizeof(struct sha1_state),
+       .base           =       {
+               .cra_name       =       "sha1",
+               .cra_driver_name=       "sha1-ssse3",
+               .cra_priority   =       150,
+               .cra_flags      =       CRYPTO_ALG_TYPE_SHASH,
+               .cra_blocksize  =       SHA1_BLOCK_SIZE,
+               .cra_module     =       THIS_MODULE,
+       }
+};
+
+#ifdef SHA1_ENABLE_AVX_SUPPORT
+static bool __init avx_usable(void)
+{
+       u64 xcr0;
+
+       if (!cpu_has_avx || !cpu_has_osxsave)
+               return false;
+
+       xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+       if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
+               pr_info("AVX detected but unusable.\n");
+
+               return false;
+       }
+
+       return true;
+}
+#endif
+
+static int __init sha1_ssse3_mod_init(void)
+{
+       /* test for SSSE3 first */
+       if (cpu_has_ssse3)
+               sha1_transform_asm = sha1_transform_ssse3;
+
+#ifdef SHA1_ENABLE_AVX_SUPPORT
+       /* allow AVX to override SSSE3, it's a little faster */
+       if (avx_usable())
+               sha1_transform_asm = sha1_transform_avx;
+#endif
+
+       if (sha1_transform_asm) {
+               pr_info("Using %s optimized SHA-1 implementation\n",
+                       sha1_transform_asm == sha1_transform_ssse3 ? "SSSE3"
+                                                                  : "AVX");
+               return crypto_register_shash(&alg);
+       }
+       pr_info("Neither AVX nor SSSE3 is available/usable.\n");
+
+       return -ENODEV;
+}
+
+static void __exit sha1_ssse3_mod_fini(void)
+{
+       crypto_unregister_shash(&alg);
+}
+
+module_init(sha1_ssse3_mod_init);
+module_exit(sha1_ssse3_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
+
+MODULE_ALIAS("sha1");
index 575331cb2a8aa1e7b3becd10f4df475edee4bbc6..658af4bb35c9ff852033d0f4858824e92d4f623b 100644 (file)
@@ -26,7 +26,7 @@
 
 #define in_blk    12  /* input byte array address parameter*/
 #define out_blk   8  /* output byte array address parameter*/
-#define tfm       4  /* Twofish context structure */
+#define ctx       4  /* Twofish context structure */
 
 #define a_offset       0
 #define b_offset       4
@@ -229,8 +229,8 @@ twofish_enc_blk:
        push    %esi
        push    %edi
 
-       mov     tfm + 16(%esp), %ebp    /* abuse the base pointer: set new base bointer to the crypto tfm */
-       add     $crypto_tfm_ctx_offset, %ebp    /* ctx address */
+       mov     ctx + 16(%esp), %ebp    /* abuse the base pointer: set new base
+                                        * pointer to the ctx address */
        mov     in_blk+16(%esp),%edi    /* input address in edi */
 
        mov     (%edi),         %eax
@@ -285,8 +285,8 @@ twofish_dec_blk:
        push    %edi
 
 
-       mov     tfm + 16(%esp), %ebp    /* abuse the base pointer: set new base bointer to the crypto tfm */
-       add     $crypto_tfm_ctx_offset, %ebp    /* ctx address */
+       mov     ctx + 16(%esp), %ebp    /* abuse the base pointer: set new base
+                                        * pointer to the ctx address */
        mov     in_blk+16(%esp),%edi    /* input address in edi */
 
        mov     (%edi),         %eax
diff --git a/arch/x86/crypto/twofish-x86_64-asm_64-3way.S b/arch/x86/crypto/twofish-x86_64-asm_64-3way.S
new file mode 100644 (file)
index 0000000..5b012a2
--- /dev/null
@@ -0,0 +1,316 @@
+/*
+ * Twofish Cipher 3-way parallel algorithm (x86_64)
+ *
+ * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+.file "twofish-x86_64-asm-3way.S"
+.text
+
+/* structure of crypto context */
+#define s0     0
+#define s1     1024
+#define s2     2048
+#define s3     3072
+#define w      4096
+#define k      4128
+
+/**********************************************************************
+  3-way twofish
+ **********************************************************************/
+#define CTX %rdi
+#define RIO %rdx
+
+#define RAB0 %rax
+#define RAB1 %rbx
+#define RAB2 %rcx
+
+#define RAB0d %eax
+#define RAB1d %ebx
+#define RAB2d %ecx
+
+#define RAB0bh %ah
+#define RAB1bh %bh
+#define RAB2bh %ch
+
+#define RAB0bl %al
+#define RAB1bl %bl
+#define RAB2bl %cl
+
+#define RCD0 %r8
+#define RCD1 %r9
+#define RCD2 %r10
+
+#define RCD0d %r8d
+#define RCD1d %r9d
+#define RCD2d %r10d
+
+#define RX0 %rbp
+#define RX1 %r11
+#define RX2 %r12
+
+#define RX0d %ebp
+#define RX1d %r11d
+#define RX2d %r12d
+
+#define RY0 %r13
+#define RY1 %r14
+#define RY2 %r15
+
+#define RY0d %r13d
+#define RY1d %r14d
+#define RY2d %r15d
+
+#define RT0 %rdx
+#define RT1 %rsi
+
+#define RT0d %edx
+#define RT1d %esi
+
+#define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
+       movzbl ab ## bl,                tmp2 ## d; \
+       movzbl ab ## bh,                tmp1 ## d; \
+       rorq $(rot),                    ab; \
+       op1##l T0(CTX, tmp2, 4),        dst ## d; \
+       op2##l T1(CTX, tmp1, 4),        dst ## d;
+
+/*
+ * Combined G1 & G2 function. Reordered with help of rotates to have moves
+ * at begining.
+ */
+#define g1g2_3(ab, cd, Tx0, Tx1, Tx2, Tx3, Ty0, Ty1, Ty2, Ty3, x, y) \
+       /* G1,1 && G2,1 */ \
+       do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 0, ab ## 0, x ## 0); \
+       do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 0, ab ## 0, y ## 0); \
+       \
+       do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 1, ab ## 1, x ## 1); \
+       do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 1, ab ## 1, y ## 1); \
+       \
+       do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 2, ab ## 2, x ## 2); \
+       do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 2, ab ## 2, y ## 2); \
+       \
+       /* G1,2 && G2,2 */ \
+       do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 0, x ## 0); \
+       do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 0, y ## 0); \
+       xchgq cd ## 0, ab ## 0; \
+       \
+       do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 1, x ## 1); \
+       do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 1, y ## 1); \
+       xchgq cd ## 1, ab ## 1; \
+       \
+       do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 2, x ## 2); \
+       do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 2, y ## 2); \
+       xchgq cd ## 2, ab ## 2;
+
+#define enc_round_end(ab, x, y, n) \
+       addl y ## d,                    x ## d; \
+       addl x ## d,                    y ## d; \
+       addl k+4*(2*(n))(CTX),          x ## d; \
+       xorl ab ## d,                   x ## d; \
+       addl k+4*(2*(n)+1)(CTX),        y ## d; \
+       shrq $32,                       ab; \
+       roll $1,                        ab ## d; \
+       xorl y ## d,                    ab ## d; \
+       shlq $32,                       ab; \
+       rorl $1,                        x ## d; \
+       orq x,                          ab;
+
+#define dec_round_end(ba, x, y, n) \
+       addl y ## d,                    x ## d; \
+       addl x ## d,                    y ## d; \
+       addl k+4*(2*(n))(CTX),          x ## d; \
+       addl k+4*(2*(n)+1)(CTX),        y ## d; \
+       xorl ba ## d,                   y ## d; \
+       shrq $32,                       ba; \
+       roll $1,                        ba ## d; \
+       xorl x ## d,                    ba ## d; \
+       shlq $32,                       ba; \
+       rorl $1,                        y ## d; \
+       orq y,                          ba;
+
+#define encrypt_round3(ab, cd, n) \
+       g1g2_3(ab, cd, s0, s1, s2, s3, s0, s1, s2, s3, RX, RY); \
+       \
+       enc_round_end(ab ## 0, RX0, RY0, n); \
+       enc_round_end(ab ## 1, RX1, RY1, n); \
+       enc_round_end(ab ## 2, RX2, RY2, n);
+
+#define decrypt_round3(ba, dc, n) \
+       g1g2_3(ba, dc, s1, s2, s3, s0, s3, s0, s1, s2, RY, RX); \
+       \
+       dec_round_end(ba ## 0, RX0, RY0, n); \
+       dec_round_end(ba ## 1, RX1, RY1, n); \
+       dec_round_end(ba ## 2, RX2, RY2, n);
+
+#define encrypt_cycle3(ab, cd, n) \
+       encrypt_round3(ab, cd, n*2); \
+       encrypt_round3(ab, cd, (n*2)+1);
+
+#define decrypt_cycle3(ba, dc, n) \
+       decrypt_round3(ba, dc, (n*2)+1); \
+       decrypt_round3(ba, dc, (n*2));
+
+#define inpack3(in, n, xy, m) \
+       movq 4*(n)(in),                 xy ## 0; \
+       xorq w+4*m(CTX),                xy ## 0; \
+       \
+       movq 4*(4+(n))(in),             xy ## 1; \
+       xorq w+4*m(CTX),                xy ## 1; \
+       \
+       movq 4*(8+(n))(in),             xy ## 2; \
+       xorq w+4*m(CTX),                xy ## 2;
+
+#define outunpack3(op, out, n, xy, m) \
+       xorq w+4*m(CTX),                xy ## 0; \
+       op ## q xy ## 0,                4*(n)(out); \
+       \
+       xorq w+4*m(CTX),                xy ## 1; \
+       op ## q xy ## 1,                4*(4+(n))(out); \
+       \
+       xorq w+4*m(CTX),                xy ## 2; \
+       op ## q xy ## 2,                4*(8+(n))(out);
+
+#define inpack_enc3() \
+       inpack3(RIO, 0, RAB, 0); \
+       inpack3(RIO, 2, RCD, 2);
+
+#define outunpack_enc3(op) \
+       outunpack3(op, RIO, 2, RAB, 6); \
+       outunpack3(op, RIO, 0, RCD, 4);
+
+#define inpack_dec3() \
+       inpack3(RIO, 0, RAB, 4); \
+       rorq $32,                       RAB0; \
+       rorq $32,                       RAB1; \
+       rorq $32,                       RAB2; \
+       inpack3(RIO, 2, RCD, 6); \
+       rorq $32,                       RCD0; \
+       rorq $32,                       RCD1; \
+       rorq $32,                       RCD2;
+
+#define outunpack_dec3() \
+       rorq $32,                       RCD0; \
+       rorq $32,                       RCD1; \
+       rorq $32,                       RCD2; \
+       outunpack3(mov, RIO, 0, RCD, 0); \
+       rorq $32,                       RAB0; \
+       rorq $32,                       RAB1; \
+       rorq $32,                       RAB2; \
+       outunpack3(mov, RIO, 2, RAB, 2);
+
+.align 8
+.global __twofish_enc_blk_3way
+.type   __twofish_enc_blk_3way,@function;
+
+__twofish_enc_blk_3way:
+       /* input:
+        *      %rdi: ctx, CTX
+        *      %rsi: dst
+        *      %rdx: src, RIO
+        *      %rcx: bool, if true: xor output
+        */
+       pushq %r15;
+       pushq %r14;
+       pushq %r13;
+       pushq %r12;
+       pushq %rbp;
+       pushq %rbx;
+
+       pushq %rcx; /* bool xor */
+       pushq %rsi; /* dst */
+
+       inpack_enc3();
+
+       encrypt_cycle3(RAB, RCD, 0);
+       encrypt_cycle3(RAB, RCD, 1);
+       encrypt_cycle3(RAB, RCD, 2);
+       encrypt_cycle3(RAB, RCD, 3);
+       encrypt_cycle3(RAB, RCD, 4);
+       encrypt_cycle3(RAB, RCD, 5);
+       encrypt_cycle3(RAB, RCD, 6);
+       encrypt_cycle3(RAB, RCD, 7);
+
+       popq RIO; /* dst */
+       popq %rbp; /* bool xor */
+
+       testb %bpl, %bpl;
+       jnz __enc_xor3;
+
+       outunpack_enc3(mov);
+
+       popq %rbx;
+       popq %rbp;
+       popq %r12;
+       popq %r13;
+       popq %r14;
+       popq %r15;
+       ret;
+
+__enc_xor3:
+       outunpack_enc3(xor);
+
+       popq %rbx;
+       popq %rbp;
+       popq %r12;
+       popq %r13;
+       popq %r14;
+       popq %r15;
+       ret;
+
+.global twofish_dec_blk_3way
+.type   twofish_dec_blk_3way,@function;
+
+twofish_dec_blk_3way:
+       /* input:
+        *      %rdi: ctx, CTX
+        *      %rsi: dst
+        *      %rdx: src, RIO
+        */
+       pushq %r15;
+       pushq %r14;
+       pushq %r13;
+       pushq %r12;
+       pushq %rbp;
+       pushq %rbx;
+
+       pushq %rsi; /* dst */
+
+       inpack_dec3();
+
+       decrypt_cycle3(RAB, RCD, 7);
+       decrypt_cycle3(RAB, RCD, 6);
+       decrypt_cycle3(RAB, RCD, 5);
+       decrypt_cycle3(RAB, RCD, 4);
+       decrypt_cycle3(RAB, RCD, 3);
+       decrypt_cycle3(RAB, RCD, 2);
+       decrypt_cycle3(RAB, RCD, 1);
+       decrypt_cycle3(RAB, RCD, 0);
+
+       popq RIO; /* dst */
+
+       outunpack_dec3();
+
+       popq %rbx;
+       popq %rbp;
+       popq %r12;
+       popq %r13;
+       popq %r14;
+       popq %r15;
+       ret;
+
index 573aa102542e5a58714dc1b2b973b4f3e0d10a83..7bcf3fcc366839003db944ea5450c4c533d2a451 100644 (file)
 twofish_enc_blk:
        pushq    R1
 
-       /* %rdi contains the crypto tfm address */
+       /* %rdi contains the ctx address */
        /* %rsi contains the output address */
        /* %rdx contains the input address */
-       add     $crypto_tfm_ctx_offset, %rdi    /* set ctx address */
        /* ctx address is moved to free one non-rex register
        as target for the 8bit high operations */
        mov     %rdi,           %r11
@@ -274,10 +273,9 @@ twofish_enc_blk:
 twofish_dec_blk:
        pushq    R1
 
-       /* %rdi contains the crypto tfm address */
+       /* %rdi contains the ctx address */
        /* %rsi contains the output address */
        /* %rdx contains the input address */
-       add     $crypto_tfm_ctx_offset, %rdi    /* set ctx address */
        /* ctx address is moved to free one non-rex register
        as target for the 8bit high operations */
        mov     %rdi,           %r11
index cefaf8b9aa18142ff6bb6586f572a6209519d86f..dc6b3fb817fcfd6b36cb1cdf9ac111d6492d4218 100644 (file)
 #include <linux/module.h>
 #include <linux/types.h>
 
-asmlinkage void twofish_enc_blk(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
-asmlinkage void twofish_dec_blk(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
+asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
+                               const u8 *src);
+EXPORT_SYMBOL_GPL(twofish_enc_blk);
+asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
+                               const u8 *src);
+EXPORT_SYMBOL_GPL(twofish_dec_blk);
 
 static void twofish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
-       twofish_enc_blk(tfm, dst, src);
+       twofish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
 }
 
 static void twofish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 {
-       twofish_dec_blk(tfm, dst, src);
+       twofish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
 }
 
 static struct crypto_alg alg = {
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
new file mode 100644 (file)
index 0000000..5ede9c4
--- /dev/null
@@ -0,0 +1,472 @@
+/*
+ * Glue Code for 3-way parallel assembler optimized version of Twofish
+ *
+ * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
+ *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
+ * CTR part based on code (crypto/ctr.c) by:
+ *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
+ * USA
+ *
+ */
+
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/types.h>
+#include <crypto/algapi.h>
+#include <crypto/twofish.h>
+#include <crypto/b128ops.h>
+
+/* regular block cipher functions from twofish_x86_64 module */
+asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
+                               const u8 *src);
+asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
+                               const u8 *src);
+
+/* 3-way parallel cipher functions */
+asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
+                                      const u8 *src, bool xor);
+asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
+                                    const u8 *src);
+
+static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
+                                       const u8 *src)
+{
+       __twofish_enc_blk_3way(ctx, dst, src, false);
+}
+
+static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
+                                           const u8 *src)
+{
+       __twofish_enc_blk_3way(ctx, dst, src, true);
+}
+
+static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
+                    void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
+                    void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
+{
+       struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = TF_BLOCK_SIZE;
+       unsigned int nbytes;
+       int err;
+
+       err = blkcipher_walk_virt(desc, walk);
+
+       while ((nbytes = walk->nbytes)) {
+               u8 *wsrc = walk->src.virt.addr;
+               u8 *wdst = walk->dst.virt.addr;
+
+               /* Process three block batch */
+               if (nbytes >= bsize * 3) {
+                       do {
+                               fn_3way(ctx, wdst, wsrc);
+
+                               wsrc += bsize * 3;
+                               wdst += bsize * 3;
+                               nbytes -= bsize * 3;
+                       } while (nbytes >= bsize * 3);
+
+                       if (nbytes < bsize)
+                               goto done;
+               }
+
+               /* Handle leftovers */
+               do {
+                       fn(ctx, wdst, wsrc);
+
+                       wsrc += bsize;
+                       wdst += bsize;
+                       nbytes -= bsize;
+               } while (nbytes >= bsize);
+
+done:
+               err = blkcipher_walk_done(desc, walk, nbytes);
+       }
+
+       return err;
+}
+
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
+}
+
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
+}
+
+static struct crypto_alg blk_ecb_alg = {
+       .cra_name               = "ecb(twofish)",
+       .cra_driver_name        = "ecb-twofish-3way",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+       .cra_blocksize          = TF_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct twofish_ctx),
+       .cra_alignmask          = 0,
+       .cra_type               = &crypto_blkcipher_type,
+       .cra_module             = THIS_MODULE,
+       .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
+       .cra_u = {
+               .blkcipher = {
+                       .min_keysize    = TF_MIN_KEY_SIZE,
+                       .max_keysize    = TF_MAX_KEY_SIZE,
+                       .setkey         = twofish_setkey,
+                       .encrypt        = ecb_encrypt,
+                       .decrypt        = ecb_decrypt,
+               },
+       },
+};
+
+static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
+                                 struct blkcipher_walk *walk)
+{
+       struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = TF_BLOCK_SIZE;
+       unsigned int nbytes = walk->nbytes;
+       u128 *src = (u128 *)walk->src.virt.addr;
+       u128 *dst = (u128 *)walk->dst.virt.addr;
+       u128 *iv = (u128 *)walk->iv;
+
+       do {
+               u128_xor(dst, src, iv);
+               twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
+               iv = dst;
+
+               src += 1;
+               dst += 1;
+               nbytes -= bsize;
+       } while (nbytes >= bsize);
+
+       u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
+       return nbytes;
+}
+
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+       int err;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       err = blkcipher_walk_virt(desc, &walk);
+
+       while ((nbytes = walk.nbytes)) {
+               nbytes = __cbc_encrypt(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, nbytes);
+       }
+
+       return err;
+}
+
+static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
+                                 struct blkcipher_walk *walk)
+{
+       struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = TF_BLOCK_SIZE;
+       unsigned int nbytes = walk->nbytes;
+       u128 *src = (u128 *)walk->src.virt.addr;
+       u128 *dst = (u128 *)walk->dst.virt.addr;
+       u128 ivs[3 - 1];
+       u128 last_iv;
+
+       /* Start of the last block. */
+       src += nbytes / bsize - 1;
+       dst += nbytes / bsize - 1;
+
+       last_iv = *src;
+
+       /* Process three block batch */
+       if (nbytes >= bsize * 3) {
+               do {
+                       nbytes -= bsize * (3 - 1);
+                       src -= 3 - 1;
+                       dst -= 3 - 1;
+
+                       ivs[0] = src[0];
+                       ivs[1] = src[1];
+
+                       twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
+
+                       u128_xor(dst + 1, dst + 1, ivs + 0);
+                       u128_xor(dst + 2, dst + 2, ivs + 1);
+
+                       nbytes -= bsize;
+                       if (nbytes < bsize)
+                               goto done;
+
+                       u128_xor(dst, dst, src - 1);
+                       src -= 1;
+                       dst -= 1;
+               } while (nbytes >= bsize * 3);
+
+               if (nbytes < bsize)
+                       goto done;
+       }
+
+       /* Handle leftovers */
+       for (;;) {
+               twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
+
+               nbytes -= bsize;
+               if (nbytes < bsize)
+                       break;
+
+               u128_xor(dst, dst, src - 1);
+               src -= 1;
+               dst -= 1;
+       }
+
+done:
+       u128_xor(dst, dst, (u128 *)walk->iv);
+       *(u128 *)walk->iv = last_iv;
+
+       return nbytes;
+}
+
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                      struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+       int err;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       err = blkcipher_walk_virt(desc, &walk);
+
+       while ((nbytes = walk.nbytes)) {
+               nbytes = __cbc_decrypt(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, nbytes);
+       }
+
+       return err;
+}
+
+static struct crypto_alg blk_cbc_alg = {
+       .cra_name               = "cbc(twofish)",
+       .cra_driver_name        = "cbc-twofish-3way",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+       .cra_blocksize          = TF_BLOCK_SIZE,
+       .cra_ctxsize            = sizeof(struct twofish_ctx),
+       .cra_alignmask          = 0,
+       .cra_type               = &crypto_blkcipher_type,
+       .cra_module             = THIS_MODULE,
+       .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
+       .cra_u = {
+               .blkcipher = {
+                       .min_keysize    = TF_MIN_KEY_SIZE,
+                       .max_keysize    = TF_MAX_KEY_SIZE,
+                       .ivsize         = TF_BLOCK_SIZE,
+                       .setkey         = twofish_setkey,
+                       .encrypt        = cbc_encrypt,
+                       .decrypt        = cbc_decrypt,
+               },
+       },
+};
+
+static inline void u128_to_be128(be128 *dst, const u128 *src)
+{
+       dst->a = cpu_to_be64(src->a);
+       dst->b = cpu_to_be64(src->b);
+}
+
+static inline void be128_to_u128(u128 *dst, const be128 *src)
+{
+       dst->a = be64_to_cpu(src->a);
+       dst->b = be64_to_cpu(src->b);
+}
+
+static inline void u128_inc(u128 *i)
+{
+       i->b++;
+       if (!i->b)
+               i->a++;
+}
+
+static void ctr_crypt_final(struct blkcipher_desc *desc,
+                           struct blkcipher_walk *walk)
+{
+       struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       u8 *ctrblk = walk->iv;
+       u8 keystream[TF_BLOCK_SIZE];
+       u8 *src = walk->src.virt.addr;
+       u8 *dst = walk->dst.virt.addr;
+       unsigned int nbytes = walk->nbytes;
+
+       twofish_enc_blk(ctx, keystream, ctrblk);
+       crypto_xor(keystream, src, nbytes);
+       memcpy(dst, keystream, nbytes);
+
+       crypto_inc(ctrblk, TF_BLOCK_SIZE);
+}
+
+static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
+                               struct blkcipher_walk *walk)
+{
+       struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+       unsigned int bsize = TF_BLOCK_SIZE;
+       unsigned int nbytes = walk->nbytes;
+       u128 *src = (u128 *)walk->src.virt.addr;
+       u128 *dst = (u128 *)walk->dst.virt.addr;
+       u128 ctrblk;
+       be128 ctrblocks[3];
+
+       be128_to_u128(&ctrblk, (be128 *)walk->iv);
+
+       /* Process three block batch */
+       if (nbytes >= bsize * 3) {
+               do {
+                       if (dst != src) {
+                               dst[0] = src[0];
+                               dst[1] = src[1];
+                               dst[2] = src[2];
+                       }
+
+                       /* create ctrblks for parallel encrypt */
+                       u128_to_be128(&ctrblocks[0], &ctrblk);
+                       u128_inc(&ctrblk);
+                       u128_to_be128(&ctrblocks[1], &ctrblk);
+                       u128_inc(&ctrblk);
+                       u128_to_be128(&ctrblocks[2], &ctrblk);
+                       u128_inc(&ctrblk);
+
+                       twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
+                                                (u8 *)ctrblocks);
+
+                       src += 3;
+                       dst += 3;
+                       nbytes -= bsize * 3;
+               } while (nbytes >= bsize * 3);
+
+               if (nbytes < bsize)
+                       goto done;
+       }
+
+       /* Handle leftovers */
+       do {
+               if (dst != src)
+                       *dst = *src;
+
+               u128_to_be128(&ctrblocks[0], &ctrblk);
+               u128_inc(&ctrblk);
+
+               twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
+               u128_xor(dst, dst, (u128 *)ctrblocks);
+
+               src += 1;
+               dst += 1;
+               nbytes -= bsize;
+       } while (nbytes >= bsize);
+
+done:
+       u128_to_be128((be128 *)walk->iv, &ctrblk);
+       return nbytes;
+}
+
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+                    struct scatterlist *src, unsigned int nbytes)
+{
+       struct blkcipher_walk walk;
+       int err;
+
+       blkcipher_walk_init(&walk, dst, src, nbytes);
+       err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
+
+       while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
+               nbytes = __ctr_crypt(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, nbytes);
+       }
+
+       if (walk.nbytes) {
+               ctr_crypt_final(desc, &walk);
+               err = blkcipher_walk_done(desc, &walk, 0);
+       }
+
+       return err;
+}
+
+static struct crypto_alg blk_ctr_alg = {
+       .cra_name               = "ctr(twofish)",
+       .cra_driver_name        = "ctr-twofish-3way",
+       .cra_priority           = 300,
+       .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
+       .cra_blocksize          = 1,
+       .cra_ctxsize            = sizeof(struct twofish_ctx),
+       .cra_alignmask          = 0,
+       .cra_type               = &crypto_blkcipher_type,
+       .cra_module             = THIS_MODULE,
+       .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
+       .cra_u = {
+               .blkcipher = {
+                       .min_keysize    = TF_MIN_KEY_SIZE,
+                       .max_keysize    = TF_MAX_KEY_SIZE,
+                       .ivsize         = TF_BLOCK_SIZE,
+                       .setkey         = twofish_setkey,
+                       .encrypt        = ctr_crypt,
+                       .decrypt        = ctr_crypt,
+               },
+       },
+};
+
+int __init init(void)
+{
+       int err;
+
+       err = crypto_register_alg(&blk_ecb_alg);
+       if (err)
+               goto ecb_err;
+       err = crypto_register_alg(&blk_cbc_alg);
+       if (err)
+               goto cbc_err;
+       err = crypto_register_alg(&blk_ctr_alg);
+       if (err)
+               goto ctr_err;
+
+       return 0;
+
+ctr_err:
+       crypto_unregister_alg(&blk_cbc_alg);
+cbc_err:
+       crypto_unregister_alg(&blk_ecb_alg);
+ecb_err:
+       return err;
+}
+
+void __exit fini(void)
+{
+       crypto_unregister_alg(&blk_ctr_alg);
+       crypto_unregister_alg(&blk_cbc_alg);
+       crypto_unregister_alg(&blk_ecb_alg);
+}
+
+module_init(init);
+module_exit(fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Twofish Cipher Algorithm, 3-way parallel asm optimized");
+MODULE_ALIAS("twofish");
+MODULE_ALIAS("twofish-asm");
index 2f84a433b6a008ce5f5a503576a08368d07751cb..f3444f700f3619eedcc8f3fa050586148e80ad70 100644 (file)
@@ -259,7 +259,9 @@ extern const char * const x86_power_flags[32];
 #define cpu_has_xmm            boot_cpu_has(X86_FEATURE_XMM)
 #define cpu_has_xmm2           boot_cpu_has(X86_FEATURE_XMM2)
 #define cpu_has_xmm3           boot_cpu_has(X86_FEATURE_XMM3)
+#define cpu_has_ssse3          boot_cpu_has(X86_FEATURE_SSSE3)
 #define cpu_has_aes            boot_cpu_has(X86_FEATURE_AES)
+#define cpu_has_avx            boot_cpu_has(X86_FEATURE_AVX)
 #define cpu_has_ht             boot_cpu_has(X86_FEATURE_HT)
 #define cpu_has_mp             boot_cpu_has(X86_FEATURE_MP)
 #define cpu_has_nx             boot_cpu_has(X86_FEATURE_NX)
@@ -287,6 +289,7 @@ extern const char * const x86_power_flags[32];
 #define cpu_has_xmm4_2         boot_cpu_has(X86_FEATURE_XMM4_2)
 #define cpu_has_x2apic         boot_cpu_has(X86_FEATURE_X2APIC)
 #define cpu_has_xsave          boot_cpu_has(X86_FEATURE_XSAVE)
+#define cpu_has_osxsave                boot_cpu_has(X86_FEATURE_OSXSAVE)
 #define cpu_has_hypervisor     boot_cpu_has(X86_FEATURE_HYPERVISOR)
 #define cpu_has_pclmulqdq      boot_cpu_has(X86_FEATURE_PCLMULQDQ)
 #define cpu_has_perfctr_core   boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
index ae27b7534ea7d14ee0c5f493e3eaf6e8f123eaa2..527a857d10b61a4bd4d23d243cf43e2551d4faed 100644 (file)
@@ -100,6 +100,14 @@ config CRYPTO_MANAGER2
        select CRYPTO_BLKCIPHER2
        select CRYPTO_PCOMP2
 
+config CRYPTO_USER
+       tristate "Userspace cryptographic algorithm configuration"
+       depends on NET
+       select CRYPTO_MANAGER
+       help
+         Userapace configuration for cryptographic instantiations such as
+         cbc(aes).
+
 config CRYPTO_MANAGER_DISABLE_TESTS
        bool "Disable run-time self tests"
        default y
@@ -407,6 +415,16 @@ config CRYPTO_SHA1
        help
          SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).
 
+config CRYPTO_SHA1_SSSE3
+       tristate "SHA1 digest algorithm (SSSE3/AVX)"
+       depends on X86 && 64BIT
+       select CRYPTO_SHA1
+       select CRYPTO_HASH
+       help
+         SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented
+         using Supplemental SSE3 (SSSE3) instructions or Advanced Vector
+         Extensions (AVX), when available.
+
 config CRYPTO_SHA256
        tristate "SHA224 and SHA256 digest algorithm"
        select CRYPTO_HASH
@@ -590,6 +608,7 @@ config CRYPTO_ARC4
 config CRYPTO_BLOWFISH
        tristate "Blowfish cipher algorithm"
        select CRYPTO_ALGAPI
+       select CRYPTO_BLOWFISH_COMMON
        help
          Blowfish cipher algorithm, by Bruce Schneier.
 
@@ -600,6 +619,30 @@ config CRYPTO_BLOWFISH
          See also:
          <http://www.schneier.com/blowfish.html>
 
+config CRYPTO_BLOWFISH_COMMON
+       tristate
+       help
+         Common parts of the Blowfish cipher algorithm shared by the
+         generic c and the assembler implementations.
+
+         See also:
+         <http://www.schneier.com/blowfish.html>
+
+config CRYPTO_BLOWFISH_X86_64
+       tristate "Blowfish cipher algorithm (x86_64)"
+       depends on (X86 || UML_X86) && 64BIT
+       select CRYPTO_ALGAPI
+       select CRYPTO_BLOWFISH_COMMON
+       help
+         Blowfish cipher algorithm (x86_64), by Bruce Schneier.
+
+         This is a variable key length cipher which can use keys from 32
+         bits to 448 bits in length.  It's fast, simple and specifically
+         designed for use on "large microprocessors".
+
+         See also:
+         <http://www.schneier.com/blowfish.html>
+
 config CRYPTO_CAMELLIA
        tristate "Camellia cipher algorithms"
        depends on CRYPTO
@@ -793,6 +836,26 @@ config CRYPTO_TWOFISH_X86_64
          See also:
          <http://www.schneier.com/twofish.html>
 
+config CRYPTO_TWOFISH_X86_64_3WAY
+       tristate "Twofish cipher algorithm (x86_64, 3-way parallel)"
+       depends on (X86 || UML_X86) && 64BIT
+       select CRYPTO_ALGAPI
+       select CRYPTO_TWOFISH_COMMON
+       select CRYPTO_TWOFISH_X86_64
+       help
+         Twofish cipher algorithm (x86_64, 3-way parallel).
+
+         Twofish was submitted as an AES (Advanced Encryption Standard)
+         candidate cipher by researchers at CounterPane Systems.  It is a
+         16 round block cipher supporting key sizes of 128, 192, and 256
+         bits.
+
+         This module provides Twofish cipher algorithm that processes three
+         blocks parallel, utilizing resources of out-of-order CPUs better.
+
+         See also:
+         <http://www.schneier.com/twofish.html>
+
 comment "Compression"
 
 config CRYPTO_DEFLATE
index ce5a813d36398254b0b213add68c0e34dbf59efa..9e6eee2c05db0735363915db0742e01ff2dabf5f 100644 (file)
@@ -31,6 +31,7 @@ obj-$(CONFIG_CRYPTO_PCOMP2) += pcompress.o
 cryptomgr-y := algboss.o testmgr.o
 
 obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o
+obj-$(CONFIG_CRYPTO_USER) += crypto_user.o
 obj-$(CONFIG_CRYPTO_HMAC) += hmac.o
 obj-$(CONFIG_CRYPTO_VMAC) += vmac.o
 obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o
@@ -60,7 +61,8 @@ obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o
 obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o
 obj-$(CONFIG_CRYPTO_DES) += des_generic.o
 obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o
-obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish.o
+obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish_generic.o
+obj-$(CONFIG_CRYPTO_BLOWFISH_COMMON) += blowfish_common.o
 obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o
 obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o
 obj-$(CONFIG_CRYPTO_SERPENT) += serpent.o
index fdc67d38660bc08269697c50b3424fc85059e1e8..a816f24f2d527993aea1fb724680857e77a06ff2 100644 (file)
@@ -23,6 +23,8 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 #include <crypto/scatterwalk.h>
 
@@ -381,6 +383,28 @@ static int crypto_init_ablkcipher_ops(struct crypto_tfm *tfm, u32 type,
        return 0;
 }
 
+static int crypto_ablkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_blkcipher rblkcipher;
+
+       snprintf(rblkcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "ablkcipher");
+       snprintf(rblkcipher.geniv, CRYPTO_MAX_ALG_NAME, "%s",
+                alg->cra_ablkcipher.geniv ?: "<default>");
+
+       rblkcipher.blocksize = alg->cra_blocksize;
+       rblkcipher.min_keysize = alg->cra_ablkcipher.min_keysize;
+       rblkcipher.max_keysize = alg->cra_ablkcipher.max_keysize;
+       rblkcipher.ivsize = alg->cra_ablkcipher.ivsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_BLKCIPHER,
+               sizeof(struct crypto_report_blkcipher), &rblkcipher);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg)
@@ -403,6 +427,7 @@ const struct crypto_type crypto_ablkcipher_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_ablkcipher_show,
 #endif
+       .report = crypto_ablkcipher_report,
 };
 EXPORT_SYMBOL_GPL(crypto_ablkcipher_type);
 
@@ -432,6 +457,28 @@ static int crypto_init_givcipher_ops(struct crypto_tfm *tfm, u32 type,
        return 0;
 }
 
+static int crypto_givcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_blkcipher rblkcipher;
+
+       snprintf(rblkcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "givcipher");
+       snprintf(rblkcipher.geniv, CRYPTO_MAX_ALG_NAME, "%s",
+                alg->cra_ablkcipher.geniv ?: "<built-in>");
+
+       rblkcipher.blocksize = alg->cra_blocksize;
+       rblkcipher.min_keysize = alg->cra_ablkcipher.min_keysize;
+       rblkcipher.max_keysize = alg->cra_ablkcipher.max_keysize;
+       rblkcipher.ivsize = alg->cra_ablkcipher.ivsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_BLKCIPHER,
+               sizeof(struct crypto_report_blkcipher), &rblkcipher);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg)
@@ -454,6 +501,7 @@ const struct crypto_type crypto_givcipher_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_givcipher_show,
 #endif
+       .report = crypto_givcipher_report,
 };
 EXPORT_SYMBOL_GPL(crypto_givcipher_type);
 
index 6729e8ff68e7346762734538cecda2b8f9909b6d..701556ffaaef0e6ca21256b52a8b594904852f9e 100644 (file)
@@ -21,6 +21,8 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 #include "internal.h"
 
@@ -109,6 +111,28 @@ static int crypto_init_aead_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
        return 0;
 }
 
+static int crypto_aead_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_aead raead;
+       struct aead_alg *aead = &alg->cra_aead;
+
+       snprintf(raead.type, CRYPTO_MAX_ALG_NAME, "%s", "aead");
+       snprintf(raead.geniv, CRYPTO_MAX_ALG_NAME, "%s",
+                aead->geniv ?: "<built-in>");
+
+       raead.blocksize = alg->cra_blocksize;
+       raead.maxauthsize = aead->maxauthsize;
+       raead.ivsize = aead->ivsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_AEAD,
+               sizeof(struct crypto_report_aead), &raead);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg)
@@ -130,6 +154,7 @@ const struct crypto_type crypto_aead_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_aead_show,
 #endif
+       .report = crypto_aead_report,
 };
 EXPORT_SYMBOL_GPL(crypto_aead_type);
 
@@ -165,6 +190,28 @@ static int crypto_init_nivaead_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
        return 0;
 }
 
+static int crypto_nivaead_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_aead raead;
+       struct aead_alg *aead = &alg->cra_aead;
+
+       snprintf(raead.type, CRYPTO_MAX_ALG_NAME, "%s", "nivaead");
+       snprintf(raead.geniv, CRYPTO_MAX_ALG_NAME, "%s", aead->geniv);
+
+       raead.blocksize = alg->cra_blocksize;
+       raead.maxauthsize = aead->maxauthsize;
+       raead.ivsize = aead->ivsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_AEAD,
+               sizeof(struct crypto_report_aead), &raead);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
+
 static void crypto_nivaead_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_nivaead_show(struct seq_file *m, struct crypto_alg *alg)
@@ -186,6 +233,7 @@ const struct crypto_type crypto_nivaead_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_nivaead_show,
 #endif
+       .report = crypto_nivaead_report,
 };
 EXPORT_SYMBOL_GPL(crypto_nivaead_type);
 
index f669822a7a443c81a04c8fb61d8d382a64546542..a3e6ef99394a9e78c7408ea4db65caa652004a1d 100644 (file)
@@ -21,6 +21,8 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 #include "internal.h"
 
@@ -397,6 +399,24 @@ static unsigned int crypto_ahash_extsize(struct crypto_alg *alg)
        return sizeof(struct crypto_shash *);
 }
 
+static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_hash rhash;
+
+       snprintf(rhash.type, CRYPTO_MAX_ALG_NAME, "%s", "ahash");
+
+       rhash.blocksize = alg->cra_blocksize;
+       rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_HASH,
+               sizeof(struct crypto_report_hash), &rhash);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
@@ -415,6 +435,7 @@ const struct crypto_type crypto_ahash_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_ahash_show,
 #endif
+       .report = crypto_ahash_report,
        .maskclear = ~CRYPTO_ALG_TYPE_MASK,
        .maskset = CRYPTO_ALG_TYPE_AHASH_MASK,
        .type = CRYPTO_ALG_TYPE_AHASH,
index c3cf1a69a47a8dcaa57e62410c9823ee4e2d045b..54dd4e33b5d61a5049713ee74ad9dc94d40670b6 100644 (file)
@@ -22,8 +22,6 @@
 
 #include "internal.h"
 
-static void crypto_remove_final(struct list_head *list);
-
 static LIST_HEAD(crypto_template_list);
 
 void crypto_larval_error(const char *name, u32 type, u32 mask)
@@ -129,9 +127,8 @@ static void crypto_remove_spawn(struct crypto_spawn *spawn,
        BUG_ON(!list_empty(&inst->alg.cra_users));
 }
 
-static void crypto_remove_spawns(struct crypto_alg *alg,
-                                struct list_head *list,
-                                struct crypto_alg *nalg)
+void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
+                         struct crypto_alg *nalg)
 {
        u32 new_type = (nalg ?: alg)->cra_flags;
        struct crypto_spawn *spawn, *n;
@@ -177,6 +174,7 @@ static void crypto_remove_spawns(struct crypto_alg *alg,
                        crypto_remove_spawn(spawn, list);
        }
 }
+EXPORT_SYMBOL_GPL(crypto_remove_spawns);
 
 static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg)
 {
@@ -321,7 +319,7 @@ unlock:
 }
 EXPORT_SYMBOL_GPL(crypto_alg_tested);
 
-static void crypto_remove_final(struct list_head *list)
+void crypto_remove_final(struct list_head *list)
 {
        struct crypto_alg *alg;
        struct crypto_alg *n;
@@ -331,6 +329,7 @@ static void crypto_remove_final(struct list_head *list)
                crypto_alg_put(alg);
        }
 }
+EXPORT_SYMBOL_GPL(crypto_remove_final);
 
 static void crypto_wait_for_test(struct crypto_larval *larval)
 {
@@ -493,6 +492,7 @@ int crypto_register_instance(struct crypto_template *tmpl,
                goto err;
 
        inst->alg.cra_module = tmpl->module;
+       inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
 
        down_write(&crypto_alg_sem);
 
index 7a7219266e3cc343d1a9a63a492a9cd38e876db5..2572d26001364b6206b300ecbc42ca18dfca9182 100644 (file)
@@ -24,6 +24,8 @@
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 #include "internal.h"
 
@@ -492,6 +494,28 @@ static int crypto_init_blkcipher_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
                return crypto_init_blkcipher_ops_async(tfm);
 }
 
+static int crypto_blkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_blkcipher rblkcipher;
+
+       snprintf(rblkcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "blkcipher");
+       snprintf(rblkcipher.geniv, CRYPTO_MAX_ALG_NAME, "%s",
+                alg->cra_blkcipher.geniv ?: "<default>");
+
+       rblkcipher.blocksize = alg->cra_blocksize;
+       rblkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
+       rblkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
+       rblkcipher.ivsize = alg->cra_blkcipher.ivsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_BLKCIPHER,
+               sizeof(struct crypto_report_blkcipher), &rblkcipher);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
@@ -511,6 +535,7 @@ const struct crypto_type crypto_blkcipher_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_blkcipher_show,
 #endif
+       .report = crypto_blkcipher_report,
 };
 EXPORT_SYMBOL_GPL(crypto_blkcipher_type);
 
diff --git a/crypto/blowfish.c b/crypto/blowfish.c
deleted file mode 100644 (file)
index a67d52e..0000000
+++ /dev/null
@@ -1,482 +0,0 @@
-/*
- * Cryptographic API.
- *
- * Blowfish Cipher Algorithm, by Bruce Schneier.
- * http://www.counterpane.com/blowfish.html
- *
- * Adapted from Kerneli implementation.
- *
- * Copyright (c) Herbert Valerio Riedel <hvr@hvrlab.org>
- * Copyright (c) Kyle McMartin <kyle@debian.org>
- * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- */
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <asm/byteorder.h>
-#include <linux/crypto.h>
-#include <linux/types.h>
-
-#define BF_BLOCK_SIZE 8
-#define BF_MIN_KEY_SIZE 4
-#define BF_MAX_KEY_SIZE 56
-
-struct bf_ctx {
-       u32 p[18];
-       u32 s[1024];
-};
-
-static const u32 bf_pbox[16 + 2] = {
-       0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344,
-       0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89,
-       0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c,
-       0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917,
-       0x9216d5d9, 0x8979fb1b,
-};
-
-static const u32 bf_sbox[256 * 4] = {
-       0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7,
-       0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99,
-       0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16,
-       0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e,
-       0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee,
-       0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013,
-       0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef,
-       0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e,
-       0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60,
-       0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440,
-       0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce,
-       0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a,
-       0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e,
-       0xafd6ba33, 0x6c24cf5c, 0x7a325381, 0x28958677,
-       0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193,
-       0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032,
-       0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88,
-       0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,
-       0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e,
-       0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0,
-       0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3,
-       0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98,
-       0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88,
-       0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe,
-       0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6,
-       0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d,
-       0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b,
-       0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7,
-       0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba,
-       0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463,
-       0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f,
-       0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09,
-       0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3,
-       0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb,
-       0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279,
-       0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,
-       0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab,
-       0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82,
-       0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db,
-       0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573,
-       0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0,
-       0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b,
-       0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790,
-       0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8,
-       0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4,
-       0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0,
-       0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7,
-       0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c,
-       0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad,
-       0x2f2f2218, 0xbe0e1777, 0xea752dfe, 0x8b021fa1,
-       0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299,
-       0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9,
-       0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477,
-       0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,
-       0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49,
-       0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af,
-       0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa,
-       0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5,
-       0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41,
-       0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915,
-       0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400,
-       0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915,
-       0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664,
-       0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a,
-       0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623,
-       0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266,
-       0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1,
-       0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e,
-       0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6,
-       0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1,
-       0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e,
-       0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,
-       0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737,
-       0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8,
-       0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff,
-       0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd,
-       0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701,
-       0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7,
-       0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41,
-       0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331,
-       0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf,
-       0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af,
-       0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e,
-       0x5512721f, 0x2e6b7124, 0x501adde6, 0x9f84cd87,
-       0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c,
-       0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2,
-       0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16,
-       0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd,
-       0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b,
-       0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509,
-       0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e,
-       0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3,
-       0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f,
-       0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a,
-       0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4,
-       0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960,
-       0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66,
-       0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28,
-       0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802,
-       0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84,
-       0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510,
-       0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf,
-       0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14,
-       0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e,
-       0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50,
-       0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7,
-       0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8,
-       0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281,
-       0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99,
-       0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696,
-       0xcdb30aeb, 0x532e3054, 0x8fd948e4, 0x6dbc3128,
-       0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73,
-       0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0,
-       0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0,
-       0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105,
-       0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250,
-       0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3,
-       0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285,
-       0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00,
-       0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061,
-       0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb,
-       0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e,
-       0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735,
-       0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc,
-       0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9,
-       0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340,
-       0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20,
-       0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7,
-       0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934,
-       0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068,
-       0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af,
-       0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840,
-       0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45,
-       0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504,
-       0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a,
-       0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb,
-       0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee,
-       0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6,
-       0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42,
-       0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b,
-       0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2,
-       0x3a6efa74, 0xdd5b4332, 0x6841e7f7, 0xca7820fb,
-       0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527,
-       0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b,
-       0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33,
-       0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c,
-       0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3,
-       0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc,
-       0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17,
-       0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564,
-       0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b,
-       0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115,
-       0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922,
-       0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728,
-       0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0,
-       0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e,
-       0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37,
-       0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d,
-       0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804,
-       0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b,
-       0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3,
-       0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb,
-       0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d,
-       0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c,
-       0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350,
-       0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9,
-       0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a,
-       0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe,
-       0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d,
-       0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc,
-       0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f,
-       0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61,
-       0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2,
-       0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9,
-       0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2,
-       0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c,
-       0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e,
-       0xb77f19b6, 0xe0a9dc09, 0x662d09a1, 0xc4324633,
-       0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10,
-       0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169,
-       0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52,
-       0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027,
-       0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5,
-       0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62,
-       0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634,
-       0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76,
-       0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24,
-       0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc,
-       0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4,
-       0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c,
-       0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837,
-       0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0,
-       0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b,
-       0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe,
-       0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b,
-       0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4,
-       0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8,
-       0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6,
-       0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304,
-       0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22,
-       0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4,
-       0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6,
-       0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9,
-       0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59,
-       0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593,
-       0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51,
-       0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28,
-       0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c,
-       0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b,
-       0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28,
-       0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c,
-       0x15056dd4, 0x88f46dba, 0x03a16125, 0x0564f0bd,
-       0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a,
-       0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319,
-       0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb,
-       0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f,
-       0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991,
-       0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32,
-       0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680,
-       0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166,
-       0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae,
-       0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb,
-       0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5,
-       0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47,
-       0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370,
-       0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d,
-       0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84,
-       0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048,
-       0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8,
-       0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd,
-       0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9,
-       0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7,
-       0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38,
-       0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f,
-       0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c,
-       0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525,
-       0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1,
-       0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442,
-       0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, 0x3278e964,
-       0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e,
-       0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8,
-       0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d,
-       0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f,
-       0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299,
-       0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02,
-       0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc,
-       0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614,
-       0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a,
-       0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6,
-       0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b,
-       0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0,
-       0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060,
-       0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e,
-       0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9,
-       0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f,
-       0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6,
-};
-
-/*
- * Round loop unrolling macros, S is a pointer to a S-Box array
- * organized in 4 unsigned longs at a row.
- */
-#define GET32_3(x) (((x) & 0xff))
-#define GET32_2(x) (((x) >> (8)) & (0xff))
-#define GET32_1(x) (((x) >> (16)) & (0xff))
-#define GET32_0(x) (((x) >> (24)) & (0xff))
-
-#define bf_F(x) (((S[GET32_0(x)] + S[256 + GET32_1(x)]) ^ \
-          S[512 + GET32_2(x)]) + S[768 + GET32_3(x)])
-
-#define ROUND(a, b, n)  b ^= P[n]; a ^= bf_F (b)
-
-/*
- * The blowfish encipher, processes 64-bit blocks.
- * NOTE: This function MUSTN'T respect endianess
- */
-static void encrypt_block(struct bf_ctx *bctx, u32 *dst, u32 *src)
-{
-       const u32 *P = bctx->p;
-       const u32 *S = bctx->s;
-       u32 yl = src[0];
-       u32 yr = src[1];
-
-       ROUND(yr, yl, 0);
-       ROUND(yl, yr, 1);
-       ROUND(yr, yl, 2);
-       ROUND(yl, yr, 3);
-       ROUND(yr, yl, 4);
-       ROUND(yl, yr, 5);
-       ROUND(yr, yl, 6);
-       ROUND(yl, yr, 7);
-       ROUND(yr, yl, 8);
-       ROUND(yl, yr, 9);
-       ROUND(yr, yl, 10);
-       ROUND(yl, yr, 11);
-       ROUND(yr, yl, 12);
-       ROUND(yl, yr, 13);
-       ROUND(yr, yl, 14);
-       ROUND(yl, yr, 15);
-
-       yl ^= P[16];
-       yr ^= P[17];
-
-       dst[0] = yr;
-       dst[1] = yl;
-}
-
-static void bf_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-       const __be32 *in_blk = (const __be32 *)src;
-       __be32 *const out_blk = (__be32 *)dst;
-       u32 in32[2], out32[2];
-
-       in32[0] = be32_to_cpu(in_blk[0]);
-       in32[1] = be32_to_cpu(in_blk[1]);
-       encrypt_block(crypto_tfm_ctx(tfm), out32, in32);
-       out_blk[0] = cpu_to_be32(out32[0]);
-       out_blk[1] = cpu_to_be32(out32[1]);
-}
-
-static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-       struct bf_ctx *ctx = crypto_tfm_ctx(tfm);
-       const __be32 *in_blk = (const __be32 *)src;
-       __be32 *const out_blk = (__be32 *)dst;
-       const u32 *P = ctx->p;
-       const u32 *S = ctx->s;
-       u32 yl = be32_to_cpu(in_blk[0]);
-       u32 yr = be32_to_cpu(in_blk[1]);
-
-       ROUND(yr, yl, 17);
-       ROUND(yl, yr, 16);
-       ROUND(yr, yl, 15);
-       ROUND(yl, yr, 14);
-       ROUND(yr, yl, 13);
-       ROUND(yl, yr, 12);
-       ROUND(yr, yl, 11);
-       ROUND(yl, yr, 10);
-       ROUND(yr, yl, 9);
-       ROUND(yl, yr, 8);
-       ROUND(yr, yl, 7);
-       ROUND(yl, yr, 6);
-       ROUND(yr, yl, 5);
-       ROUND(yl, yr, 4);
-       ROUND(yr, yl, 3);
-       ROUND(yl, yr, 2);
-
-       yl ^= P[1];
-       yr ^= P[0];
-
-       out_blk[0] = cpu_to_be32(yr);
-       out_blk[1] = cpu_to_be32(yl);
-}
-
-/*
- * Calculates the blowfish S and P boxes for encryption and decryption.
- */
-static int bf_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen)
-{
-       struct bf_ctx *ctx = crypto_tfm_ctx(tfm);
-       u32 *P = ctx->p;
-       u32 *S = ctx->s;
-       short i, j, count;
-       u32 data[2], temp;
-
-       /* Copy the initialization s-boxes */
-       for (i = 0, count = 0; i < 256; i++)
-               for (j = 0; j < 4; j++, count++)
-                       S[count] = bf_sbox[count];
-
-       /* Set the p-boxes */
-       for (i = 0; i < 16 + 2; i++)
-               P[i] = bf_pbox[i];
-
-       /* Actual subkey generation */
-       for (j = 0, i = 0; i < 16 + 2; i++) {
-               temp = (((u32)key[j] << 24) |
-                       ((u32)key[(j + 1) % keylen] << 16) |
-                       ((u32)key[(j + 2) % keylen] << 8) |
-                       ((u32)key[(j + 3) % keylen]));
-
-               P[i] = P[i] ^ temp;
-               j = (j + 4) % keylen;
-       }
-
-       data[0] = 0x00000000;
-       data[1] = 0x00000000;
-
-       for (i = 0; i < 16 + 2; i += 2) {
-               encrypt_block((struct bf_ctx *)ctx, data, data);
-
-               P[i] = data[0];
-               P[i + 1] = data[1];
-       }
-
-       for (i = 0; i < 4; i++) {
-               for (j = 0, count = i * 256; j < 256; j += 2, count += 2) {
-                       encrypt_block((struct bf_ctx *)ctx, data, data);
-
-                       S[count] = data[0];
-                       S[count + 1] = data[1];
-               }
-       }
-
-       /* Bruce says not to bother with the weak key check. */
-       return 0;
-}
-
-static struct crypto_alg alg = {
-       .cra_name               =       "blowfish",
-       .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
-       .cra_blocksize          =       BF_BLOCK_SIZE,
-       .cra_ctxsize            =       sizeof(struct bf_ctx),
-       .cra_alignmask          =       3,
-       .cra_module             =       THIS_MODULE,
-       .cra_list               =       LIST_HEAD_INIT(alg.cra_list),
-       .cra_u                  =       { .cipher = {
-       .cia_min_keysize        =       BF_MIN_KEY_SIZE,
-       .cia_max_keysize        =       BF_MAX_KEY_SIZE,
-       .cia_setkey             =       bf_setkey,
-       .cia_encrypt            =       bf_encrypt,
-       .cia_decrypt            =       bf_decrypt } }
-};
-
-static int __init blowfish_mod_init(void)
-{
-       return crypto_register_alg(&alg);
-}
-
-static void __exit blowfish_mod_fini(void)
-{
-       crypto_unregister_alg(&alg);
-}
-
-module_init(blowfish_mod_init);
-module_exit(blowfish_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Blowfish Cipher Algorithm");
diff --git a/crypto/blowfish_common.c b/crypto/blowfish_common.c
new file mode 100644 (file)
index 0000000..f636aab
--- /dev/null
@@ -0,0 +1,402 @@
+/*
+ * Cryptographic API.
+ *
+ * Common Blowfish algorithm parts shared between the c and assembler
+ * implementations.
+ *
+ * Blowfish Cipher Algorithm, by Bruce Schneier.
+ * http://www.counterpane.com/blowfish.html
+ *
+ * Adapted from Kerneli implementation.
+ *
+ * Copyright (c) Herbert Valerio Riedel <hvr@hvrlab.org>
+ * Copyright (c) Kyle McMartin <kyle@debian.org>
+ * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <asm/byteorder.h>
+#include <linux/crypto.h>
+#include <linux/types.h>
+#include <crypto/blowfish.h>
+
+static const u32 bf_pbox[16 + 2] = {
+       0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344,
+       0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89,
+       0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c,
+       0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917,
+       0x9216d5d9, 0x8979fb1b,
+};
+
+static const u32 bf_sbox[256 * 4] = {
+       0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7,
+       0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99,
+       0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16,
+       0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e,
+       0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee,
+       0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013,
+       0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef,
+       0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e,
+       0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60,
+       0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440,
+       0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce,
+       0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a,
+       0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e,
+       0xafd6ba33, 0x6c24cf5c, 0x7a325381, 0x28958677,
+       0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193,
+       0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032,
+       0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88,
+       0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,
+       0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e,
+       0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0,
+       0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3,
+       0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98,
+       0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88,
+       0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe,
+       0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6,
+       0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d,
+       0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b,
+       0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7,
+       0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba,
+       0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463,
+       0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f,
+       0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09,
+       0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3,
+       0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb,
+       0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279,
+       0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,
+       0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab,
+       0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82,
+       0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db,
+       0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573,
+       0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0,
+       0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b,
+       0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790,
+       0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8,
+       0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4,
+       0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0,
+       0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7,
+       0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c,
+       0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad,
+       0x2f2f2218, 0xbe0e1777, 0xea752dfe, 0x8b021fa1,
+       0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299,
+       0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9,
+       0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477,
+       0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,
+       0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49,
+       0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af,
+       0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa,
+       0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5,
+       0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41,
+       0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915,
+       0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400,
+       0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915,
+       0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664,
+       0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a,
+       0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623,
+       0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266,
+       0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1,
+       0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e,
+       0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6,
+       0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1,
+       0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e,
+       0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,
+       0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737,
+       0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8,
+       0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff,
+       0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd,
+       0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701,
+       0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7,
+       0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41,
+       0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331,
+       0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf,
+       0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af,
+       0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e,
+       0x5512721f, 0x2e6b7124, 0x501adde6, 0x9f84cd87,
+       0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c,
+       0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2,
+       0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16,
+       0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd,
+       0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b,
+       0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509,
+       0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e,
+       0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3,
+       0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f,
+       0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a,
+       0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4,
+       0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960,
+       0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66,
+       0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28,
+       0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802,
+       0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84,
+       0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510,
+       0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf,
+       0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14,
+       0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e,
+       0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50,
+       0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7,
+       0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8,
+       0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281,
+       0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99,
+       0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696,
+       0xcdb30aeb, 0x532e3054, 0x8fd948e4, 0x6dbc3128,
+       0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73,
+       0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0,
+       0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0,
+       0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105,
+       0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250,
+       0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3,
+       0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285,
+       0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00,
+       0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061,
+       0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb,
+       0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e,
+       0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735,
+       0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc,
+       0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9,
+       0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340,
+       0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20,
+       0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7,
+       0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934,
+       0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068,
+       0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af,
+       0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840,
+       0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45,
+       0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504,
+       0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a,
+       0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb,
+       0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee,
+       0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6,
+       0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42,
+       0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b,
+       0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2,
+       0x3a6efa74, 0xdd5b4332, 0x6841e7f7, 0xca7820fb,
+       0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527,
+       0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b,
+       0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33,
+       0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c,
+       0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3,
+       0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc,
+       0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17,
+       0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564,
+       0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b,
+       0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115,
+       0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922,
+       0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728,
+       0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0,
+       0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e,
+       0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37,
+       0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d,
+       0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804,
+       0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b,
+       0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3,
+       0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb,
+       0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d,
+       0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c,
+       0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350,
+       0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9,
+       0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a,
+       0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe,
+       0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d,
+       0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc,
+       0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f,
+       0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61,
+       0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2,
+       0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9,
+       0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2,
+       0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c,
+       0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e,
+       0xb77f19b6, 0xe0a9dc09, 0x662d09a1, 0xc4324633,
+       0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10,
+       0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169,
+       0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52,
+       0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027,
+       0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5,
+       0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62,
+       0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634,
+       0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76,
+       0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24,
+       0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc,
+       0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4,
+       0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c,
+       0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837,
+       0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0,
+       0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b,
+       0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe,
+       0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b,
+       0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4,
+       0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8,
+       0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6,
+       0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304,
+       0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22,
+       0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4,
+       0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6,
+       0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9,
+       0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59,
+       0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593,
+       0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51,
+       0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28,
+       0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c,
+       0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b,
+       0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28,
+       0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c,
+       0x15056dd4, 0x88f46dba, 0x03a16125, 0x0564f0bd,
+       0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a,
+       0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319,
+       0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb,
+       0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f,
+       0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991,
+       0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32,
+       0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680,
+       0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166,
+       0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae,
+       0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb,
+       0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5,
+       0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47,
+       0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370,
+       0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d,
+       0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84,
+       0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048,
+       0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8,
+       0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd,
+       0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9,
+       0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7,
+       0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38,
+       0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f,
+       0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c,
+       0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525,
+       0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1,
+       0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442,
+       0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, 0x3278e964,
+       0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e,
+       0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8,
+       0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d,
+       0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f,
+       0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299,
+       0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02,
+       0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc,
+       0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614,
+       0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a,
+       0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6,
+       0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b,
+       0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0,
+       0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060,
+       0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e,
+       0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9,
+       0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f,
+       0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6,
+};
+
+/*
+ * Round loop unrolling macros, S is a pointer to a S-Box array
+ * organized in 4 unsigned longs at a row.
+ */
+#define GET32_3(x) (((x) & 0xff))
+#define GET32_2(x) (((x) >> (8)) & (0xff))
+#define GET32_1(x) (((x) >> (16)) & (0xff))
+#define GET32_0(x) (((x) >> (24)) & (0xff))
+
+#define bf_F(x) (((S[GET32_0(x)] + S[256 + GET32_1(x)]) ^ \
+               S[512 + GET32_2(x)]) + S[768 + GET32_3(x)])
+
+#define ROUND(a, b, n) ({ b ^= P[n]; a ^= bf_F(b); })
+
+/*
+ * The blowfish encipher, processes 64-bit blocks.
+ * NOTE: This function MUSTN'T respect endianess
+ */
+static void encrypt_block(struct bf_ctx *bctx, u32 *dst, u32 *src)
+{
+       const u32 *P = bctx->p;
+       const u32 *S = bctx->s;
+       u32 yl = src[0];
+       u32 yr = src[1];
+
+       ROUND(yr, yl, 0);
+       ROUND(yl, yr, 1);
+       ROUND(yr, yl, 2);
+       ROUND(yl, yr, 3);
+       ROUND(yr, yl, 4);
+       ROUND(yl, yr, 5);
+       ROUND(yr, yl, 6);
+       ROUND(yl, yr, 7);
+       ROUND(yr, yl, 8);
+       ROUND(yl, yr, 9);
+       ROUND(yr, yl, 10);
+       ROUND(yl, yr, 11);
+       ROUND(yr, yl, 12);
+       ROUND(yl, yr, 13);
+       ROUND(yr, yl, 14);
+       ROUND(yl, yr, 15);
+
+       yl ^= P[16];
+       yr ^= P[17];
+
+       dst[0] = yr;
+       dst[1] = yl;
+}
+
+/*
+ * Calculates the blowfish S and P boxes for encryption and decryption.
+ */
+int blowfish_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen)
+{
+       struct bf_ctx *ctx = crypto_tfm_ctx(tfm);
+       u32 *P = ctx->p;
+       u32 *S = ctx->s;
+       short i, j, count;
+       u32 data[2], temp;
+
+       /* Copy the initialization s-boxes */
+       for (i = 0, count = 0; i < 256; i++)
+               for (j = 0; j < 4; j++, count++)
+                       S[count] = bf_sbox[count];
+
+       /* Set the p-boxes */
+       for (i = 0; i < 16 + 2; i++)
+               P[i] = bf_pbox[i];
+
+       /* Actual subkey generation */
+       for (j = 0, i = 0; i < 16 + 2; i++) {
+               temp = (((u32)key[j] << 24) |
+                       ((u32)key[(j + 1) % keylen] << 16) |
+                       ((u32)key[(j + 2) % keylen] << 8) |
+                       ((u32)key[(j + 3) % keylen]));
+
+               P[i] = P[i] ^ temp;
+               j = (j + 4) % keylen;
+       }
+
+       data[0] = 0x00000000;
+       data[1] = 0x00000000;
+
+       for (i = 0; i < 16 + 2; i += 2) {
+               encrypt_block((struct bf_ctx *)ctx, data, data);
+
+               P[i] = data[0];
+               P[i + 1] = data[1];
+       }
+
+       for (i = 0; i < 4; i++) {
+               for (j = 0, count = i * 256; j < 256; j += 2, count += 2) {
+                       encrypt_block((struct bf_ctx *)ctx, data, data);
+
+                       S[count] = data[0];
+                       S[count + 1] = data[1];
+               }
+       }
+
+       /* Bruce says not to bother with the weak key check. */
+       return 0;
+}
+EXPORT_SYMBOL_GPL(blowfish_setkey);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Blowfish Cipher common functions");
diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c
new file mode 100644 (file)
index 0000000..6f269b5
--- /dev/null
@@ -0,0 +1,142 @@
+/*
+ * Cryptographic API.
+ *
+ * Blowfish Cipher Algorithm, by Bruce Schneier.
+ * http://www.counterpane.com/blowfish.html
+ *
+ * Adapted from Kerneli implementation.
+ *
+ * Copyright (c) Herbert Valerio Riedel <hvr@hvrlab.org>
+ * Copyright (c) Kyle McMartin <kyle@debian.org>
+ * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <asm/byteorder.h>
+#include <linux/crypto.h>
+#include <linux/types.h>
+#include <crypto/blowfish.h>
+
+/*
+ * Round loop unrolling macros, S is a pointer to a S-Box array
+ * organized in 4 unsigned longs at a row.
+ */
+#define GET32_3(x) (((x) & 0xff))
+#define GET32_2(x) (((x) >> (8)) & (0xff))
+#define GET32_1(x) (((x) >> (16)) & (0xff))
+#define GET32_0(x) (((x) >> (24)) & (0xff))
+
+#define bf_F(x) (((S[GET32_0(x)] + S[256 + GET32_1(x)]) ^ \
+               S[512 + GET32_2(x)]) + S[768 + GET32_3(x)])
+
+#define ROUND(a, b, n) ({ b ^= P[n]; a ^= bf_F(b); })
+
+static void bf_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+       struct bf_ctx *ctx = crypto_tfm_ctx(tfm);
+       const __be32 *in_blk = (const __be32 *)src;
+       __be32 *const out_blk = (__be32 *)dst;
+       const u32 *P = ctx->p;
+       const u32 *S = ctx->s;
+       u32 yl = be32_to_cpu(in_blk[0]);
+       u32 yr = be32_to_cpu(in_blk[1]);
+
+       ROUND(yr, yl, 0);
+       ROUND(yl, yr, 1);
+       ROUND(yr, yl, 2);
+       ROUND(yl, yr, 3);
+       ROUND(yr, yl, 4);
+       ROUND(yl, yr, 5);
+       ROUND(yr, yl, 6);
+       ROUND(yl, yr, 7);
+       ROUND(yr, yl, 8);
+       ROUND(yl, yr, 9);
+       ROUND(yr, yl, 10);
+       ROUND(yl, yr, 11);
+       ROUND(yr, yl, 12);
+       ROUND(yl, yr, 13);
+       ROUND(yr, yl, 14);
+       ROUND(yl, yr, 15);
+
+       yl ^= P[16];
+       yr ^= P[17];
+
+       out_blk[0] = cpu_to_be32(yr);
+       out_blk[1] = cpu_to_be32(yl);
+}
+
+static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+       struct bf_ctx *ctx = crypto_tfm_ctx(tfm);
+       const __be32 *in_blk = (const __be32 *)src;
+       __be32 *const out_blk = (__be32 *)dst;
+       const u32 *P = ctx->p;
+       const u32 *S = ctx->s;
+       u32 yl = be32_to_cpu(in_blk[0]);
+       u32 yr = be32_to_cpu(in_blk[1]);
+
+       ROUND(yr, yl, 17);
+       ROUND(yl, yr, 16);
+       ROUND(yr, yl, 15);
+       ROUND(yl, yr, 14);
+       ROUND(yr, yl, 13);
+       ROUND(yl, yr, 12);
+       ROUND(yr, yl, 11);
+       ROUND(yl, yr, 10);
+       ROUND(yr, yl, 9);
+       ROUND(yl, yr, 8);
+       ROUND(yr, yl, 7);
+       ROUND(yl, yr, 6);
+       ROUND(yr, yl, 5);
+       ROUND(yl, yr, 4);
+       ROUND(yr, yl, 3);
+       ROUND(yl, yr, 2);
+
+       yl ^= P[1];
+       yr ^= P[0];
+
+       out_blk[0] = cpu_to_be32(yr);
+       out_blk[1] = cpu_to_be32(yl);
+}
+
+static struct crypto_alg alg = {
+       .cra_name               =       "blowfish",
+       .cra_driver_name        =       "blowfish-generic",
+       .cra_priority           =       100,
+       .cra_flags              =       CRYPTO_ALG_TYPE_CIPHER,
+       .cra_blocksize          =       BF_BLOCK_SIZE,
+       .cra_ctxsize            =       sizeof(struct bf_ctx),
+       .cra_alignmask          =       3,
+       .cra_module             =       THIS_MODULE,
+       .cra_list               =       LIST_HEAD_INIT(alg.cra_list),
+       .cra_u                  =       { .cipher = {
+       .cia_min_keysize        =       BF_MIN_KEY_SIZE,
+       .cia_max_keysize        =       BF_MAX_KEY_SIZE,
+       .cia_setkey             =       blowfish_setkey,
+       .cia_encrypt            =       bf_encrypt,
+       .cia_decrypt            =       bf_decrypt } }
+};
+
+static int __init blowfish_mod_init(void)
+{
+       return crypto_register_alg(&alg);
+}
+
+static void __exit blowfish_mod_fini(void)
+{
+       crypto_unregister_alg(&alg);
+}
+
+module_init(blowfish_mod_init);
+module_exit(blowfish_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Blowfish Cipher Algorithm");
+MODULE_ALIAS("blowfish");
index e46d21ae26bc0538003cf72b42fe1a8e90eb51cd..671d4d6d14df106b3b0278364340ef3290d7a185 100644 (file)
@@ -945,7 +945,7 @@ static void __exit cryptd_exit(void)
        crypto_unregister_template(&cryptd_tmpl);
 }
 
-module_init(cryptd_init);
+subsys_initcall(cryptd_init);
 module_exit(cryptd_exit);
 
 MODULE_LICENSE("GPL");
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
new file mode 100644 (file)
index 0000000..2abca78
--- /dev/null
@@ -0,0 +1,438 @@
+/*
+ * Crypto user configuration API.
+ *
+ * Copyright (C) 2011 secunet Security Networks AG
+ * Copyright (C) 2011 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <linux/module.h>
+#include <linux/crypto.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
+#include <linux/security.h>
+#include <net/net_namespace.h>
+#include "internal.h"
+
+DEFINE_MUTEX(crypto_cfg_mutex);
+
+/* The crypto netlink socket */
+static struct sock *crypto_nlsk;
+
+struct crypto_dump_info {
+       struct sk_buff *in_skb;
+       struct sk_buff *out_skb;
+       u32 nlmsg_seq;
+       u16 nlmsg_flags;
+};
+
+static struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact)
+{
+       struct crypto_alg *q, *alg = NULL;
+
+       down_read(&crypto_alg_sem);
+
+       if (list_empty(&crypto_alg_list))
+               return NULL;
+
+       list_for_each_entry(q, &crypto_alg_list, cra_list) {
+               int match = 0;
+
+               if ((q->cra_flags ^ p->cru_type) & p->cru_mask)
+                       continue;
+
+               if (strlen(p->cru_driver_name))
+                       match = !strcmp(q->cra_driver_name,
+                                       p->cru_driver_name);
+               else if (!exact)
+                       match = !strcmp(q->cra_name, p->cru_name);
+
+               if (match) {
+                       alg = q;
+                       break;
+               }
+       }
+
+       up_read(&crypto_alg_sem);
+
+       return alg;
+}
+
+static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_cipher rcipher;
+
+       snprintf(rcipher.type, CRYPTO_MAX_ALG_NAME, "%s", "cipher");
+
+       rcipher.blocksize = alg->cra_blocksize;
+       rcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
+       rcipher.max_keysize = alg->cra_cipher.cia_max_keysize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_CIPHER,
+               sizeof(struct crypto_report_cipher), &rcipher);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
+static int crypto_report_comp(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_comp rcomp;
+
+       snprintf(rcomp.type, CRYPTO_MAX_ALG_NAME, "%s", "compression");
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_COMPRESS,
+               sizeof(struct crypto_report_comp), &rcomp);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
+static int crypto_report_one(struct crypto_alg *alg,
+                            struct crypto_user_alg *ualg, struct sk_buff *skb)
+{
+       memcpy(&ualg->cru_name, &alg->cra_name, sizeof(ualg->cru_name));
+       memcpy(&ualg->cru_driver_name, &alg->cra_driver_name,
+              sizeof(ualg->cru_driver_name));
+       memcpy(&ualg->cru_module_name, module_name(alg->cra_module),
+              CRYPTO_MAX_ALG_NAME);
+
+       ualg->cru_flags = alg->cra_flags;
+       ualg->cru_refcnt = atomic_read(&alg->cra_refcnt);
+
+       NLA_PUT_U32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority);
+
+       if (alg->cra_flags & CRYPTO_ALG_LARVAL) {
+               struct crypto_report_larval rl;
+
+               snprintf(rl.type, CRYPTO_MAX_ALG_NAME, "%s", "larval");
+
+               NLA_PUT(skb, CRYPTOCFGA_REPORT_LARVAL,
+                       sizeof(struct crypto_report_larval), &rl);
+
+               goto out;
+       }
+
+       if (alg->cra_type && alg->cra_type->report) {
+               if (alg->cra_type->report(skb, alg))
+                       goto nla_put_failure;
+
+               goto out;
+       }
+
+       switch (alg->cra_flags & (CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_LARVAL)) {
+       case CRYPTO_ALG_TYPE_CIPHER:
+               if (crypto_report_cipher(skb, alg))
+                       goto nla_put_failure;
+
+               break;
+       case CRYPTO_ALG_TYPE_COMPRESS:
+               if (crypto_report_comp(skb, alg))
+                       goto nla_put_failure;
+
+               break;
+       }
+
+out:
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
+static int crypto_report_alg(struct crypto_alg *alg,
+                            struct crypto_dump_info *info)
+{
+       struct sk_buff *in_skb = info->in_skb;
+       struct sk_buff *skb = info->out_skb;
+       struct nlmsghdr *nlh;
+       struct crypto_user_alg *ualg;
+       int err = 0;
+
+       nlh = nlmsg_put(skb, NETLINK_CB(in_skb).pid, info->nlmsg_seq,
+                       CRYPTO_MSG_GETALG, sizeof(*ualg), info->nlmsg_flags);
+       if (!nlh) {
+               err = -EMSGSIZE;
+               goto out;
+       }
+
+       ualg = nlmsg_data(nlh);
+
+       err = crypto_report_one(alg, ualg, skb);
+       if (err) {
+               nlmsg_cancel(skb, nlh);
+               goto out;
+       }
+
+       nlmsg_end(skb, nlh);
+
+out:
+       return err;
+}
+
+static int crypto_report(struct sk_buff *in_skb, struct nlmsghdr *in_nlh,
+                        struct nlattr **attrs)
+{
+       struct crypto_user_alg *p = nlmsg_data(in_nlh);
+       struct crypto_alg *alg;
+       struct sk_buff *skb;
+       struct crypto_dump_info info;
+       int err;
+
+       if (!p->cru_driver_name)
+               return -EINVAL;
+
+       alg = crypto_alg_match(p, 1);
+       if (!alg)
+               return -ENOENT;
+
+       skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+       if (!skb)
+               return -ENOMEM;
+
+       info.in_skb = in_skb;
+       info.out_skb = skb;
+       info.nlmsg_seq = in_nlh->nlmsg_seq;
+       info.nlmsg_flags = 0;
+
+       err = crypto_report_alg(alg, &info);
+       if (err)
+               return err;
+
+       return nlmsg_unicast(crypto_nlsk, skb, NETLINK_CB(in_skb).pid);
+}
+
+static int crypto_dump_report(struct sk_buff *skb, struct netlink_callback *cb)
+{
+       struct crypto_alg *alg;
+       struct crypto_dump_info info;
+       int err;
+
+       if (cb->args[0])
+               goto out;
+
+       cb->args[0] = 1;
+
+       info.in_skb = cb->skb;
+       info.out_skb = skb;
+       info.nlmsg_seq = cb->nlh->nlmsg_seq;
+       info.nlmsg_flags = NLM_F_MULTI;
+
+       list_for_each_entry(alg, &crypto_alg_list, cra_list) {
+               err = crypto_report_alg(alg, &info);
+               if (err)
+                       goto out_err;
+       }
+
+out:
+       return skb->len;
+out_err:
+       return err;
+}
+
+static int crypto_dump_report_done(struct netlink_callback *cb)
+{
+       return 0;
+}
+
+static int crypto_update_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
+                            struct nlattr **attrs)
+{
+       struct crypto_alg *alg;
+       struct crypto_user_alg *p = nlmsg_data(nlh);
+       struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];
+       LIST_HEAD(list);
+
+       if (priority && !strlen(p->cru_driver_name))
+               return -EINVAL;
+
+       alg = crypto_alg_match(p, 1);
+       if (!alg)
+               return -ENOENT;
+
+       down_write(&crypto_alg_sem);
+
+       crypto_remove_spawns(alg, &list, NULL);
+
+       if (priority)
+               alg->cra_priority = nla_get_u32(priority);
+
+       up_write(&crypto_alg_sem);
+
+       crypto_remove_final(&list);
+
+       return 0;
+}
+
+static int crypto_del_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
+                         struct nlattr **attrs)
+{
+       struct crypto_alg *alg;
+       struct crypto_user_alg *p = nlmsg_data(nlh);
+
+       alg = crypto_alg_match(p, 1);
+       if (!alg)
+               return -ENOENT;
+
+       /* We can not unregister core algorithms such as aes-generic.
+        * We would loose the reference in the crypto_alg_list to this algorithm
+        * if we try to unregister. Unregistering such an algorithm without
+        * removing the module is not possible, so we restrict to crypto
+        * instances that are build from templates. */
+       if (!(alg->cra_flags & CRYPTO_ALG_INSTANCE))
+               return -EINVAL;
+
+       if (atomic_read(&alg->cra_refcnt) != 1)
+               return -EBUSY;
+
+       return crypto_unregister_alg(alg);
+}
+
+static int crypto_add_alg(struct sk_buff *skb, struct nlmsghdr *nlh,
+                         struct nlattr **attrs)
+{
+       int exact;
+       const char *name;
+       struct crypto_alg *alg;
+       struct crypto_user_alg *p = nlmsg_data(nlh);
+       struct nlattr *priority = attrs[CRYPTOCFGA_PRIORITY_VAL];
+
+       if (strlen(p->cru_driver_name))
+               exact = 1;
+
+       if (priority && !exact)
+               return -EINVAL;
+
+       alg = crypto_alg_match(p, exact);
+       if (alg)
+               return -EEXIST;
+
+       if (strlen(p->cru_driver_name))
+               name = p->cru_driver_name;
+       else
+               name = p->cru_name;
+
+       alg = crypto_alg_mod_lookup(name, p->cru_type, p->cru_mask);
+       if (IS_ERR(alg))
+               return PTR_ERR(alg);
+
+       down_write(&crypto_alg_sem);
+
+       if (priority)
+               alg->cra_priority = nla_get_u32(priority);
+
+       up_write(&crypto_alg_sem);
+
+       crypto_mod_put(alg);
+
+       return 0;
+}
+
+#define MSGSIZE(type) sizeof(struct type)
+
+static const int crypto_msg_min[CRYPTO_NR_MSGTYPES] = {
+       [CRYPTO_MSG_NEWALG      - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
+       [CRYPTO_MSG_DELALG      - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
+       [CRYPTO_MSG_UPDATEALG   - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
+       [CRYPTO_MSG_GETALG      - CRYPTO_MSG_BASE] = MSGSIZE(crypto_user_alg),
+};
+
+static const struct nla_policy crypto_policy[CRYPTOCFGA_MAX+1] = {
+       [CRYPTOCFGA_PRIORITY_VAL]   = { .type = NLA_U32},
+};
+
+#undef MSGSIZE
+
+static struct crypto_link {
+       int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **);
+       int (*dump)(struct sk_buff *, struct netlink_callback *);
+       int (*done)(struct netlink_callback *);
+} crypto_dispatch[CRYPTO_NR_MSGTYPES] = {
+       [CRYPTO_MSG_NEWALG      - CRYPTO_MSG_BASE] = { .doit = crypto_add_alg},
+       [CRYPTO_MSG_DELALG      - CRYPTO_MSG_BASE] = { .doit = crypto_del_alg},
+       [CRYPTO_MSG_UPDATEALG   - CRYPTO_MSG_BASE] = { .doit = crypto_update_alg},
+       [CRYPTO_MSG_GETALG      - CRYPTO_MSG_BASE] = { .doit = crypto_report,
+                                                      .dump = crypto_dump_report,
+                                                      .done = crypto_dump_report_done},
+};
+
+static int crypto_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
+{
+       struct nlattr *attrs[CRYPTOCFGA_MAX+1];
+       struct crypto_link *link;
+       int type, err;
+
+       type = nlh->nlmsg_type;
+       if (type > CRYPTO_MSG_MAX)
+               return -EINVAL;
+
+       type -= CRYPTO_MSG_BASE;
+       link = &crypto_dispatch[type];
+
+       if (security_netlink_recv(skb, CAP_NET_ADMIN))
+               return -EPERM;
+
+       if ((type == (CRYPTO_MSG_GETALG - CRYPTO_MSG_BASE) &&
+           (nlh->nlmsg_flags & NLM_F_DUMP))) {
+               if (link->dump == NULL)
+                       return -EINVAL;
+
+               return netlink_dump_start(crypto_nlsk, skb, nlh,
+                                         link->dump, link->done, 0);
+       }
+
+       err = nlmsg_parse(nlh, crypto_msg_min[type], attrs, CRYPTOCFGA_MAX,
+                         crypto_policy);
+       if (err < 0)
+               return err;
+
+       if (link->doit == NULL)
+               return -EINVAL;
+
+       return link->doit(skb, nlh, attrs);
+}
+
+static void crypto_netlink_rcv(struct sk_buff *skb)
+{
+       mutex_lock(&crypto_cfg_mutex);
+       netlink_rcv_skb(skb, &crypto_user_rcv_msg);
+       mutex_unlock(&crypto_cfg_mutex);
+}
+
+static int __init crypto_user_init(void)
+{
+       crypto_nlsk = netlink_kernel_create(&init_net, NETLINK_CRYPTO,
+                                           0, crypto_netlink_rcv,
+                                           NULL, THIS_MODULE);
+       if (!crypto_nlsk)
+               return -ENOMEM;
+
+       return 0;
+}
+
+static void __exit crypto_user_exit(void)
+{
+       netlink_kernel_release(crypto_nlsk);
+}
+
+module_init(crypto_user_init);
+module_exit(crypto_user_exit);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
+MODULE_DESCRIPTION("Crypto userspace configuration API");
index d4384b08ab29563b9d529666c2c01f5766c4cd5f..b865ca1a8613b33baa9b64ffa6c32b89039f3bf8 100644 (file)
@@ -86,6 +86,9 @@ struct crypto_alg *crypto_larval_lookup(const char *name, u32 type, u32 mask);
 void crypto_larval_error(const char *name, u32 type, u32 mask);
 void crypto_alg_tested(const char *name, int err);
 
+void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
+                         struct crypto_alg *nalg);
+void crypto_remove_final(struct list_head *list);
 void crypto_shoot_alg(struct crypto_alg *alg);
 struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
                                      u32 mask);
index f7c4a7d7412ee69600ddd018a3099e9e8bb51e69..fefda78a6a2aa925d29e776dc87a956a605a2ab2 100644 (file)
@@ -24,6 +24,8 @@
 #include <linux/module.h>
 #include <linux/seq_file.h>
 #include <linux/string.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 #include <crypto/compress.h>
 #include <crypto/internal/compress.h>
@@ -46,6 +48,21 @@ static int crypto_pcomp_init_tfm(struct crypto_tfm *tfm)
        return 0;
 }
 
+static int crypto_pcomp_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_comp rpcomp;
+
+       snprintf(rpcomp.type, CRYPTO_MAX_ALG_NAME, "%s", "pcomp");
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_COMPRESS,
+               sizeof(struct crypto_report_comp), &rpcomp);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_pcomp_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_pcomp_show(struct seq_file *m, struct crypto_alg *alg)
@@ -60,6 +77,7 @@ static const struct crypto_type crypto_pcomp_type = {
 #ifdef CONFIG_PROC_FS
        .show           = crypto_pcomp_show,
 #endif
+       .report         = crypto_pcomp_report,
        .maskclear      = ~CRYPTO_ALG_TYPE_MASK,
        .maskset        = CRYPTO_ALG_TYPE_MASK,
        .type           = CRYPTO_ALG_TYPE_PCOMPRESS,
index 45229ae782be91d4bec33185bccdee77ce95f1d5..feb7de00f437380fc8e95a3bf21398c9b4b1ce23 100644 (file)
@@ -21,6 +21,8 @@
 #include <linux/seq_file.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 static DEFINE_MUTEX(crypto_default_rng_lock);
 struct crypto_rng *crypto_default_rng;
@@ -58,6 +60,23 @@ static int crypto_init_rng_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
        return 0;
 }
 
+static int crypto_rng_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_rng rrng;
+
+       snprintf(rrng.type, CRYPTO_MAX_ALG_NAME, "%s", "rng");
+
+       rrng.seedsize = alg->cra_rng.seedsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_RNG,
+               sizeof(struct crypto_report_rng), &rrng);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg)
@@ -78,6 +97,7 @@ const struct crypto_type crypto_rng_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_rng_show,
 #endif
+       .report = crypto_rng_report,
 };
 EXPORT_SYMBOL_GPL(crypto_rng_type);
 
index 00ae60eb925435efe5295a9d7315283bd258e613..42794803c480531a60cc465657741a42ea5485dc 100644 (file)
@@ -36,7 +36,7 @@ static int sha1_init(struct shash_desc *desc)
        return 0;
 }
 
-static int sha1_update(struct shash_desc *desc, const u8 *data,
+int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
                        unsigned int len)
 {
        struct sha1_state *sctx = shash_desc_ctx(desc);
@@ -71,6 +71,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data,
 
        return 0;
 }
+EXPORT_SYMBOL(crypto_sha1_update);
 
 
 /* Add padding and return the message digest. */
@@ -87,10 +88,10 @@ static int sha1_final(struct shash_desc *desc, u8 *out)
        /* Pad out to 56 mod 64 */
        index = sctx->count & 0x3f;
        padlen = (index < 56) ? (56 - index) : ((64+56) - index);
-       sha1_update(desc, padding, padlen);
+       crypto_sha1_update(desc, padding, padlen);
 
        /* Append length */
-       sha1_update(desc, (const u8 *)&bits, sizeof(bits));
+       crypto_sha1_update(desc, (const u8 *)&bits, sizeof(bits));
 
        /* Store state in digest */
        for (i = 0; i < 5; i++)
@@ -121,7 +122,7 @@ static int sha1_import(struct shash_desc *desc, const void *in)
 static struct shash_alg alg = {
        .digestsize     =       SHA1_DIGEST_SIZE,
        .init           =       sha1_init,
-       .update         =       sha1_update,
+       .update         =       crypto_sha1_update,
        .final          =       sha1_final,
        .export         =       sha1_export,
        .import         =       sha1_import,
index 76f74b96315119c039970519597251ff1e729c39..ea8a9c6e21e371170606863ecb5f8cc7a5030ea8 100644 (file)
@@ -17,6 +17,8 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
+#include <linux/cryptouser.h>
+#include <net/netlink.h>
 
 #include "internal.h"
 
@@ -522,6 +524,24 @@ static unsigned int crypto_shash_extsize(struct crypto_alg *alg)
        return alg->cra_ctxsize;
 }
 
+static int crypto_shash_report(struct sk_buff *skb, struct crypto_alg *alg)
+{
+       struct crypto_report_hash rhash;
+       struct shash_alg *salg = __crypto_shash_alg(alg);
+
+       snprintf(rhash.type, CRYPTO_MAX_ALG_NAME, "%s", "shash");
+       rhash.blocksize = alg->cra_blocksize;
+       rhash.digestsize = salg->digestsize;
+
+       NLA_PUT(skb, CRYPTOCFGA_REPORT_HASH,
+               sizeof(struct crypto_report_hash), &rhash);
+
+       return 0;
+
+nla_put_failure:
+       return -EMSGSIZE;
+}
+
 static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg)
        __attribute__ ((unused));
 static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg)
@@ -541,6 +561,7 @@ static const struct crypto_type crypto_shash_type = {
 #ifdef CONFIG_PROC_FS
        .show = crypto_shash_show,
 #endif
+       .report = crypto_shash_report,
        .maskclear = ~CRYPTO_ALG_TYPE_MASK,
        .maskset = CRYPTO_ALG_TYPE_MASK,
        .type = CRYPTO_ALG_TYPE_SHASH,
index 2222617b3bedc631e28c5599bdbd5a7cef595413..0c4e80f34651bd01dabaed7632a42922fa3dca7e 100644 (file)
@@ -782,11 +782,13 @@ static int do_test(int m)
        case 7:
                ret += tcrypt_test("ecb(blowfish)");
                ret += tcrypt_test("cbc(blowfish)");
+               ret += tcrypt_test("ctr(blowfish)");
                break;
 
        case 8:
                ret += tcrypt_test("ecb(twofish)");
                ret += tcrypt_test("cbc(twofish)");
+               ret += tcrypt_test("ctr(twofish)");
                break;
 
        case 9:
@@ -1039,6 +1041,10 @@ static int do_test(int m)
                                speed_template_16_24_32);
                test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
                                speed_template_16_24_32);
+               test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
+                               speed_template_16_24_32);
+               test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
+                               speed_template_16_24_32);
                break;
 
        case 203:
@@ -1050,6 +1056,10 @@ static int do_test(int m)
                                  speed_template_8_32);
                test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
                                  speed_template_8_32);
+               test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
+                                 speed_template_8_32);
+               test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
+                                 speed_template_8_32);
                break;
 
        case 204:
index b6b93d41635149692f473d19fdc53118d4de29fe..e91c1eb1722a9b3c604f53d84cec306615761fd6 100644 (file)
@@ -1755,6 +1755,36 @@ static const struct alg_test_desc alg_test_descs[] = {
                                }
                        }
                }
+       }, {
+               .alg = "ctr(blowfish)",
+               .test = alg_test_skcipher,
+               .suite = {
+                       .cipher = {
+                               .enc = {
+                                       .vecs = bf_ctr_enc_tv_template,
+                                       .count = BF_CTR_ENC_TEST_VECTORS
+                               },
+                               .dec = {
+                                       .vecs = bf_ctr_dec_tv_template,
+                                       .count = BF_CTR_DEC_TEST_VECTORS
+                               }
+                       }
+               }
+       }, {
+               .alg = "ctr(twofish)",
+               .test = alg_test_skcipher,
+               .suite = {
+                       .cipher = {
+                               .enc = {
+                                       .vecs = tf_ctr_enc_tv_template,
+                                       .count = TF_CTR_ENC_TEST_VECTORS
+                               },
+                               .dec = {
+                                       .vecs = tf_ctr_dec_tv_template,
+                                       .count = TF_CTR_DEC_TEST_VECTORS
+                               }
+                       }
+               }
        }, {
                .alg = "cts(cbc(aes))",
                .test = alg_test_skcipher,
index 27adc92842bae0699da45335adabad7b9cd80fe9..37b4d8f4544750e520cda8cc56af887f632701a0 100644 (file)
@@ -2391,10 +2391,12 @@ static struct cipher_testvec des3_ede_cbc_dec_tv_template[] = {
 /*
  * Blowfish test vectors.
  */
-#define BF_ENC_TEST_VECTORS    6
-#define BF_DEC_TEST_VECTORS    6
-#define BF_CBC_ENC_TEST_VECTORS        1
-#define BF_CBC_DEC_TEST_VECTORS        1
+#define BF_ENC_TEST_VECTORS    7
+#define BF_DEC_TEST_VECTORS    7
+#define BF_CBC_ENC_TEST_VECTORS        2
+#define BF_CBC_DEC_TEST_VECTORS        2
+#define BF_CTR_ENC_TEST_VECTORS        2
+#define BF_CTR_DEC_TEST_VECTORS        2
 
 static struct cipher_testvec bf_enc_tv_template[] = {
        { /* DES test vectors from OpenSSL */
@@ -2448,6 +2450,24 @@ static struct cipher_testvec bf_enc_tv_template[] = {
                .ilen   = 8,
                .result = "\xc0\x45\x04\x01\x2e\x4e\x1f\x53",
                .rlen   = 8,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
+               .ilen   = 40,
+               .result = "\x96\x87\x3D\x0C\x7B\xFB\xBD\x1F"
+                         "\xE3\xC1\x99\x6D\x39\xD4\xC2\x7D"
+                         "\xD7\x87\xA1\xF2\xDF\x51\x71\x26"
+                         "\xC2\xF4\x6D\xFF\xF6\xCD\x6B\x40"
+                         "\xE1\xB3\xBF\xD4\x38\x2B\xC8\x3B",
+               .rlen   = 40,
        },
 };
 
@@ -2503,6 +2523,24 @@ static struct cipher_testvec bf_dec_tv_template[] = {
                .ilen   = 8,
                .result = "\xfe\xdc\xba\x98\x76\x54\x32\x10",
                .rlen   = 8,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .input  = "\x96\x87\x3D\x0C\x7B\xFB\xBD\x1F"
+                         "\xE3\xC1\x99\x6D\x39\xD4\xC2\x7D"
+                         "\xD7\x87\xA1\xF2\xDF\x51\x71\x26"
+                         "\xC2\xF4\x6D\xFF\xF6\xCD\x6B\x40"
+                         "\xE1\xB3\xBF\xD4\x38\x2B\xC8\x3B",
+               .ilen   = 40,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
+               .rlen   = 40,
        },
 };
 
@@ -2522,6 +2560,25 @@ static struct cipher_testvec bf_cbc_enc_tv_template[] = {
                          "\x58\xde\xb9\xe7\x15\x46\x16\xd9"
                          "\x59\xf1\x65\x2b\xd5\xff\x92\xcc",
                .rlen   = 32,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
+               .ilen   = 40,
+               .result = "\xB4\xFE\xA5\xBB\x3D\x2C\x27\x06"
+                         "\x06\x2B\x3A\x92\xB2\xF5\x5E\x62"
+                         "\x84\xCD\xF7\x66\x7E\x41\x6C\x8E"
+                         "\x1B\xD9\x02\xB6\x48\xB0\x87\x25"
+                         "\x01\x9C\x93\x63\x51\x60\x82\xD2",
+               .rlen   = 40,
        },
 };
 
@@ -2541,16 +2598,125 @@ static struct cipher_testvec bf_cbc_dec_tv_template[] = {
                          "\x68\x65\x20\x74\x69\x6d\x65\x20"
                          "\x66\x6f\x72\x20\x00\x00\x00\x00",
                .rlen   = 32,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+               .input  = "\xB4\xFE\xA5\xBB\x3D\x2C\x27\x06"
+                         "\x06\x2B\x3A\x92\xB2\xF5\x5E\x62"
+                         "\x84\xCD\xF7\x66\x7E\x41\x6C\x8E"
+                         "\x1B\xD9\x02\xB6\x48\xB0\x87\x25"
+                         "\x01\x9C\x93\x63\x51\x60\x82\xD2",
+               .ilen   = 40,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
+               .rlen   = 40,
+       },
+};
+
+static struct cipher_testvec bf_ctr_enc_tv_template[] = {
+       { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
+               .ilen   = 40,
+               .result = "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
+                         "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
+                         "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
+                         "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
+                         "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC",
+               .rlen   = 40,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B",
+               .ilen   = 43,
+               .result = "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
+                         "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
+                         "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
+                         "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
+                         "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC"
+                         "\x3D\xA7\xE9",
+               .rlen   = 43,
+       },
+};
+
+static struct cipher_testvec bf_ctr_dec_tv_template[] = {
+       { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+               .input  = "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
+                         "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
+                         "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
+                         "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
+                         "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC",
+               .ilen   = 40,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9",
+               .rlen   = 40,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F",
+               .input  = "\xC7\xA3\xDF\xB9\x05\xF4\x9E\x8D"
+                         "\x9E\xDF\x38\x18\x83\x07\xEF\xC1"
+                         "\x93\x3C\xAA\xAA\xFE\x06\x42\xCC"
+                         "\x0D\x70\x86\x5A\x44\xAD\x85\x17"
+                         "\xE4\x1F\x5E\xA5\x89\xAC\x32\xBC"
+                         "\x3D\xA7\xE9",
+               .ilen   = 43,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B",
+               .rlen   = 43,
        },
 };
 
 /*
  * Twofish test vectors.
  */
-#define TF_ENC_TEST_VECTORS            3
-#define TF_DEC_TEST_VECTORS            3
-#define TF_CBC_ENC_TEST_VECTORS                4
-#define TF_CBC_DEC_TEST_VECTORS                4
+#define TF_ENC_TEST_VECTORS            4
+#define TF_DEC_TEST_VECTORS            4
+#define TF_CBC_ENC_TEST_VECTORS                5
+#define TF_CBC_DEC_TEST_VECTORS                5
+#define TF_CTR_ENC_TEST_VECTORS                2
+#define TF_CTR_DEC_TEST_VECTORS                2
 
 static struct cipher_testvec tf_enc_tv_template[] = {
        {
@@ -2582,6 +2748,30 @@ static struct cipher_testvec tf_enc_tv_template[] = {
                .result = "\x37\x52\x7b\xe0\x05\x23\x34\xb8"
                          "\x9f\x0c\xfc\xca\xe8\x7c\xfa\x20",
                .rlen   = 16,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
+                         "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
+                         "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
+                         "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+               .klen   = 32,
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C",
+               .ilen   = 64,
+               .result = "\x88\xCB\x1E\xC2\xAF\x8A\x97\xFF"
+                         "\xF6\x90\x46\x9C\x4A\x0F\x08\xDC"
+                         "\xDE\xAB\xAD\xFA\xFC\xA8\xC2\x3D"
+                         "\xE0\xE4\x8B\x3F\xD5\xA3\xF7\x14"
+                         "\x34\x9E\xB6\x08\xB2\xDD\xA8\xF5"
+                         "\xDF\xFA\xC7\xE8\x09\x50\x76\x08"
+                         "\xA2\xB6\x6A\x59\xC0\x2B\x6D\x05"
+                         "\x89\xF6\x82\xF0\xD3\xDB\x06\x02",
+               .rlen   = 64,
        },
 };
 
@@ -2615,6 +2805,30 @@ static struct cipher_testvec tf_dec_tv_template[] = {
                .ilen   = 16,
                .result = zeroed_string,
                .rlen   = 16,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x3F\x85\x62\x3F\x1C\xF9\xD6\x1C"
+                         "\xF9\xD6\xB3\x90\x6D\x4A\x90\x6D"
+                         "\x4A\x27\x04\xE1\x27\x04\xE1\xBE"
+                         "\x9B\x78\xBE\x9B\x78\x55\x32\x0F",
+               .klen   = 32,
+               .input  = "\x88\xCB\x1E\xC2\xAF\x8A\x97\xFF"
+                         "\xF6\x90\x46\x9C\x4A\x0F\x08\xDC"
+                         "\xDE\xAB\xAD\xFA\xFC\xA8\xC2\x3D"
+                         "\xE0\xE4\x8B\x3F\xD5\xA3\xF7\x14"
+                         "\x34\x9E\xB6\x08\xB2\xDD\xA8\xF5"
+                         "\xDF\xFA\xC7\xE8\x09\x50\x76\x08"
+                         "\xA2\xB6\x6A\x59\xC0\x2B\x6D\x05"
+                         "\x89\xF6\x82\xF0\xD3\xDB\x06\x02",
+               .ilen   = 64,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C",
+               .rlen   = 64,
        },
 };
 
@@ -2661,6 +2875,32 @@ static struct cipher_testvec tf_cbc_enc_tv_template[] = {
                          "\x05\xef\x8c\x61\xa8\x11\x58\x26"
                          "\x34\xba\x5c\xb7\x10\x6a\xa6\x41",
                .rlen   = 48,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+                         "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C",
+               .ilen   = 64,
+               .result = "\xC8\xFF\xF2\x53\xA6\x27\x09\xD1"
+                         "\x33\x38\xC2\xC0\x0C\x14\x7E\xB5"
+                         "\x26\x1B\x05\x0C\x05\x12\x3F\xC0"
+                         "\xF9\x1C\x02\x28\x40\x96\x6F\xD0"
+                         "\x3D\x32\xDF\xDA\x56\x00\x6E\xEE"
+                         "\x5B\x2A\x72\x9D\xC2\x4D\x19\xBC"
+                         "\x8C\x53\xFA\x87\x6F\xDD\x81\xA3"
+                         "\xB1\xD3\x44\x65\xDF\xE7\x63\x38",
+               .rlen   = 64,
        },
 };
 
@@ -2707,6 +2947,148 @@ static struct cipher_testvec tf_cbc_dec_tv_template[] = {
                .ilen   = 48,
                .result = zeroed_string,
                .rlen   = 48,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+                         "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+               .input  = "\xC8\xFF\xF2\x53\xA6\x27\x09\xD1"
+                         "\x33\x38\xC2\xC0\x0C\x14\x7E\xB5"
+                         "\x26\x1B\x05\x0C\x05\x12\x3F\xC0"
+                         "\xF9\x1C\x02\x28\x40\x96\x6F\xD0"
+                         "\x3D\x32\xDF\xDA\x56\x00\x6E\xEE"
+                         "\x5B\x2A\x72\x9D\xC2\x4D\x19\xBC"
+                         "\x8C\x53\xFA\x87\x6F\xDD\x81\xA3"
+                         "\xB1\xD3\x44\x65\xDF\xE7\x63\x38",
+               .ilen   = 64,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C",
+               .rlen   = 64,
+       },
+};
+
+static struct cipher_testvec tf_ctr_enc_tv_template[] = {
+       { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+                         "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C",
+               .ilen   = 64,
+               .result = "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
+                         "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
+                         "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
+                         "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
+                         "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
+                         "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
+                         "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
+                         "\x01\x41\x21\x12\x38\xAB\x52\x4F",
+               .rlen   = 64,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+                         "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+               .input  = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+                         "\xC3\x37\xCE",
+               .ilen   = 67,
+               .result = "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
+                         "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
+                         "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
+                         "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
+                         "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
+                         "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
+                         "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
+                         "\x01\x41\x21\x12\x38\xAB\x52\x4F"
+                         "\xA8\x57\x20",
+               .rlen   = 67,
+       },
+};
+
+static struct cipher_testvec tf_ctr_dec_tv_template[] = {
+       { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+                         "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+               .input  = "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
+                         "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
+                         "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
+                         "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
+                         "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
+                         "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
+                         "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
+                         "\x01\x41\x21\x12\x38\xAB\x52\x4F",
+               .ilen   = 64,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C",
+               .rlen   = 64,
+       }, { /* Generated with Crypto++ */
+               .key    = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9"
+                         "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A"
+                         "\x27\x04\xE1\x27\x04\xE1\xBE\x9B"
+                         "\x78\xBE\x9B\x78\x55\x32\x0F\x55",
+               .klen   = 32,
+               .iv     = "\xE2\x24\x89\xEE\x53\xB8\x1D\x5F"
+                         "\xC4\x29\x8E\xF3\x35\x9A\xFF\x64",
+               .input  = "\xDF\xDD\x69\xFA\xB0\x2E\xFD\xFE"
+                         "\x70\x9E\xC5\x4B\xC9\xD4\xA1\x30"
+                         "\x26\x9B\x89\xA1\xEE\x43\xE0\x52"
+                         "\x55\x17\x4E\xC7\x0E\x33\x1F\xF1"
+                         "\x9F\x8D\x40\x9F\x24\xFD\x92\xA0"
+                         "\xBC\x8F\x35\xDD\x67\x38\xD8\xAA"
+                         "\xCF\xF8\x48\xCA\xFB\xE4\x5C\x60"
+                         "\x01\x41\x21\x12\x38\xAB\x52\x4F"
+                         "\xA8\x57\x20",
+               .ilen   = 67,
+               .result = "\x56\xED\x84\x1B\x8F\x26\xBD\x31"
+                         "\xC8\x5F\xF6\x6A\x01\x98\x0C\xA3"
+                         "\x3A\xD1\x45\xDC\x73\x0A\x7E\x15"
+                         "\xAC\x20\xB7\x4E\xE5\x59\xF0\x87"
+                         "\x1E\x92\x29\xC0\x34\xCB\x62\xF9"
+                         "\x6D\x04\x9B\x0F\xA6\x3D\xD4\x48"
+                         "\xDF\x76\x0D\x81\x18\xAF\x23\xBA"
+                         "\x51\xE8\x5C\xF3\x8A\x21\x95\x2C"
+                         "\xC3\x37\xCE",
+               .rlen   = 67,
        },
 };
 
index 723427273687011b3c4e8fd995e1ec8d1f6546d8..71719a2be25afa5ca341bd65306ec976b52d42b2 100644 (file)
@@ -762,11 +762,17 @@ static const u64 C7[256] = {
        0x86228644a411c286ULL,
 };
 
-static const u64 rc[WHIRLPOOL_ROUNDS + 1] = {
-       0x0000000000000000ULL, 0x1823c6e887b8014fULL, 0x36a6d2f5796f9152ULL,
-       0x60bc9b8ea30c7b35ULL, 0x1de0d7c22e4bfe57ULL, 0x157737e59ff04adaULL,
-       0x58c9290ab1a06b85ULL, 0xbd5d10f4cb3e0567ULL, 0xe427418ba77d95d8ULL,
-       0xfbee7c66dd17479eULL, 0xca2dbf07ad5a8333ULL,
+static const u64 rc[WHIRLPOOL_ROUNDS] = {
+       0x1823c6e887b8014fULL,
+       0x36a6d2f5796f9152ULL,
+       0x60bc9b8ea30c7b35ULL,
+       0x1de0d7c22e4bfe57ULL,
+       0x157737e59ff04adaULL,
+       0x58c9290ab1a06b85ULL,
+       0xbd5d10f4cb3e0567ULL,
+       0xe427418ba77d95d8ULL,
+       0xfbee7c66dd17479eULL,
+       0xca2dbf07ad5a8333ULL,
 };
 
 /**
@@ -793,7 +799,7 @@ static void wp512_process_buffer(struct wp512_ctx *wctx) {
        state[6] = block[6] ^ (K[6] = wctx->hash[6]);
        state[7] = block[7] ^ (K[7] = wctx->hash[7]);
 
-       for (r = 1; r <= WHIRLPOOL_ROUNDS; r++) {
+       for (r = 0; r < WHIRLPOOL_ROUNDS; r++) {
 
                L[0] = C0[(int)(K[0] >> 56)       ] ^
                           C1[(int)(K[7] >> 48) & 0xff] ^
index e0b25de1e339249773287c13d8f457d1f9e94352..6d16b4b0d7a09bb2b2b3d6d77a230401388b89d6 100644 (file)
@@ -200,6 +200,7 @@ config CRYPTO_DEV_HIFN_795X
        select CRYPTO_BLKCIPHER
        select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG
        depends on PCI
+       depends on !ARCH_DMA_ADDR_T_64BIT
        help
          This option allows you to have support for HIFN 795x crypto adapters.
 
@@ -266,7 +267,7 @@ config CRYPTO_DEV_OMAP_AES
 
 config CRYPTO_DEV_PICOXCELL
        tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
-       depends on ARCH_PICOXCELL
+       depends on ARCH_PICOXCELL && HAVE_CLK
        select CRYPTO_AES
        select CRYPTO_AUTHENC
        select CRYPTO_ALGAPI
index a84250a5dd5111f08f79c58c9ff34875b744ff65..fe765f49de589a1229205447e476678da5d0ff6b 100644 (file)
@@ -2744,10 +2744,8 @@ static int __init hifn_init(void)
        unsigned int freq;
        int err;
 
-       if (sizeof(dma_addr_t) > 4) {
-               printk(KERN_INFO "HIFN supports only 32-bit addresses.\n");
-               return -EINVAL;
-       }
+       /* HIFN supports only 32-bit addresses */
+       BUILD_BUG_ON(sizeof(dma_addr_t) != 4);
 
        if (strncmp(hifn_pll_ref, "ext", 3) &&
            strncmp(hifn_pll_ref, "pci", 3)) {
index d0183ddb3076d1dec3b668e355e2967eb5046bc7..8944dabc0e3c79cae9e960a89cbb7b937072912a 100644 (file)
@@ -1006,9 +1006,9 @@ static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
 
        spin_unlock_irqrestore(&qp->lock, flags);
 
+out:
        put_cpu();
 
-out:
        n2_chunk_complete(req, NULL);
        return err;
 }
@@ -1096,9 +1096,9 @@ static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
 
        spin_unlock_irqrestore(&qp->lock, flags);
 
+out:
        put_cpu();
 
-out:
        n2_chunk_complete(req, err ? NULL : final_iv_addr);
        return err;
 }
index db33d300aa232a4dec0c4234b2ab768a5dfc7023..29b9469f83789eb42ee8c5d47ca7901d85fb70af 100644 (file)
@@ -508,10 +508,8 @@ static int __init padlock_init(void)
        int ret;
        struct cpuinfo_x86 *c = &cpu_data(0);
 
-       if (!cpu_has_xcrypt) {
-               printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
+       if (!cpu_has_xcrypt)
                return -ENODEV;
-       }
 
        if (!cpu_has_xcrypt_enabled) {
                printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
index 230b5b8cda1f9b2ff82bbd410a5f27eb026b805f..a2b553eabbdb6db38839ba1f14cc26a61edb31d2 100644 (file)
@@ -34,6 +34,7 @@
 #include <linux/io.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
 #include <linux/rtnetlink.h>
@@ -1241,8 +1242,8 @@ static void spacc_spacc_complete(unsigned long data)
        spin_unlock_irqrestore(&engine->hw_lock, flags);
 
        list_for_each_entry_safe(req, tmp, &completed, list) {
-               req->complete(req);
                list_del(&req->list);
+               req->complete(req);
        }
 }
 
@@ -1657,10 +1658,33 @@ static struct spacc_alg l2_engine_algs[] = {
        },
 };
 
-static int __devinit spacc_probe(struct platform_device *pdev,
-                                unsigned max_ctxs, size_t cipher_pg_sz,
-                                size_t hash_pg_sz, size_t fifo_sz,
-                                struct spacc_alg *algs, size_t num_algs)
+#ifdef CONFIG_OF
+static const struct of_device_id spacc_of_id_table[] = {
+       { .compatible = "picochip,spacc-ipsec" },
+       { .compatible = "picochip,spacc-l2" },
+       {}
+};
+#else /* CONFIG_OF */
+#define spacc_of_id_table NULL
+#endif /* CONFIG_OF */
+
+static bool spacc_is_compatible(struct platform_device *pdev,
+                               const char *spacc_type)
+{
+       const struct platform_device_id *platid = platform_get_device_id(pdev);
+
+       if (platid && !strcmp(platid->name, spacc_type))
+               return true;
+
+#ifdef CONFIG_OF
+       if (of_device_is_compatible(pdev->dev.of_node, spacc_type))
+               return true;
+#endif /* CONFIG_OF */
+
+       return false;
+}
+
+static int __devinit spacc_probe(struct platform_device *pdev)
 {
        int i, err, ret = -EINVAL;
        struct resource *mem, *irq;
@@ -1669,13 +1693,25 @@ static int __devinit spacc_probe(struct platform_device *pdev,
        if (!engine)
                return -ENOMEM;
 
-       engine->max_ctxs        = max_ctxs;
-       engine->cipher_pg_sz    = cipher_pg_sz;
-       engine->hash_pg_sz      = hash_pg_sz;
-       engine->fifo_sz         = fifo_sz;
-       engine->algs            = algs;
-       engine->num_algs        = num_algs;
-       engine->name            = dev_name(&pdev->dev);
+       if (spacc_is_compatible(pdev, "picochip,spacc-ipsec")) {
+               engine->max_ctxs        = SPACC_CRYPTO_IPSEC_MAX_CTXS;
+               engine->cipher_pg_sz    = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;
+               engine->hash_pg_sz      = SPACC_CRYPTO_IPSEC_HASH_PG_SZ;
+               engine->fifo_sz         = SPACC_CRYPTO_IPSEC_FIFO_SZ;
+               engine->algs            = ipsec_engine_algs;
+               engine->num_algs        = ARRAY_SIZE(ipsec_engine_algs);
+       } else if (spacc_is_compatible(pdev, "picochip,spacc-l2")) {
+               engine->max_ctxs        = SPACC_CRYPTO_L2_MAX_CTXS;
+               engine->cipher_pg_sz    = SPACC_CRYPTO_L2_CIPHER_PG_SZ;
+               engine->hash_pg_sz      = SPACC_CRYPTO_L2_HASH_PG_SZ;
+               engine->fifo_sz         = SPACC_CRYPTO_L2_FIFO_SZ;
+               engine->algs            = l2_engine_algs;
+               engine->num_algs        = ARRAY_SIZE(l2_engine_algs);
+       } else {
+               return -EINVAL;
+       }
+
+       engine->name = dev_name(&pdev->dev);
 
        mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
        irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
@@ -1711,7 +1747,7 @@ static int __devinit spacc_probe(struct platform_device *pdev,
 
        spin_lock_init(&engine->hw_lock);
 
-       engine->clk = clk_get(&pdev->dev, NULL);
+       engine->clk = clk_get(&pdev->dev, "ref");
        if (IS_ERR(engine->clk)) {
                dev_info(&pdev->dev, "clk unavailable\n");
                device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
@@ -1800,72 +1836,33 @@ static int __devexit spacc_remove(struct platform_device *pdev)
        return 0;
 }
 
-static int __devinit ipsec_probe(struct platform_device *pdev)
-{
-       return spacc_probe(pdev, SPACC_CRYPTO_IPSEC_MAX_CTXS,
-                          SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ,
-                          SPACC_CRYPTO_IPSEC_HASH_PG_SZ,
-                          SPACC_CRYPTO_IPSEC_FIFO_SZ, ipsec_engine_algs,
-                          ARRAY_SIZE(ipsec_engine_algs));
-}
-
-static struct platform_driver ipsec_driver = {
-       .probe          = ipsec_probe,
-       .remove         = __devexit_p(spacc_remove),
-       .driver         = {
-               .name   = "picoxcell-ipsec",
-#ifdef CONFIG_PM
-               .pm     = &spacc_pm_ops,
-#endif /* CONFIG_PM */
-       },
+static const struct platform_device_id spacc_id_table[] = {
+       { "picochip,spacc-ipsec", },
+       { "picochip,spacc-l2", },
 };
 
-static int __devinit l2_probe(struct platform_device *pdev)
-{
-       return spacc_probe(pdev, SPACC_CRYPTO_L2_MAX_CTXS,
-                          SPACC_CRYPTO_L2_CIPHER_PG_SZ,
-                          SPACC_CRYPTO_L2_HASH_PG_SZ, SPACC_CRYPTO_L2_FIFO_SZ,
-                          l2_engine_algs, ARRAY_SIZE(l2_engine_algs));
-}
-
-static struct platform_driver l2_driver = {
-       .probe          = l2_probe,
+static struct platform_driver spacc_driver = {
+       .probe          = spacc_probe,
        .remove         = __devexit_p(spacc_remove),
        .driver         = {
-               .name   = "picoxcell-l2",
+               .name   = "picochip,spacc",
 #ifdef CONFIG_PM
                .pm     = &spacc_pm_ops,
 #endif /* CONFIG_PM */
+               .of_match_table = spacc_of_id_table,
        },
+       .id_table       = spacc_id_table,
 };
 
 static int __init spacc_init(void)
 {
-       int ret = platform_driver_register(&ipsec_driver);
-       if (ret) {
-               pr_err("failed to register ipsec spacc driver");
-               goto out;
-       }
-
-       ret = platform_driver_register(&l2_driver);
-       if (ret) {
-               pr_err("failed to register l2 spacc driver");
-               goto l2_failed;
-       }
-
-       return 0;
-
-l2_failed:
-       platform_driver_unregister(&ipsec_driver);
-out:
-       return ret;
+       return platform_driver_register(&spacc_driver);
 }
 module_init(spacc_init);
 
 static void __exit spacc_exit(void)
 {
-       platform_driver_unregister(&ipsec_driver);
-       platform_driver_unregister(&l2_driver);
+       platform_driver_unregister(&spacc_driver);
 }
 module_exit(spacc_exit);
 
index 8a0bb417aa1120da0af3173b2a81acee64b0a348..dbe76b5df9cf2550b2e2621e2b051cf490396f76 100644 (file)
@@ -416,7 +416,7 @@ static void talitos_done(unsigned long data)
 /*
  * locate current (offending) descriptor
  */
-static struct talitos_desc *current_desc(struct device *dev, int ch)
+static u32 current_desc_hdr(struct device *dev, int ch)
 {
        struct talitos_private *priv = dev_get_drvdata(dev);
        int tail = priv->chan[ch].tail;
@@ -428,23 +428,25 @@ static struct talitos_desc *current_desc(struct device *dev, int ch)
                tail = (tail + 1) & (priv->fifo_len - 1);
                if (tail == priv->chan[ch].tail) {
                        dev_err(dev, "couldn't locate current descriptor\n");
-                       return NULL;
+                       return 0;
                }
        }
 
-       return priv->chan[ch].fifo[tail].desc;
+       return priv->chan[ch].fifo[tail].desc->hdr;
 }
 
 /*
  * user diagnostics; report root cause of error based on execution unit status
  */
-static void report_eu_error(struct device *dev, int ch,
-                           struct talitos_desc *desc)
+static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
 {
        struct talitos_private *priv = dev_get_drvdata(dev);
        int i;
 
-       switch (desc->hdr & DESC_HDR_SEL0_MASK) {
+       if (!desc_hdr)
+               desc_hdr = in_be32(priv->reg + TALITOS_DESCBUF(ch));
+
+       switch (desc_hdr & DESC_HDR_SEL0_MASK) {
        case DESC_HDR_SEL0_AFEU:
                dev_err(dev, "AFEUISR 0x%08x_%08x\n",
                        in_be32(priv->reg + TALITOS_AFEUISR),
@@ -488,7 +490,7 @@ static void report_eu_error(struct device *dev, int ch,
                break;
        }
 
-       switch (desc->hdr & DESC_HDR_SEL1_MASK) {
+       switch (desc_hdr & DESC_HDR_SEL1_MASK) {
        case DESC_HDR_SEL1_MDEUA:
        case DESC_HDR_SEL1_MDEUB:
                dev_err(dev, "MDEUISR 0x%08x_%08x\n",
@@ -550,7 +552,7 @@ static void talitos_error(unsigned long data, u32 isr, u32 isr_lo)
                if (v_lo & TALITOS_CCPSR_LO_IEU)
                        dev_err(dev, "invalid execution unit error\n");
                if (v_lo & TALITOS_CCPSR_LO_EU)
-                       report_eu_error(dev, ch, current_desc(dev, ch));
+                       report_eu_error(dev, ch, current_desc_hdr(dev, ch));
                if (v_lo & TALITOS_CCPSR_LO_GB)
                        dev_err(dev, "gather boundary error\n");
                if (v_lo & TALITOS_CCPSR_LO_GRL)
index 59c3e5bd2c064cb59baa4248f75618f15270b543..ecc721def10c8ae2a6ba4dee6fead57d20af48f0 100644 (file)
@@ -15,6 +15,7 @@
 #include <linux/crypto.h>
 #include <linux/list.h>
 #include <linux/kernel.h>
+#include <linux/skbuff.h>
 
 struct module;
 struct rtattr;
@@ -26,6 +27,7 @@ struct crypto_type {
        int (*init)(struct crypto_tfm *tfm, u32 type, u32 mask);
        int (*init_tfm)(struct crypto_tfm *tfm);
        void (*show)(struct seq_file *m, struct crypto_alg *alg);
+       int (*report)(struct sk_buff *skb, struct crypto_alg *alg);
        struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask);
 
        unsigned int type;
diff --git a/include/crypto/blowfish.h b/include/crypto/blowfish.h
new file mode 100644 (file)
index 0000000..1450d4a
--- /dev/null
@@ -0,0 +1,23 @@
+/*
+ * Common values for blowfish algorithms
+ */
+
+#ifndef _CRYPTO_BLOWFISH_H
+#define _CRYPTO_BLOWFISH_H
+
+#include <linux/types.h>
+#include <linux/crypto.h>
+
+#define BF_BLOCK_SIZE 8
+#define BF_MIN_KEY_SIZE 4
+#define BF_MAX_KEY_SIZE 56
+
+struct bf_ctx {
+       u32 p[18];
+       u32 s[1024];
+};
+
+int blowfish_setkey(struct crypto_tfm *tfm, const u8 *key,
+                   unsigned int key_len);
+
+#endif
index 069e85ba97e1874a6f92cb98dc0f4794e8969249..c6c9c1fe460c20e1d11e923c500a872f225d3c29 100644 (file)
@@ -82,4 +82,9 @@ struct sha512_state {
        u8 buf[SHA512_BLOCK_SIZE];
 };
 
+struct shash_desc;
+
+extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
+                             unsigned int len);
+
 #endif
index e5e468e9133d645411fe18314bb24d81e8701471..de9adec5693c7fd17a9c738439c6d6dda6facf8c 100644 (file)
 
 #define CRYPTO_ALG_TESTED              0x00000400
 
+/*
+ * Set if the algorithm is an instance that is build from templates.
+ */
+#define CRYPTO_ALG_INSTANCE            0x00000800
+
 /*
  * Transform masks and values (for crt_flags).
  */
diff --git a/include/linux/cryptouser.h b/include/linux/cryptouser.h
new file mode 100644 (file)
index 0000000..532fb58
--- /dev/null
@@ -0,0 +1,102 @@
+/*
+ * Crypto user configuration API.
+ *
+ * Copyright (C) 2011 secunet Security Networks AG
+ * Copyright (C) 2011 Steffen Klassert <steffen.klassert@secunet.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* Netlink configuration messages.  */
+enum {
+       CRYPTO_MSG_BASE = 0x10,
+       CRYPTO_MSG_NEWALG = 0x10,
+       CRYPTO_MSG_DELALG,
+       CRYPTO_MSG_UPDATEALG,
+       CRYPTO_MSG_GETALG,
+       __CRYPTO_MSG_MAX
+};
+#define CRYPTO_MSG_MAX (__CRYPTO_MSG_MAX - 1)
+#define CRYPTO_NR_MSGTYPES (CRYPTO_MSG_MAX + 1 - CRYPTO_MSG_BASE)
+
+#define CRYPTO_MAX_NAME CRYPTO_MAX_ALG_NAME
+
+/* Netlink message attributes.  */
+enum crypto_attr_type_t {
+       CRYPTOCFGA_UNSPEC,
+       CRYPTOCFGA_PRIORITY_VAL,        /* __u32 */
+       CRYPTOCFGA_REPORT_LARVAL,       /* struct crypto_report_larval */
+       CRYPTOCFGA_REPORT_HASH,         /* struct crypto_report_hash */
+       CRYPTOCFGA_REPORT_BLKCIPHER,    /* struct crypto_report_blkcipher */
+       CRYPTOCFGA_REPORT_AEAD,         /* struct crypto_report_aead */
+       CRYPTOCFGA_REPORT_COMPRESS,     /* struct crypto_report_comp */
+       CRYPTOCFGA_REPORT_RNG,          /* struct crypto_report_rng */
+       CRYPTOCFGA_REPORT_CIPHER,       /* struct crypto_report_cipher */
+       __CRYPTOCFGA_MAX
+
+#define CRYPTOCFGA_MAX (__CRYPTOCFGA_MAX - 1)
+};
+
+struct crypto_user_alg {
+       char cru_name[CRYPTO_MAX_ALG_NAME];
+       char cru_driver_name[CRYPTO_MAX_ALG_NAME];
+       char cru_module_name[CRYPTO_MAX_ALG_NAME];
+       __u32 cru_type;
+       __u32 cru_mask;
+       __u32 cru_refcnt;
+       __u32 cru_flags;
+};
+
+struct crypto_report_larval {
+       char type[CRYPTO_MAX_NAME];
+};
+
+struct crypto_report_hash {
+       char type[CRYPTO_MAX_NAME];
+       unsigned int blocksize;
+       unsigned int digestsize;
+};
+
+struct crypto_report_cipher {
+       char type[CRYPTO_MAX_ALG_NAME];
+       unsigned int blocksize;
+       unsigned int min_keysize;
+       unsigned int max_keysize;
+};
+
+struct crypto_report_blkcipher {
+       char type[CRYPTO_MAX_NAME];
+       char geniv[CRYPTO_MAX_NAME];
+       unsigned int blocksize;
+       unsigned int min_keysize;
+       unsigned int max_keysize;
+       unsigned int ivsize;
+};
+
+struct crypto_report_aead {
+       char type[CRYPTO_MAX_NAME];
+       char geniv[CRYPTO_MAX_NAME];
+       unsigned int blocksize;
+       unsigned int maxauthsize;
+       unsigned int ivsize;
+};
+
+struct crypto_report_comp {
+       char type[CRYPTO_MAX_NAME];
+};
+
+struct crypto_report_rng {
+       char type[CRYPTO_MAX_NAME];
+       unsigned int seedsize;
+};
index 8180cd9d73d5b13d61d65c14326e9918d5f4f890..8374d29673625db129e01ba4a13b29761d769688 100644 (file)
@@ -25,6 +25,7 @@
 #define NETLINK_SCSITRANSPORT  18      /* SCSI Transports */
 #define NETLINK_ECRYPTFS       19
 #define NETLINK_RDMA           20
+#define NETLINK_CRYPTO         21      /* Crypto layer */
 
 #define MAX_LINKS 32