Pileus Git - ~andy/linux/blob - arch/x86/crypto/aesni-intel_glue.c

   1 /*
   2  * Support for Intel AES-NI instructions. This file contains glue
   3  * code, the real AES implementation is in intel-aes_asm.S.
   4  *
   5  * Copyright (C) 2008, Intel Corp.
   6  *    Author: Huang Ying <ying.huang@intel.com>
   7  *
   8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
   9  * interface for 64-bit kernels.
  10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
  11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  12  *             Tadeusz Struk (tadeusz.struk@intel.com)
  13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  14  *    Copyright (c) 2010, Intel Corporation.
  15  *
  16  * This program is free software; you can redistribute it and/or modify
  17  * it under the terms of the GNU General Public License as published by
  18  * the Free Software Foundation; either version 2 of the License, or
  19  * (at your option) any later version.
  20  */
  21
  22 #include <linux/hardirq.h>
  23 #include <linux/types.h>
  24 #include <linux/crypto.h>
  25 #include <linux/err.h>
  26 #include <crypto/algapi.h>
  27 #include <crypto/aes.h>
  28 #include <crypto/cryptd.h>
  29 #include <crypto/ctr.h>
  30 #include <asm/i387.h>
  31 #include <asm/aes.h>
  32 #include <crypto/scatterwalk.h>
  33 #include <crypto/internal/aead.h>
  34 #include <linux/workqueue.h>
  35 #include <linux/spinlock.h>
  36
  37 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
  38 #define HAS_CTR
  39 #endif
  40
  41 #if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
  42 #define HAS_LRW
  43 #endif
  44
  45 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
  46 #define HAS_PCBC
  47 #endif
  48
  49 #if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
  50 #define HAS_XTS
  51 #endif
  52
  53 struct async_aes_ctx {
  54         struct cryptd_ablkcipher *cryptd_tfm;
  55 };
  56
  57 /* This data is stored at the end of the crypto_tfm struct.
  58  * It's a type of per "session" data storage location.
  59  * This needs to be 16 byte aligned.
  60  */
  61 struct aesni_rfc4106_gcm_ctx {
  62         u8 hash_subkey[16];
  63         struct crypto_aes_ctx aes_key_expanded;
  64         u8 nonce[4];
  65         struct cryptd_aead *cryptd_tfm;
  66 };
  67
  68 struct aesni_gcm_set_hash_subkey_result {
  69         int err;
  70         struct completion completion;
  71 };
  72
  73 struct aesni_hash_subkey_req_data {
  74         u8 iv[16];
  75         struct aesni_gcm_set_hash_subkey_result result;
  76         struct scatterlist sg;
  77 };
  78
  79 #define AESNI_ALIGN     (16)
  80 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
  81 #define RFC4106_HASH_SUBKEY_SIZE 16
  82
  83 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
  84                              unsigned int key_len);
  85 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
  86                           const u8 *in);
  87 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
  88                           const u8 *in);
  89 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
  90                               const u8 *in, unsigned int len);
  91 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
  92                               const u8 *in, unsigned int len);
  93 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
  94                               const u8 *in, unsigned int len, u8 *iv);
  95 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
  96                               const u8 *in, unsigned int len, u8 *iv);
  97 #ifdef CONFIG_X86_64
  98 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
  99                               const u8 *in, unsigned int len, u8 *iv);
 100
 101 /* asmlinkage void aesni_gcm_enc()
 102  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
 103  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
 104  * const u8 *in, Plaintext input
 105  * unsigned long plaintext_len, Length of data in bytes for encryption.
 106  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 107  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 108  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 109  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 110  * const u8 *aad, Additional Authentication Data (AAD)
 111  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
 112  *          is going to be 8 or 12 bytes
 113  * u8 *auth_tag, Authenticated Tag output.
 114  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
 115  *          Valid values are 16 (most likely), 12 or 8.
 116  */
 117 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
 118                         const u8 *in, unsigned long plaintext_len, u8 *iv,
 119                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 120                         u8 *auth_tag, unsigned long auth_tag_len);
 121
 122 /* asmlinkage void aesni_gcm_dec()
 123  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
 124  * u8 *out, Plaintext output. Decrypt in-place is allowed.
 125  * const u8 *in, Ciphertext input
 126  * unsigned long ciphertext_len, Length of data in bytes for decryption.
 127  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
 128  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
 129  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
 130  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
 131  * const u8 *aad, Additional Authentication Data (AAD)
 132  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
 133  * to be 8 or 12 bytes
 134  * u8 *auth_tag, Authenticated Tag output.
 135  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
 136  * Valid values are 16 (most likely), 12 or 8.
 137  */
 138 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
 139                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
 140                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
 141                         u8 *auth_tag, unsigned long auth_tag_len);
 142
 143 static inline struct
 144 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
 145 {
 146         return
 147                 (struct aesni_rfc4106_gcm_ctx *)
 148                 PTR_ALIGN((u8 *)
 149                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
 150 }
 151 #endif
 152
 153 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
 154 {
 155         unsigned long addr = (unsigned long)raw_ctx;
 156         unsigned long align = AESNI_ALIGN;
 157
 158         if (align <= crypto_tfm_ctx_alignment())
 159                 align = 1;
 160         return (struct crypto_aes_ctx *)ALIGN(addr, align);
 161 }
 162
 163 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
 164                               const u8 *in_key, unsigned int key_len)
 165 {
 166         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
 167         u32 *flags = &tfm->crt_flags;
 168         int err;
 169
 170         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
 171             key_len != AES_KEYSIZE_256) {
 172                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 173                 return -EINVAL;
 174         }
 175
 176         if (!irq_fpu_usable())
 177                 err = crypto_aes_expand_key(ctx, in_key, key_len);
 178         else {
 179                 kernel_fpu_begin();
 180                 err = aesni_set_key(ctx, in_key, key_len);
 181                 kernel_fpu_end();
 182         }
 183
 184         return err;
 185 }
 186
 187 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 188                        unsigned int key_len)
 189 {
 190         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
 191 }
 192
 193 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 194 {
 195         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 196
 197         if (!irq_fpu_usable())
 198                 crypto_aes_encrypt_x86(ctx, dst, src);
 199         else {
 200                 kernel_fpu_begin();
 201                 aesni_enc(ctx, dst, src);
 202                 kernel_fpu_end();
 203         }
 204 }
 205
 206 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 207 {
 208         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 209
 210         if (!irq_fpu_usable())
 211                 crypto_aes_decrypt_x86(ctx, dst, src);
 212         else {
 213                 kernel_fpu_begin();
 214                 aesni_dec(ctx, dst, src);
 215                 kernel_fpu_end();
 216         }
 217 }
 218
 219 static struct crypto_alg aesni_alg = {
 220         .cra_name               = "aes",
 221         .cra_driver_name        = "aes-aesni",
 222         .cra_priority           = 300,
 223         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 224         .cra_blocksize          = AES_BLOCK_SIZE,
 225         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 226         .cra_alignmask          = 0,
 227         .cra_module             = THIS_MODULE,
 228         .cra_list               = LIST_HEAD_INIT(aesni_alg.cra_list),
 229         .cra_u  = {
 230                 .cipher = {
 231                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 232                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 233                         .cia_setkey             = aes_set_key,
 234                         .cia_encrypt            = aes_encrypt,
 235                         .cia_decrypt            = aes_decrypt
 236                 }
 237         }
 238 };
 239
 240 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 241 {
 242         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 243
 244         aesni_enc(ctx, dst, src);
 245 }
 246
 247 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 248 {
 249         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
 250
 251         aesni_dec(ctx, dst, src);
 252 }
 253
 254 static struct crypto_alg __aesni_alg = {
 255         .cra_name               = "__aes-aesni",
 256         .cra_driver_name        = "__driver-aes-aesni",
 257         .cra_priority           = 0,
 258         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
 259         .cra_blocksize          = AES_BLOCK_SIZE,
 260         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 261         .cra_alignmask          = 0,
 262         .cra_module             = THIS_MODULE,
 263         .cra_list               = LIST_HEAD_INIT(__aesni_alg.cra_list),
 264         .cra_u  = {
 265                 .cipher = {
 266                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
 267                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
 268                         .cia_setkey             = aes_set_key,
 269                         .cia_encrypt            = __aes_encrypt,
 270                         .cia_decrypt            = __aes_decrypt
 271                 }
 272         }
 273 };
 274
 275 static int ecb_encrypt(struct blkcipher_desc *desc,
 276                        struct scatterlist *dst, struct scatterlist *src,
 277                        unsigned int nbytes)
 278 {
 279         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 280         struct blkcipher_walk walk;
 281         int err;
 282
 283         blkcipher_walk_init(&walk, dst, src, nbytes);
 284         err = blkcipher_walk_virt(desc, &walk);
 285         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 286
 287         kernel_fpu_begin();
 288         while ((nbytes = walk.nbytes)) {
 289                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 290                               nbytes & AES_BLOCK_MASK);
 291                 nbytes &= AES_BLOCK_SIZE - 1;
 292                 err = blkcipher_walk_done(desc, &walk, nbytes);
 293         }
 294         kernel_fpu_end();
 295
 296         return err;
 297 }
 298
 299 static int ecb_decrypt(struct blkcipher_desc *desc,
 300                        struct scatterlist *dst, struct scatterlist *src,
 301                        unsigned int nbytes)
 302 {
 303         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 304         struct blkcipher_walk walk;
 305         int err;
 306
 307         blkcipher_walk_init(&walk, dst, src, nbytes);
 308         err = blkcipher_walk_virt(desc, &walk);
 309         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 310
 311         kernel_fpu_begin();
 312         while ((nbytes = walk.nbytes)) {
 313                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 314                               nbytes & AES_BLOCK_MASK);
 315                 nbytes &= AES_BLOCK_SIZE - 1;
 316                 err = blkcipher_walk_done(desc, &walk, nbytes);
 317         }
 318         kernel_fpu_end();
 319
 320         return err;
 321 }
 322
 323 static struct crypto_alg blk_ecb_alg = {
 324         .cra_name               = "__ecb-aes-aesni",
 325         .cra_driver_name        = "__driver-ecb-aes-aesni",
 326         .cra_priority           = 0,
 327         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 328         .cra_blocksize          = AES_BLOCK_SIZE,
 329         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 330         .cra_alignmask          = 0,
 331         .cra_type               = &crypto_blkcipher_type,
 332         .cra_module             = THIS_MODULE,
 333         .cra_list               = LIST_HEAD_INIT(blk_ecb_alg.cra_list),
 334         .cra_u = {
 335                 .blkcipher = {
 336                         .min_keysize    = AES_MIN_KEY_SIZE,
 337                         .max_keysize    = AES_MAX_KEY_SIZE,
 338                         .setkey         = aes_set_key,
 339                         .encrypt        = ecb_encrypt,
 340                         .decrypt        = ecb_decrypt,
 341                 },
 342         },
 343 };
 344
 345 static int cbc_encrypt(struct blkcipher_desc *desc,
 346                        struct scatterlist *dst, struct scatterlist *src,
 347                        unsigned int nbytes)
 348 {
 349         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 350         struct blkcipher_walk walk;
 351         int err;
 352
 353         blkcipher_walk_init(&walk, dst, src, nbytes);
 354         err = blkcipher_walk_virt(desc, &walk);
 355         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 356
 357         kernel_fpu_begin();
 358         while ((nbytes = walk.nbytes)) {
 359                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 360                               nbytes & AES_BLOCK_MASK, walk.iv);
 361                 nbytes &= AES_BLOCK_SIZE - 1;
 362                 err = blkcipher_walk_done(desc, &walk, nbytes);
 363         }
 364         kernel_fpu_end();
 365
 366         return err;
 367 }
 368
 369 static int cbc_decrypt(struct blkcipher_desc *desc,
 370                        struct scatterlist *dst, struct scatterlist *src,
 371                        unsigned int nbytes)
 372 {
 373         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 374         struct blkcipher_walk walk;
 375         int err;
 376
 377         blkcipher_walk_init(&walk, dst, src, nbytes);
 378         err = blkcipher_walk_virt(desc, &walk);
 379         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 380
 381         kernel_fpu_begin();
 382         while ((nbytes = walk.nbytes)) {
 383                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 384                               nbytes & AES_BLOCK_MASK, walk.iv);
 385                 nbytes &= AES_BLOCK_SIZE - 1;
 386                 err = blkcipher_walk_done(desc, &walk, nbytes);
 387         }
 388         kernel_fpu_end();
 389
 390         return err;
 391 }
 392
 393 static struct crypto_alg blk_cbc_alg = {
 394         .cra_name               = "__cbc-aes-aesni",
 395         .cra_driver_name        = "__driver-cbc-aes-aesni",
 396         .cra_priority           = 0,
 397         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 398         .cra_blocksize          = AES_BLOCK_SIZE,
 399         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 400         .cra_alignmask          = 0,
 401         .cra_type               = &crypto_blkcipher_type,
 402         .cra_module             = THIS_MODULE,
 403         .cra_list               = LIST_HEAD_INIT(blk_cbc_alg.cra_list),
 404         .cra_u = {
 405                 .blkcipher = {
 406                         .min_keysize    = AES_MIN_KEY_SIZE,
 407                         .max_keysize    = AES_MAX_KEY_SIZE,
 408                         .setkey         = aes_set_key,
 409                         .encrypt        = cbc_encrypt,
 410                         .decrypt        = cbc_decrypt,
 411                 },
 412         },
 413 };
 414
 415 #ifdef CONFIG_X86_64
 416 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
 417                             struct blkcipher_walk *walk)
 418 {
 419         u8 *ctrblk = walk->iv;
 420         u8 keystream[AES_BLOCK_SIZE];
 421         u8 *src = walk->src.virt.addr;
 422         u8 *dst = walk->dst.virt.addr;
 423         unsigned int nbytes = walk->nbytes;
 424
 425         aesni_enc(ctx, keystream, ctrblk);
 426         crypto_xor(keystream, src, nbytes);
 427         memcpy(dst, keystream, nbytes);
 428         crypto_inc(ctrblk, AES_BLOCK_SIZE);
 429 }
 430
 431 static int ctr_crypt(struct blkcipher_desc *desc,
 432                      struct scatterlist *dst, struct scatterlist *src,
 433                      unsigned int nbytes)
 434 {
 435         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
 436         struct blkcipher_walk walk;
 437         int err;
 438
 439         blkcipher_walk_init(&walk, dst, src, nbytes);
 440         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 441         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 442
 443         kernel_fpu_begin();
 444         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
 445                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
 446                               nbytes & AES_BLOCK_MASK, walk.iv);
 447                 nbytes &= AES_BLOCK_SIZE - 1;
 448                 err = blkcipher_walk_done(desc, &walk, nbytes);
 449         }
 450         if (walk.nbytes) {
 451                 ctr_crypt_final(ctx, &walk);
 452                 err = blkcipher_walk_done(desc, &walk, 0);
 453         }
 454         kernel_fpu_end();
 455
 456         return err;
 457 }
 458
 459 static struct crypto_alg blk_ctr_alg = {
 460         .cra_name               = "__ctr-aes-aesni",
 461         .cra_driver_name        = "__driver-ctr-aes-aesni",
 462         .cra_priority           = 0,
 463         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
 464         .cra_blocksize          = 1,
 465         .cra_ctxsize            = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1,
 466         .cra_alignmask          = 0,
 467         .cra_type               = &crypto_blkcipher_type,
 468         .cra_module             = THIS_MODULE,
 469         .cra_list               = LIST_HEAD_INIT(blk_ctr_alg.cra_list),
 470         .cra_u = {
 471                 .blkcipher = {
 472                         .min_keysize    = AES_MIN_KEY_SIZE,
 473                         .max_keysize    = AES_MAX_KEY_SIZE,
 474                         .ivsize         = AES_BLOCK_SIZE,
 475                         .setkey         = aes_set_key,
 476                         .encrypt        = ctr_crypt,
 477                         .decrypt        = ctr_crypt,
 478                 },
 479         },
 480 };
 481 #endif
 482
 483 static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
 484                         unsigned int key_len)
 485 {
 486         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 487         struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
 488         int err;
 489
 490         crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 491         crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
 492                                     & CRYPTO_TFM_REQ_MASK);
 493         err = crypto_ablkcipher_setkey(child, key, key_len);
 494         crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
 495                                     & CRYPTO_TFM_RES_MASK);
 496         return err;
 497 }
 498
 499 static int ablk_encrypt(struct ablkcipher_request *req)
 500 {
 501         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 502         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 503
 504         if (!irq_fpu_usable()) {
 505                 struct ablkcipher_request *cryptd_req =
 506                         ablkcipher_request_ctx(req);
 507                 memcpy(cryptd_req, req, sizeof(*req));
 508                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 509                 return crypto_ablkcipher_encrypt(cryptd_req);
 510         } else {
 511                 struct blkcipher_desc desc;
 512                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 513                 desc.info = req->info;
 514                 desc.flags = 0;
 515                 return crypto_blkcipher_crt(desc.tfm)->encrypt(
 516                         &desc, req->dst, req->src, req->nbytes);
 517         }
 518 }
 519
 520 static int ablk_decrypt(struct ablkcipher_request *req)
 521 {
 522         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 523         struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 524
 525         if (!irq_fpu_usable()) {
 526                 struct ablkcipher_request *cryptd_req =
 527                         ablkcipher_request_ctx(req);
 528                 memcpy(cryptd_req, req, sizeof(*req));
 529                 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
 530                 return crypto_ablkcipher_decrypt(cryptd_req);
 531         } else {
 532                 struct blkcipher_desc desc;
 533                 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
 534                 desc.info = req->info;
 535                 desc.flags = 0;
 536                 return crypto_blkcipher_crt(desc.tfm)->decrypt(
 537                         &desc, req->dst, req->src, req->nbytes);
 538         }
 539 }
 540
 541 static void ablk_exit(struct crypto_tfm *tfm)
 542 {
 543         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 544
 545         cryptd_free_ablkcipher(ctx->cryptd_tfm);
 546 }
 547
 548 static void ablk_init_common(struct crypto_tfm *tfm,
 549                              struct cryptd_ablkcipher *cryptd_tfm)
 550 {
 551         struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 552
 553         ctx->cryptd_tfm = cryptd_tfm;
 554         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
 555                 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
 556 }
 557
 558 static int ablk_ecb_init(struct crypto_tfm *tfm)
 559 {
 560         struct cryptd_ablkcipher *cryptd_tfm;
 561
 562         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0);
 563         if (IS_ERR(cryptd_tfm))
 564                 return PTR_ERR(cryptd_tfm);
 565         ablk_init_common(tfm, cryptd_tfm);
 566         return 0;
 567 }
 568
 569 static struct crypto_alg ablk_ecb_alg = {
 570         .cra_name               = "ecb(aes)",
 571         .cra_driver_name        = "ecb-aes-aesni",
 572         .cra_priority           = 400,
 573         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 574         .cra_blocksize          = AES_BLOCK_SIZE,
 575         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 576         .cra_alignmask          = 0,
 577         .cra_type               = &crypto_ablkcipher_type,
 578         .cra_module             = THIS_MODULE,
 579         .cra_list               = LIST_HEAD_INIT(ablk_ecb_alg.cra_list),
 580         .cra_init               = ablk_ecb_init,
 581         .cra_exit               = ablk_exit,
 582         .cra_u = {
 583                 .ablkcipher = {
 584                         .min_keysize    = AES_MIN_KEY_SIZE,
 585                         .max_keysize    = AES_MAX_KEY_SIZE,
 586                         .setkey         = ablk_set_key,
 587                         .encrypt        = ablk_encrypt,
 588                         .decrypt        = ablk_decrypt,
 589                 },
 590         },
 591 };
 592
 593 static int ablk_cbc_init(struct crypto_tfm *tfm)
 594 {
 595         struct cryptd_ablkcipher *cryptd_tfm;
 596
 597         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0);
 598         if (IS_ERR(cryptd_tfm))
 599                 return PTR_ERR(cryptd_tfm);
 600         ablk_init_common(tfm, cryptd_tfm);
 601         return 0;
 602 }
 603
 604 static struct crypto_alg ablk_cbc_alg = {
 605         .cra_name               = "cbc(aes)",
 606         .cra_driver_name        = "cbc-aes-aesni",
 607         .cra_priority           = 400,
 608         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 609         .cra_blocksize          = AES_BLOCK_SIZE,
 610         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 611         .cra_alignmask          = 0,
 612         .cra_type               = &crypto_ablkcipher_type,
 613         .cra_module             = THIS_MODULE,
 614         .cra_list               = LIST_HEAD_INIT(ablk_cbc_alg.cra_list),
 615         .cra_init               = ablk_cbc_init,
 616         .cra_exit               = ablk_exit,
 617         .cra_u = {
 618                 .ablkcipher = {
 619                         .min_keysize    = AES_MIN_KEY_SIZE,
 620                         .max_keysize    = AES_MAX_KEY_SIZE,
 621                         .ivsize         = AES_BLOCK_SIZE,
 622                         .setkey         = ablk_set_key,
 623                         .encrypt        = ablk_encrypt,
 624                         .decrypt        = ablk_decrypt,
 625                 },
 626         },
 627 };
 628
 629 #ifdef CONFIG_X86_64
 630 static int ablk_ctr_init(struct crypto_tfm *tfm)
 631 {
 632         struct cryptd_ablkcipher *cryptd_tfm;
 633
 634         cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ctr-aes-aesni", 0, 0);
 635         if (IS_ERR(cryptd_tfm))
 636                 return PTR_ERR(cryptd_tfm);
 637         ablk_init_common(tfm, cryptd_tfm);
 638         return 0;
 639 }
 640
 641 static struct crypto_alg ablk_ctr_alg = {
 642         .cra_name               = "ctr(aes)",
 643         .cra_driver_name        = "ctr-aes-aesni",
 644         .cra_priority           = 400,
 645         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 646         .cra_blocksize          = 1,
 647         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 648         .cra_alignmask          = 0,
 649         .cra_type               = &crypto_ablkcipher_type,
 650         .cra_module             = THIS_MODULE,
 651         .cra_list               = LIST_HEAD_INIT(ablk_ctr_alg.cra_list),
 652         .cra_init               = ablk_ctr_init,
 653         .cra_exit               = ablk_exit,
 654         .cra_u = {
 655                 .ablkcipher = {
 656                         .min_keysize    = AES_MIN_KEY_SIZE,
 657                         .max_keysize    = AES_MAX_KEY_SIZE,
 658                         .ivsize         = AES_BLOCK_SIZE,
 659                         .setkey         = ablk_set_key,
 660                         .encrypt        = ablk_encrypt,
 661                         .decrypt        = ablk_encrypt,
 662                         .geniv          = "chainiv",
 663                 },
 664         },
 665 };
 666
 667 #ifdef HAS_CTR
 668 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
 669 {
 670         struct cryptd_ablkcipher *cryptd_tfm;
 671
 672         cryptd_tfm = cryptd_alloc_ablkcipher(
 673                 "rfc3686(__driver-ctr-aes-aesni)", 0, 0);
 674         if (IS_ERR(cryptd_tfm))
 675                 return PTR_ERR(cryptd_tfm);
 676         ablk_init_common(tfm, cryptd_tfm);
 677         return 0;
 678 }
 679
 680 static struct crypto_alg ablk_rfc3686_ctr_alg = {
 681         .cra_name               = "rfc3686(ctr(aes))",
 682         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
 683         .cra_priority           = 400,
 684         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 685         .cra_blocksize          = 1,
 686         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 687         .cra_alignmask          = 0,
 688         .cra_type               = &crypto_ablkcipher_type,
 689         .cra_module             = THIS_MODULE,
 690         .cra_list               = LIST_HEAD_INIT(ablk_rfc3686_ctr_alg.cra_list),
 691         .cra_init               = ablk_rfc3686_ctr_init,
 692         .cra_exit               = ablk_exit,
 693         .cra_u = {
 694                 .ablkcipher = {
 695                         .min_keysize = AES_MIN_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 696                         .max_keysize = AES_MAX_KEY_SIZE+CTR_RFC3686_NONCE_SIZE,
 697                         .ivsize      = CTR_RFC3686_IV_SIZE,
 698                         .setkey      = ablk_set_key,
 699                         .encrypt     = ablk_encrypt,
 700                         .decrypt     = ablk_decrypt,
 701                         .geniv       = "seqiv",
 702                 },
 703         },
 704 };
 705 #endif
 706 #endif
 707
 708 #ifdef HAS_LRW
 709 static int ablk_lrw_init(struct crypto_tfm *tfm)
 710 {
 711         struct cryptd_ablkcipher *cryptd_tfm;
 712
 713         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(lrw(__driver-aes-aesni))",
 714                                              0, 0);
 715         if (IS_ERR(cryptd_tfm))
 716                 return PTR_ERR(cryptd_tfm);
 717         ablk_init_common(tfm, cryptd_tfm);
 718         return 0;
 719 }
 720
 721 static struct crypto_alg ablk_lrw_alg = {
 722         .cra_name               = "lrw(aes)",
 723         .cra_driver_name        = "lrw-aes-aesni",
 724         .cra_priority           = 400,
 725         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 726         .cra_blocksize          = AES_BLOCK_SIZE,
 727         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 728         .cra_alignmask          = 0,
 729         .cra_type               = &crypto_ablkcipher_type,
 730         .cra_module             = THIS_MODULE,
 731         .cra_list               = LIST_HEAD_INIT(ablk_lrw_alg.cra_list),
 732         .cra_init               = ablk_lrw_init,
 733         .cra_exit               = ablk_exit,
 734         .cra_u = {
 735                 .ablkcipher = {
 736                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
 737                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
 738                         .ivsize         = AES_BLOCK_SIZE,
 739                         .setkey         = ablk_set_key,
 740                         .encrypt        = ablk_encrypt,
 741                         .decrypt        = ablk_decrypt,
 742                 },
 743         },
 744 };
 745 #endif
 746
 747 #ifdef HAS_PCBC
 748 static int ablk_pcbc_init(struct crypto_tfm *tfm)
 749 {
 750         struct cryptd_ablkcipher *cryptd_tfm;
 751
 752         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))",
 753                                              0, 0);
 754         if (IS_ERR(cryptd_tfm))
 755                 return PTR_ERR(cryptd_tfm);
 756         ablk_init_common(tfm, cryptd_tfm);
 757         return 0;
 758 }
 759
 760 static struct crypto_alg ablk_pcbc_alg = {
 761         .cra_name               = "pcbc(aes)",
 762         .cra_driver_name        = "pcbc-aes-aesni",
 763         .cra_priority           = 400,
 764         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 765         .cra_blocksize          = AES_BLOCK_SIZE,
 766         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 767         .cra_alignmask          = 0,
 768         .cra_type               = &crypto_ablkcipher_type,
 769         .cra_module             = THIS_MODULE,
 770         .cra_list               = LIST_HEAD_INIT(ablk_pcbc_alg.cra_list),
 771         .cra_init               = ablk_pcbc_init,
 772         .cra_exit               = ablk_exit,
 773         .cra_u = {
 774                 .ablkcipher = {
 775                         .min_keysize    = AES_MIN_KEY_SIZE,
 776                         .max_keysize    = AES_MAX_KEY_SIZE,
 777                         .ivsize         = AES_BLOCK_SIZE,
 778                         .setkey         = ablk_set_key,
 779                         .encrypt        = ablk_encrypt,
 780                         .decrypt        = ablk_decrypt,
 781                 },
 782         },
 783 };
 784 #endif
 785
 786 #ifdef HAS_XTS
 787 static int ablk_xts_init(struct crypto_tfm *tfm)
 788 {
 789         struct cryptd_ablkcipher *cryptd_tfm;
 790
 791         cryptd_tfm = cryptd_alloc_ablkcipher("fpu(xts(__driver-aes-aesni))",
 792                                              0, 0);
 793         if (IS_ERR(cryptd_tfm))
 794                 return PTR_ERR(cryptd_tfm);
 795         ablk_init_common(tfm, cryptd_tfm);
 796         return 0;
 797 }
 798
 799 static struct crypto_alg ablk_xts_alg = {
 800         .cra_name               = "xts(aes)",
 801         .cra_driver_name        = "xts-aes-aesni",
 802         .cra_priority           = 400,
 803         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 804         .cra_blocksize          = AES_BLOCK_SIZE,
 805         .cra_ctxsize            = sizeof(struct async_aes_ctx),
 806         .cra_alignmask          = 0,
 807         .cra_type               = &crypto_ablkcipher_type,
 808         .cra_module             = THIS_MODULE,
 809         .cra_list               = LIST_HEAD_INIT(ablk_xts_alg.cra_list),
 810         .cra_init               = ablk_xts_init,
 811         .cra_exit               = ablk_exit,
 812         .cra_u = {
 813                 .ablkcipher = {
 814                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 815                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 816                         .ivsize         = AES_BLOCK_SIZE,
 817                         .setkey         = ablk_set_key,
 818                         .encrypt        = ablk_encrypt,
 819                         .decrypt        = ablk_decrypt,
 820                 },
 821         },
 822 };
 823 #endif
 824
 825 #ifdef CONFIG_X86_64
 826 static int rfc4106_init(struct crypto_tfm *tfm)
 827 {
 828         struct cryptd_aead *cryptd_tfm;
 829         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
 830                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 831         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
 832         if (IS_ERR(cryptd_tfm))
 833                 return PTR_ERR(cryptd_tfm);
 834         ctx->cryptd_tfm = cryptd_tfm;
 835         tfm->crt_aead.reqsize = sizeof(struct aead_request)
 836                 + crypto_aead_reqsize(&cryptd_tfm->base);
 837         return 0;
 838 }
 839
 840 static void rfc4106_exit(struct crypto_tfm *tfm)
 841 {
 842         struct aesni_rfc4106_gcm_ctx *ctx =
 843                 (struct aesni_rfc4106_gcm_ctx *)
 844                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
 845         if (!IS_ERR(ctx->cryptd_tfm))
 846                 cryptd_free_aead(ctx->cryptd_tfm);
 847         return;
 848 }
 849
 850 static void
 851 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
 852 {
 853         struct aesni_gcm_set_hash_subkey_result *result = req->data;
 854
 855         if (err == -EINPROGRESS)
 856                 return;
 857         result->err = err;
 858         complete(&result->completion);
 859 }
 860
 861 static int
 862 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
 863 {
 864         struct crypto_ablkcipher *ctr_tfm;
 865         struct ablkcipher_request *req;
 866         int ret = -EINVAL;
 867         struct aesni_hash_subkey_req_data *req_data;
 868
 869         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
 870         if (IS_ERR(ctr_tfm))
 871                 return PTR_ERR(ctr_tfm);
 872
 873         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
 874
 875         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
 876         if (ret)
 877                 goto out_free_ablkcipher;
 878
 879         ret = -ENOMEM;
 880         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
 881         if (!req)
 882                 goto out_free_ablkcipher;
 883
 884         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
 885         if (!req_data)
 886                 goto out_free_request;
 887
 888         memset(req_data->iv, 0, sizeof(req_data->iv));
 889
 890         /* Clear the data in the hash sub key container to zero.*/
 891         /* We want to cipher all zeros to create the hash sub key. */
 892         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
 893
 894         init_completion(&req_data->result.completion);
 895         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
 896         ablkcipher_request_set_tfm(req, ctr_tfm);
 897         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
 898                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
 899                                         rfc4106_set_hash_subkey_done,
 900                                         &req_data->result);
 901
 902         ablkcipher_request_set_crypt(req, &req_data->sg,
 903                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
 904
 905         ret = crypto_ablkcipher_encrypt(req);
 906         if (ret == -EINPROGRESS || ret == -EBUSY) {
 907                 ret = wait_for_completion_interruptible
 908                         (&req_data->result.completion);
 909                 if (!ret)
 910                         ret = req_data->result.err;
 911         }
 912         kfree(req_data);
 913 out_free_request:
 914         ablkcipher_request_free(req);
 915 out_free_ablkcipher:
 916         crypto_free_ablkcipher(ctr_tfm);
 917         return ret;
 918 }
 919
 920 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
 921                                                    unsigned int key_len)
 922 {
 923         int ret = 0;
 924         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
 925         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 926         u8 *new_key_mem = NULL;
 927
 928         if (key_len < 4) {
 929                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 930                 return -EINVAL;
 931         }
 932         /*Account for 4 byte nonce at the end.*/
 933         key_len -= 4;
 934         if (key_len != AES_KEYSIZE_128) {
 935                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 936                 return -EINVAL;
 937         }
 938
 939         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
 940         /*This must be on a 16 byte boundary!*/
 941         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
 942                 return -EINVAL;
 943
 944         if ((unsigned long)key % AESNI_ALIGN) {
 945                 /*key is not aligned: use an auxuliar aligned pointer*/
 946                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
 947                 if (!new_key_mem)
 948                         return -ENOMEM;
 949
 950                 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
 951                 memcpy(new_key_mem, key, key_len);
 952                 key = new_key_mem;
 953         }
 954
 955         if (!irq_fpu_usable())
 956                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
 957                 key, key_len);
 958         else {
 959                 kernel_fpu_begin();
 960                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
 961                 kernel_fpu_end();
 962         }
 963         /*This must be on a 16 byte boundary!*/
 964         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
 965                 ret = -EINVAL;
 966                 goto exit;
 967         }
 968         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
 969 exit:
 970         kfree(new_key_mem);
 971         return ret;
 972 }
 973
 974 /* This is the Integrity Check Value (aka the authentication tag length and can
 975  * be 8, 12 or 16 bytes long. */
 976 static int rfc4106_set_authsize(struct crypto_aead *parent,
 977                                 unsigned int authsize)
 978 {
 979         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
 980         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
 981
 982         switch (authsize) {
 983         case 8:
 984         case 12:
 985         case 16:
 986                 break;
 987         default:
 988                 return -EINVAL;
 989         }
 990         crypto_aead_crt(parent)->authsize = authsize;
 991         crypto_aead_crt(cryptd_child)->authsize = authsize;
 992         return 0;
 993 }
 994
 995 static int rfc4106_encrypt(struct aead_request *req)
 996 {
 997         int ret;
 998         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 999         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1000         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1001
1002         if (!irq_fpu_usable()) {
1003                 struct aead_request *cryptd_req =
1004                         (struct aead_request *) aead_request_ctx(req);
1005                 memcpy(cryptd_req, req, sizeof(*req));
1006                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1007                 return crypto_aead_encrypt(cryptd_req);
1008         } else {
1009                 kernel_fpu_begin();
1010                 ret = cryptd_child->base.crt_aead.encrypt(req);
1011                 kernel_fpu_end();
1012                 return ret;
1013         }
1014 }
1015
1016 static int rfc4106_decrypt(struct aead_request *req)
1017 {
1018         int ret;
1019         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1020         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1021         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
1022
1023         if (!irq_fpu_usable()) {
1024                 struct aead_request *cryptd_req =
1025                         (struct aead_request *) aead_request_ctx(req);
1026                 memcpy(cryptd_req, req, sizeof(*req));
1027                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
1028                 return crypto_aead_decrypt(cryptd_req);
1029         } else {
1030                 kernel_fpu_begin();
1031                 ret = cryptd_child->base.crt_aead.decrypt(req);
1032                 kernel_fpu_end();
1033                 return ret;
1034         }
1035 }
1036
1037 static struct crypto_alg rfc4106_alg = {
1038         .cra_name = "rfc4106(gcm(aes))",
1039         .cra_driver_name = "rfc4106-gcm-aesni",
1040         .cra_priority = 400,
1041         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1042         .cra_blocksize = 1,
1043         .cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1044         .cra_alignmask = 0,
1045         .cra_type = &crypto_nivaead_type,
1046         .cra_module = THIS_MODULE,
1047         .cra_list = LIST_HEAD_INIT(rfc4106_alg.cra_list),
1048         .cra_init = rfc4106_init,
1049         .cra_exit = rfc4106_exit,
1050         .cra_u = {
1051                 .aead = {
1052                         .setkey = rfc4106_set_key,
1053                         .setauthsize = rfc4106_set_authsize,
1054                         .encrypt = rfc4106_encrypt,
1055                         .decrypt = rfc4106_decrypt,
1056                         .geniv = "seqiv",
1057                         .ivsize = 8,
1058                         .maxauthsize = 16,
1059                 },
1060         },
1061 };
1062
1063 static int __driver_rfc4106_encrypt(struct aead_request *req)
1064 {
1065         u8 one_entry_in_sg = 0;
1066         u8 *src, *dst, *assoc;
1067         __be32 counter = cpu_to_be32(1);
1068         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1069         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1070         void *aes_ctx = &(ctx->aes_key_expanded);
1071         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1072         u8 iv_tab[16+AESNI_ALIGN];
1073         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
1074         struct scatter_walk src_sg_walk;
1075         struct scatter_walk assoc_sg_walk;
1076         struct scatter_walk dst_sg_walk;
1077         unsigned int i;
1078
1079         /* Assuming we are supporting rfc4106 64-bit extended */
1080         /* sequence numbers We need to have the AAD length equal */
1081         /* to 8 or 12 bytes */
1082         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
1083                 return -EINVAL;
1084         /* IV below built */
1085         for (i = 0; i < 4; i++)
1086                 *(iv+i) = ctx->nonce[i];
1087         for (i = 0; i < 8; i++)
1088                 *(iv+4+i) = req->iv[i];
1089         *((__be32 *)(iv+12)) = counter;
1090
1091         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1092                 one_entry_in_sg = 1;
1093                 scatterwalk_start(&src_sg_walk, req->src);
1094                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1095                 src = scatterwalk_map(&src_sg_walk, 0);
1096                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1097                 dst = src;
1098                 if (unlikely(req->src != req->dst)) {
1099                         scatterwalk_start(&dst_sg_walk, req->dst);
1100                         dst = scatterwalk_map(&dst_sg_walk, 0);
1101                 }
1102
1103         } else {
1104                 /* Allocate memory for src, dst, assoc */
1105                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
1106                         GFP_ATOMIC);
1107                 if (unlikely(!src))
1108                         return -ENOMEM;
1109                 assoc = (src + req->cryptlen + auth_tag_len);
1110                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1111                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1112                                         req->assoclen, 0);
1113                 dst = src;
1114         }
1115
1116         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
1117                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
1118                 + ((unsigned long)req->cryptlen), auth_tag_len);
1119
1120         /* The authTag (aka the Integrity Check Value) needs to be written
1121          * back to the packet. */
1122         if (one_entry_in_sg) {
1123                 if (unlikely(req->src != req->dst)) {
1124                         scatterwalk_unmap(dst, 0);
1125                         scatterwalk_done(&dst_sg_walk, 0, 0);
1126                 }
1127                 scatterwalk_unmap(src, 0);
1128                 scatterwalk_unmap(assoc, 0);
1129                 scatterwalk_done(&src_sg_walk, 0, 0);
1130                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1131         } else {
1132                 scatterwalk_map_and_copy(dst, req->dst, 0,
1133                         req->cryptlen + auth_tag_len, 1);
1134                 kfree(src);
1135         }
1136         return 0;
1137 }
1138
1139 static int __driver_rfc4106_decrypt(struct aead_request *req)
1140 {
1141         u8 one_entry_in_sg = 0;
1142         u8 *src, *dst, *assoc;
1143         unsigned long tempCipherLen = 0;
1144         __be32 counter = cpu_to_be32(1);
1145         int retval = 0;
1146         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1147         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
1148         void *aes_ctx = &(ctx->aes_key_expanded);
1149         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
1150         u8 iv_and_authTag[32+AESNI_ALIGN];
1151         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
1152         u8 *authTag = iv + 16;
1153         struct scatter_walk src_sg_walk;
1154         struct scatter_walk assoc_sg_walk;
1155         struct scatter_walk dst_sg_walk;
1156         unsigned int i;
1157
1158         if (unlikely((req->cryptlen < auth_tag_len) ||
1159                 (req->assoclen != 8 && req->assoclen != 12)))
1160                 return -EINVAL;
1161         /* Assuming we are supporting rfc4106 64-bit extended */
1162         /* sequence numbers We need to have the AAD length */
1163         /* equal to 8 or 12 bytes */
1164
1165         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
1166         /* IV below built */
1167         for (i = 0; i < 4; i++)
1168                 *(iv+i) = ctx->nonce[i];
1169         for (i = 0; i < 8; i++)
1170                 *(iv+4+i) = req->iv[i];
1171         *((__be32 *)(iv+12)) = counter;
1172
1173         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
1174                 one_entry_in_sg = 1;
1175                 scatterwalk_start(&src_sg_walk, req->src);
1176                 scatterwalk_start(&assoc_sg_walk, req->assoc);
1177                 src = scatterwalk_map(&src_sg_walk, 0);
1178                 assoc = scatterwalk_map(&assoc_sg_walk, 0);
1179                 dst = src;
1180                 if (unlikely(req->src != req->dst)) {
1181                         scatterwalk_start(&dst_sg_walk, req->dst);
1182                         dst = scatterwalk_map(&dst_sg_walk, 0);
1183                 }
1184
1185         } else {
1186                 /* Allocate memory for src, dst, assoc */
1187                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
1188                 if (!src)
1189                         return -ENOMEM;
1190                 assoc = (src + req->cryptlen + auth_tag_len);
1191                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
1192                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
1193                         req->assoclen, 0);
1194                 dst = src;
1195         }
1196
1197         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
1198                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
1199                 authTag, auth_tag_len);
1200
1201         /* Compare generated tag with passed in tag. */
1202         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
1203                 -EBADMSG : 0;
1204
1205         if (one_entry_in_sg) {
1206                 if (unlikely(req->src != req->dst)) {
1207                         scatterwalk_unmap(dst, 0);
1208                         scatterwalk_done(&dst_sg_walk, 0, 0);
1209                 }
1210                 scatterwalk_unmap(src, 0);
1211                 scatterwalk_unmap(assoc, 0);
1212                 scatterwalk_done(&src_sg_walk, 0, 0);
1213                 scatterwalk_done(&assoc_sg_walk, 0, 0);
1214         } else {
1215                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
1216                 kfree(src);
1217         }
1218         return retval;
1219 }
1220
1221 static struct crypto_alg __rfc4106_alg = {
1222         .cra_name               = "__gcm-aes-aesni",
1223         .cra_driver_name        = "__driver-gcm-aes-aesni",
1224         .cra_priority           = 0,
1225         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1226         .cra_blocksize          = 1,
1227         .cra_ctxsize    = sizeof(struct aesni_rfc4106_gcm_ctx) + AESNI_ALIGN,
1228         .cra_alignmask          = 0,
1229         .cra_type               = &crypto_aead_type,
1230         .cra_module             = THIS_MODULE,
1231         .cra_list               = LIST_HEAD_INIT(__rfc4106_alg.cra_list),
1232         .cra_u = {
1233                 .aead = {
1234                         .encrypt        = __driver_rfc4106_encrypt,
1235                         .decrypt        = __driver_rfc4106_decrypt,
1236                 },
1237         },
1238 };
1239 #endif
1240
1241 static int __init aesni_init(void)
1242 {
1243         int err;
1244
1245         if (!cpu_has_aes) {
1246                 printk(KERN_INFO "Intel AES-NI instructions are not detected.\n");
1247                 return -ENODEV;
1248         }
1249
1250         if ((err = crypto_register_alg(&aesni_alg)))
1251                 goto aes_err;
1252         if ((err = crypto_register_alg(&__aesni_alg)))
1253                 goto __aes_err;
1254         if ((err = crypto_register_alg(&blk_ecb_alg)))
1255                 goto blk_ecb_err;
1256         if ((err = crypto_register_alg(&blk_cbc_alg)))
1257                 goto blk_cbc_err;
1258         if ((err = crypto_register_alg(&ablk_ecb_alg)))
1259                 goto ablk_ecb_err;
1260         if ((err = crypto_register_alg(&ablk_cbc_alg)))
1261                 goto ablk_cbc_err;
1262 #ifdef CONFIG_X86_64
1263         if ((err = crypto_register_alg(&blk_ctr_alg)))
1264                 goto blk_ctr_err;
1265         if ((err = crypto_register_alg(&ablk_ctr_alg)))
1266                 goto ablk_ctr_err;
1267         if ((err = crypto_register_alg(&__rfc4106_alg)))
1268                 goto __aead_gcm_err;
1269         if ((err = crypto_register_alg(&rfc4106_alg)))
1270                 goto aead_gcm_err;
1271 #ifdef HAS_CTR
1272         if ((err = crypto_register_alg(&ablk_rfc3686_ctr_alg)))
1273                 goto ablk_rfc3686_ctr_err;
1274 #endif
1275 #endif
1276 #ifdef HAS_LRW
1277         if ((err = crypto_register_alg(&ablk_lrw_alg)))
1278                 goto ablk_lrw_err;
1279 #endif
1280 #ifdef HAS_PCBC
1281         if ((err = crypto_register_alg(&ablk_pcbc_alg)))
1282                 goto ablk_pcbc_err;
1283 #endif
1284 #ifdef HAS_XTS
1285         if ((err = crypto_register_alg(&ablk_xts_alg)))
1286                 goto ablk_xts_err;
1287 #endif
1288         return err;
1289
1290 #ifdef HAS_XTS
1291 ablk_xts_err:
1292 #endif
1293 #ifdef HAS_PCBC
1294         crypto_unregister_alg(&ablk_pcbc_alg);
1295 ablk_pcbc_err:
1296 #endif
1297 #ifdef HAS_LRW
1298         crypto_unregister_alg(&ablk_lrw_alg);
1299 ablk_lrw_err:
1300 #endif
1301 #ifdef CONFIG_X86_64
1302 #ifdef HAS_CTR
1303         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1304 ablk_rfc3686_ctr_err:
1305 #endif
1306         crypto_unregister_alg(&rfc4106_alg);
1307 aead_gcm_err:
1308         crypto_unregister_alg(&__rfc4106_alg);
1309 __aead_gcm_err:
1310         crypto_unregister_alg(&ablk_ctr_alg);
1311 ablk_ctr_err:
1312         crypto_unregister_alg(&blk_ctr_alg);
1313 blk_ctr_err:
1314 #endif
1315         crypto_unregister_alg(&ablk_cbc_alg);
1316 ablk_cbc_err:
1317         crypto_unregister_alg(&ablk_ecb_alg);
1318 ablk_ecb_err:
1319         crypto_unregister_alg(&blk_cbc_alg);
1320 blk_cbc_err:
1321         crypto_unregister_alg(&blk_ecb_alg);
1322 blk_ecb_err:
1323         crypto_unregister_alg(&__aesni_alg);
1324 __aes_err:
1325         crypto_unregister_alg(&aesni_alg);
1326 aes_err:
1327         return err;
1328 }
1329
1330 static void __exit aesni_exit(void)
1331 {
1332 #ifdef HAS_XTS
1333         crypto_unregister_alg(&ablk_xts_alg);
1334 #endif
1335 #ifdef HAS_PCBC
1336         crypto_unregister_alg(&ablk_pcbc_alg);
1337 #endif
1338 #ifdef HAS_LRW
1339         crypto_unregister_alg(&ablk_lrw_alg);
1340 #endif
1341 #ifdef CONFIG_X86_64
1342 #ifdef HAS_CTR
1343         crypto_unregister_alg(&ablk_rfc3686_ctr_alg);
1344 #endif
1345         crypto_unregister_alg(&rfc4106_alg);
1346         crypto_unregister_alg(&__rfc4106_alg);
1347         crypto_unregister_alg(&ablk_ctr_alg);
1348         crypto_unregister_alg(&blk_ctr_alg);
1349 #endif
1350         crypto_unregister_alg(&ablk_cbc_alg);
1351         crypto_unregister_alg(&ablk_ecb_alg);
1352         crypto_unregister_alg(&blk_cbc_alg);
1353         crypto_unregister_alg(&blk_ecb_alg);
1354         crypto_unregister_alg(&__aesni_alg);
1355         crypto_unregister_alg(&aesni_alg);
1356 }
1357
1358 module_init(aesni_init);
1359 module_exit(aesni_exit);
1360
1361 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1362 MODULE_LICENSE("GPL");
1363 MODULE_ALIAS("aes");