target/linux/generic/files/crypto/ocf/c7108/aes-7108.c

   1 /*
   2  * Copyright (C) 2006 Micronas USA
   3  *
   4  * 1. Redistributions of source code must retain the above copyright
   5  *   notice, this list of conditions and the following disclaimer.
   6  * 2. Redistributions in binary form must reproduce the above copyright
   7  *   notice, this list of conditions and the following disclaimer in the
   8  *   documentation and/or other materials provided with the distribution.
   9  * 3. The name of the author may not be used to endorse or promote products
  10  *   derived from this software without specific prior written permission.
  11  *
  12  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  13  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  14  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  15  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  16  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  17  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  18  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  19  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  20  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  21  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  22  *
  23  * Effort sponsored in part by the Defense Advanced Research Projects
  24  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  25  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  26  *
  27  */
  28
  29 //#include <linux/config.h>
  30 #include <linux/module.h>
  31 #include <linux/init.h>
  32 #include <linux/list.h>
  33 #include <linux/slab.h>
  34 #include <linux/sched.h>
  35 #include <linux/wait.h>
  36 #include <linux/crypto.h>
  37 #include <linux/mm.h>
  38 #include <linux/skbuff.h>
  39 #include <linux/random.h>
  40 #include <asm/io.h>
  41 #include <asm/delay.h>
  42 //#include <asm/scatterlist.h>
  43 #include <linux/scatterlist.h>
  44 #include <linux/dma-mapping.h>
  45 #include <linux/highmem.h>
  46 #include <cryptodev.h>
  47 #include <uio.h>
  48 #include <aes-7108.h>
  49
  50 /* Runtime mode */
  51 static int c7108_crypto_mode = C7108_AES_CTRL_MODE_CTR;
  52 //static int c7108_crypto_mode = C7108_AES_CTRL_MODE_CBC;
  53
  54 static int32_t c7108_id = -1;
  55 static struct cipher_7108 **c7108_sessions = NULL;
  56 static u_int32_t c7108_sesnum = 0;
  57 static unsigned long iobar;
  58
  59 /* Crypto entry points */
  60 static  int c7108_process(void *, struct cryptop *, int);
  61 static  int c7108_newsession(void *, u_int32_t *, struct cryptoini *);
  62 static  int c7108_freesession(void *, u_int64_t);
  63
  64 /* Globals */
  65 static int debug = 0;
  66 static spinlock_t csr_mutex;
  67
  68 /* Generic controller-based lock */
  69 #define AES_LOCK()\
  70           spin_lock(&csr_mutex)
  71 #define AES_UNLOCK()\
  72           spin_unlock(&csr_mutex)
  73
  74 /* 7108 AES register access */
  75 #define c7108_reg_wr8(a,d)   iowrite8(d, (void*)(iobar+(a)))
  76 #define c7108_reg_wr16(a,d)  iowrite16(d, (void*)(iobar+(a)))
  77 #define c7108_reg_wr32(a,d)  iowrite32(d, (void*)(iobar+(a)))
  78 #define c7108_reg_rd8(a)     ioread8((void*)(iobar+(a)))
  79 #define c7108_reg_rd16(a)    ioread16((void*)(iobar+(a)))
  80 #define c7108_reg_rd32(a)    ioread32((void*)(iobar+(a)))
  81
  82 static int
  83 c7108_xlate_key(int klen, u8* k8ptr, u32* k32ptr)
  84 {
  85         int i, nw=0;
  86         nw = ((klen >= 256) ? 8 : (klen >= 192) ? 6 : 4);
  87         for ( i = 0; i < nw; i++) {
  88             k32ptr[i] =    (k8ptr[i+3] << 24) | (k8ptr[i+2] << 16) |
  89                            (k8ptr[i+1] << 8)  | k8ptr[i];
  90
  91         }
  92         return 0;
  93 }
  94
  95 static int
  96 c7108_cache_key(int klen, u32* k32ptr, u8* k8ptr)
  97 {
  98         int i, nb=0;
  99         u8* ptr = (u8*)k32ptr;
 100         nb = ((klen >= 256) ? 32 : (klen >= 192) ? 24 : 16);
 101         for ( i = 0; i < nb; i++)
 102             k8ptr[i] = ptr[i];
 103         return 0;
 104 }
 105
 106 static int
 107 c7108_aes_setup_dma(u32 src, u32 dst, u32 len)
 108 {
 109         if (len < 16) {
 110             printk("len < 16\n");
 111             return -10;
 112         }
 113         if (len % 16) {
 114             printk("len not multiple of 16\n");
 115             return -11;
 116         }
 117         c7108_reg_wr16(C7108_AES_DMA_SRC0_LO, (u16) src);
 118         c7108_reg_wr16(C7108_AES_DMA_SRC0_HI, (u16)((src & 0xffff0000) >> 16));
 119         c7108_reg_wr16(C7108_AES_DMA_DST0_LO, (u16) dst);
 120         c7108_reg_wr16(C7108_AES_DMA_DST0_HI, (u16)((dst & 0xffff0000) >> 16));
 121         c7108_reg_wr16(C7108_AES_DMA_LEN, (u16) ((len / 16) - 1));
 122
 123         return 0;
 124 }
 125
 126 static int
 127 c7108_aes_set_hw_iv(u8 iv[16])
 128 {
 129         c7108_reg_wr16(C7108_AES_IV0_LO, (u16) ((iv[1] << 8) | iv[0]));
 130         c7108_reg_wr16(C7108_AES_IV0_HI, (u16) ((iv[3] << 8) | iv[2]));
 131         c7108_reg_wr16(C7108_AES_IV1_LO, (u16) ((iv[5] << 8) | iv[4]));
 132         c7108_reg_wr16(C7108_AES_IV1_HI, (u16) ((iv[7] << 8) | iv[6]));
 133         c7108_reg_wr16(C7108_AES_IV2_LO, (u16) ((iv[9] << 8) | iv[8]));
 134         c7108_reg_wr16(C7108_AES_IV2_HI, (u16) ((iv[11] << 8) | iv[10]));
 135         c7108_reg_wr16(C7108_AES_IV3_LO, (u16) ((iv[13] << 8) | iv[12]));
 136         c7108_reg_wr16(C7108_AES_IV3_HI, (u16) ((iv[15] << 8) | iv[14]));
 137
 138     return 0;
 139 }
 140
 141 static void
 142 c7108_aes_read_dkey(u32 * dkey)
 143 {
 144         dkey[0] = (c7108_reg_rd16(C7108_AES_EKEY0_HI) << 16) |
 145                    c7108_reg_rd16(C7108_AES_EKEY0_LO);
 146         dkey[1] = (c7108_reg_rd16(C7108_AES_EKEY1_HI) << 16) |
 147                    c7108_reg_rd16(C7108_AES_EKEY1_LO);
 148         dkey[2] = (c7108_reg_rd16(C7108_AES_EKEY2_HI) << 16) |
 149                    c7108_reg_rd16(C7108_AES_EKEY2_LO);
 150         dkey[3] = (c7108_reg_rd16(C7108_AES_EKEY3_HI) << 16) |
 151                    c7108_reg_rd16(C7108_AES_EKEY3_LO);
 152         dkey[4] = (c7108_reg_rd16(C7108_AES_EKEY4_HI) << 16) |
 153                    c7108_reg_rd16(C7108_AES_EKEY4_LO);
 154         dkey[5] = (c7108_reg_rd16(C7108_AES_EKEY5_HI) << 16) |
 155                    c7108_reg_rd16(C7108_AES_EKEY5_LO);
 156         dkey[6] = (c7108_reg_rd16(C7108_AES_EKEY6_HI) << 16) |
 157                    c7108_reg_rd16(C7108_AES_EKEY6_LO);
 158         dkey[7] = (c7108_reg_rd16(C7108_AES_EKEY7_HI) << 16) |
 159                    c7108_reg_rd16(C7108_AES_EKEY7_LO);
 160 }
 161
 162 static int
 163 c7108_aes_cipher(int op,
 164                  u32 dst,
 165                  u32 src,
 166                  u32 len,
 167                  int klen,
 168                  u16 mode,
 169                  u32 key[8],
 170                  u8 iv[16])
 171 {
 172         int rv = 0, cnt=0;
 173         u16 ctrl = 0, stat = 0;
 174
 175         AES_LOCK();
 176
 177         /* Setup key length */
 178         if (klen == 128) {
 179             ctrl |= C7108_AES_KEY_LEN_128;
 180         } else if (klen == 192) {
 181             ctrl |= C7108_AES_KEY_LEN_192;
 182         } else if (klen == 256) {
 183             ctrl |= C7108_AES_KEY_LEN_256;
 184         } else {
 185             AES_UNLOCK();
 186             return -3;
 187         }
 188
 189         /* Check opcode */
 190         if (C7108_AES_ENCRYPT == op) {
 191             ctrl |= C7108_AES_ENCRYPT;
 192         } else if (C7108_AES_DECRYPT == op) {
 193             ctrl |= C7108_AES_DECRYPT;
 194         } else {
 195             AES_UNLOCK();
 196             return -4;
 197         }
 198
 199         /* check mode */
 200         if ( (mode != C7108_AES_CTRL_MODE_CBC) &&
 201              (mode != C7108_AES_CTRL_MODE_CFB) &&
 202              (mode != C7108_AES_CTRL_MODE_OFB) &&
 203              (mode != C7108_AES_CTRL_MODE_CTR) &&
 204              (mode != C7108_AES_CTRL_MODE_ECB) ) {
 205             AES_UNLOCK();
 206             return -5;
 207         }
 208
 209         /* Now set mode */
 210         ctrl |= mode;
 211
 212         /* For CFB, OFB, and CTR, neither backward key
 213          * expansion nor key inversion is required.
 214          */
 215         if ( (C7108_AES_DECRYPT == op) &&
 216              (C7108_AES_CTRL_MODE_CBC == mode ||
 217               C7108_AES_CTRL_MODE_ECB == mode ) ){
 218
 219             /* Program Key */
 220             c7108_reg_wr16(C7108_AES_KEY0_LO, (u16) key[4]);
 221             c7108_reg_wr16(C7108_AES_KEY0_HI, (u16) (key[4] >> 16));
 222             c7108_reg_wr16(C7108_AES_KEY1_LO, (u16) key[5]);
 223             c7108_reg_wr16(C7108_AES_KEY1_HI, (u16) (key[5] >> 16));
 224             c7108_reg_wr16(C7108_AES_KEY2_LO, (u16) key[6]);
 225             c7108_reg_wr16(C7108_AES_KEY2_HI, (u16) (key[6] >> 16));
 226             c7108_reg_wr16(C7108_AES_KEY3_LO, (u16) key[7]);
 227             c7108_reg_wr16(C7108_AES_KEY3_HI, (u16) (key[7] >> 16));
 228             c7108_reg_wr16(C7108_AES_KEY6_LO, (u16) key[2]);
 229             c7108_reg_wr16(C7108_AES_KEY6_HI, (u16) (key[2] >> 16));
 230             c7108_reg_wr16(C7108_AES_KEY7_LO, (u16) key[3]);
 231             c7108_reg_wr16(C7108_AES_KEY7_HI, (u16) (key[3] >> 16));
 232
 233
 234             if (192 == klen) {
 235                 c7108_reg_wr16(C7108_AES_KEY4_LO, (u16) key[7]);
 236                 c7108_reg_wr16(C7108_AES_KEY4_HI, (u16) (key[7] >> 16));
 237                 c7108_reg_wr16(C7108_AES_KEY5_LO, (u16) key[7]);
 238                 c7108_reg_wr16(C7108_AES_KEY5_HI, (u16) (key[7] >> 16));
 239
 240             } else if (256 == klen) {
 241                 /* 256 */
 242                 c7108_reg_wr16(C7108_AES_KEY4_LO, (u16) key[0]);
 243                 c7108_reg_wr16(C7108_AES_KEY4_HI, (u16) (key[0] >> 16));
 244                 c7108_reg_wr16(C7108_AES_KEY5_LO, (u16) key[1]);
 245                 c7108_reg_wr16(C7108_AES_KEY5_HI, (u16) (key[1] >> 16));
 246
 247             }
 248
 249         } else {
 250             /* Program Key */
 251             c7108_reg_wr16(C7108_AES_KEY0_LO, (u16) key[0]);
 252             c7108_reg_wr16(C7108_AES_KEY0_HI, (u16) (key[0] >> 16));
 253             c7108_reg_wr16(C7108_AES_KEY1_LO, (u16) key[1]);
 254             c7108_reg_wr16(C7108_AES_KEY1_HI, (u16) (key[1] >> 16));
 255             c7108_reg_wr16(C7108_AES_KEY2_LO, (u16) key[2]);
 256             c7108_reg_wr16(C7108_AES_KEY2_HI, (u16) (key[2] >> 16));
 257             c7108_reg_wr16(C7108_AES_KEY3_LO, (u16) key[3]);
 258             c7108_reg_wr16(C7108_AES_KEY3_HI, (u16) (key[3] >> 16));
 259             c7108_reg_wr16(C7108_AES_KEY4_LO, (u16) key[4]);
 260             c7108_reg_wr16(C7108_AES_KEY4_HI, (u16) (key[4] >> 16));
 261             c7108_reg_wr16(C7108_AES_KEY5_LO, (u16) key[5]);
 262             c7108_reg_wr16(C7108_AES_KEY5_HI, (u16) (key[5] >> 16));
 263             c7108_reg_wr16(C7108_AES_KEY6_LO, (u16) key[6]);
 264             c7108_reg_wr16(C7108_AES_KEY6_HI, (u16) (key[6] >> 16));
 265             c7108_reg_wr16(C7108_AES_KEY7_LO, (u16) key[7]);
 266             c7108_reg_wr16(C7108_AES_KEY7_HI, (u16) (key[7] >> 16));
 267
 268         }
 269
 270         /* Set IV always */
 271         c7108_aes_set_hw_iv(iv);
 272
 273         /* Program DMA addresses */
 274         if ((rv = c7108_aes_setup_dma(src, dst, len)) < 0) {
 275             AES_UNLOCK();
 276             return rv;
 277         }
 278
 279
 280         /* Start AES cipher */
 281         c7108_reg_wr16(C7108_AES_CTRL, ctrl | C7108_AES_GO);
 282
 283         //printk("Ctrl: 0x%x\n", ctrl | C7108_AES_GO);
 284         do {
 285             /* TODO: interrupt mode */
 286             //        printk("aes_stat=0x%x\n", stat);
 287             //udelay(100);
 288         } while ((cnt++ < 1000000) &&
 289                  !((stat=c7108_reg_rd16(C7108_AES_CTRL))&C7108_AES_OP_DONE));
 290
 291
 292         if ((mode == C7108_AES_CTRL_MODE_ECB)||
 293             (mode == C7108_AES_CTRL_MODE_CBC)) {
 294             /* Save out key when the lock is held ... */
 295             c7108_aes_read_dkey(key);
 296         }
 297
 298         AES_UNLOCK();
 299         return 0;
 300
 301 }
 302
 303 /*
 304  * Generate a new crypto device session.
 305  */
 306 static int
 307 c7108_newsession(void *arg, u_int32_t *sid, struct cryptoini *cri)
 308 {
 309         struct cipher_7108 **swd;
 310         u_int32_t i;
 311         char *algo;
 312         int mode, xfm_type;
 313
 314         dprintk("%s()\n", __FUNCTION__);
 315         if (sid == NULL || cri == NULL) {
 316                 dprintk("%s,%d - EINVAL\n", __FILE__, __LINE__);
 317                 return EINVAL;
 318         }
 319
 320         if (c7108_sessions) {
 321                 for (i = 1; i < c7108_sesnum; i++)
 322                         if (c7108_sessions[i] == NULL)
 323                                 break;
 324         } else
 325                 i = 1;          /* NB: to silence compiler warning */
 326
 327         if (c7108_sessions == NULL || i == c7108_sesnum) {
 328             if (c7108_sessions == NULL) {
 329                 i = 1; /* We leave c7108_sessions[0] empty */
 330                 c7108_sesnum = CRYPTO_SW_SESSIONS;
 331             } else
 332                 c7108_sesnum *= 2;
 333
 334             swd = kmalloc(c7108_sesnum * sizeof(struct cipher_7108 *),
 335                           GFP_ATOMIC);
 336             if (swd == NULL) {
 337                 /* Reset session number */
 338                 if (c7108_sesnum == CRYPTO_SW_SESSIONS)
 339                     c7108_sesnum = 0;
 340                 else
 341                     c7108_sesnum /= 2;
 342                 dprintk("%s,%d: ENOBUFS\n", __FILE__, __LINE__);
 343                 return ENOBUFS;
 344             }
 345             memset(swd, 0, c7108_sesnum * sizeof(struct cipher_7108 *));
 346
 347             /* Copy existing sessions */
 348             if (c7108_sessions) {
 349                 memcpy(swd, c7108_sessions,
 350                        (c7108_sesnum / 2) * sizeof(struct cipher_7108 *));
 351                 kfree(c7108_sessions);
 352             }
 353
 354             c7108_sessions = swd;
 355
 356         }
 357
 358         swd = &c7108_sessions[i];
 359         *sid = i;
 360
 361         while (cri) {
 362                 *swd = (struct cipher_7108 *)
 363                     kmalloc(sizeof(struct cipher_7108), GFP_ATOMIC);
 364                 if (*swd == NULL) {
 365                     c7108_freesession(NULL, i);
 366                     dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 367                     return ENOBUFS;
 368                 }
 369                 memset(*swd, 0, sizeof(struct cipher_7108));
 370
 371                 algo = NULL;
 372                 mode = 0;
 373                 xfm_type = HW_TYPE_CIPHER;
 374
 375                 switch (cri->cri_alg) {
 376
 377                 case CRYPTO_AES_CBC:
 378                         algo = "aes";
 379                         mode = CRYPTO_TFM_MODE_CBC;
 380                         c7108_crypto_mode = C7108_AES_CTRL_MODE_CBC;
 381                         break;
 382 #if 0
 383                 case CRYPTO_AES_CTR:
 384                         algo = "aes_ctr";
 385                         mode = CRYPTO_TFM_MODE_CBC;
 386                         c7108_crypto_mode = C7108_AES_CTRL_MODE_CTR;
 387                         break;
 388                 case CRYPTO_AES_ECB:
 389                         algo = "aes_ecb";
 390                         mode = CRYPTO_TFM_MODE_CBC;
 391                         c7108_crypto_mode = C7108_AES_CTRL_MODE_ECB;
 392                         break;
 393                 case CRYPTO_AES_OFB:
 394                         algo = "aes_ofb";
 395                         mode = CRYPTO_TFM_MODE_CBC;
 396                         c7108_crypto_mode = C7108_AES_CTRL_MODE_OFB;
 397                         break;
 398                 case CRYPTO_AES_CFB:
 399                         algo = "aes_cfb";
 400                         mode = CRYPTO_TFM_MODE_CBC;
 401                         c7108_crypto_mode = C7108_AES_CTRL_MODE_CFB;
 402                         break;
 403 #endif
 404                 default:
 405                         printk("unsupported crypto algorithm: %d\n",
 406                                cri->cri_alg);
 407                         return -EINVAL;
 408                         break;
 409                 }
 410
 411
 412                 if (!algo || !*algo) {
 413                     printk("cypher_7108_crypto: Unknown algo 0x%x\n",
 414                            cri->cri_alg);
 415                     c7108_freesession(NULL, i);
 416                     return EINVAL;
 417                 }
 418
 419                 if (xfm_type == HW_TYPE_CIPHER) {
 420                     if (debug) {
 421                         dprintk("%s key:", __FUNCTION__);
 422                         for (i = 0; i < (cri->cri_klen + 7) / 8; i++)
 423                             dprintk("%s0x%02x", (i % 8) ? " " : "\n    ",
 424                                     cri->cri_key[i]);
 425                         dprintk("\n");
 426                     }
 427
 428                 } else if (xfm_type == SW_TYPE_HMAC ||
 429                            xfm_type == SW_TYPE_HASH) {
 430                     printk("cypher_7108_crypto: HMAC unsupported!\n");
 431                     return -EINVAL;
 432                     c7108_freesession(NULL, i);
 433                 } else {
 434                     printk("cypher_7108_crypto: "
 435                            "Unhandled xfm_type %d\n", xfm_type);
 436                     c7108_freesession(NULL, i);
 437                     return EINVAL;
 438                 }
 439
 440                 (*swd)->cri_alg = cri->cri_alg;
 441                 (*swd)->xfm_type = xfm_type;
 442
 443                 cri = cri->cri_next;
 444                 swd = &((*swd)->next);
 445         }
 446         return 0;
 447 }
 448
 449 /*
 450  * Free a session.
 451  */
 452 static int
 453 c7108_freesession(void *arg, u_int64_t tid)
 454 {
 455         struct cipher_7108 *swd;
 456         u_int32_t sid = CRYPTO_SESID2LID(tid);
 457
 458         dprintk("%s()\n", __FUNCTION__);
 459         if (sid > c7108_sesnum || c7108_sessions == NULL ||
 460                         c7108_sessions[sid] == NULL) {
 461                 dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 462                 return(EINVAL);
 463         }
 464
 465         /* Silently accept and return */
 466         if (sid == 0)
 467                 return(0);
 468
 469         while ((swd = c7108_sessions[sid]) != NULL) {
 470                 c7108_sessions[sid] = swd->next;
 471                 kfree(swd);
 472         }
 473         return 0;
 474 }
 475
 476 /*
 477  * Process a hardware request.
 478  */
 479 static int
 480 c7108_process(void *arg, struct cryptop *crp, int hint)
 481 {
 482         struct cryptodesc *crd;
 483         struct cipher_7108 *sw;
 484         u_int32_t lid;
 485         int type;
 486         u32 hwkey[8];
 487
 488 #define SCATTERLIST_MAX 16
 489         struct scatterlist sg[SCATTERLIST_MAX];
 490         int sg_num, sg_len, skip;
 491         struct sk_buff *skb = NULL;
 492         struct uio *uiop = NULL;
 493
 494         dprintk("%s()\n", __FUNCTION__);
 495         /* Sanity check */
 496         if (crp == NULL) {
 497                 dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 498                 return EINVAL;
 499         }
 500
 501         crp->crp_etype = 0;
 502
 503         if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
 504                 dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 505                 crp->crp_etype = EINVAL;
 506                 goto done;
 507         }
 508
 509         lid = crp->crp_sid & 0xffffffff;
 510         if (lid >= c7108_sesnum || lid == 0 || c7108_sessions == NULL ||
 511                         c7108_sessions[lid] == NULL) {
 512                 crp->crp_etype = ENOENT;
 513                 dprintk("%s,%d: ENOENT\n", __FILE__, __LINE__);
 514                 goto done;
 515         }
 516
 517         /*
 518          * do some error checking outside of the loop for SKB and IOV
 519          * processing this leaves us with valid skb or uiop pointers
 520          * for later
 521          */
 522         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 523                 skb = (struct sk_buff *) crp->crp_buf;
 524                 if (skb_shinfo(skb)->nr_frags >= SCATTERLIST_MAX) {
 525                         printk("%s,%d: %d nr_frags > SCATTERLIST_MAX",
 526                                __FILE__, __LINE__,
 527                                skb_shinfo(skb)->nr_frags);
 528                         goto done;
 529                 }
 530         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 531                 uiop = (struct uio *) crp->crp_buf;
 532                 if (uiop->uio_iovcnt > SCATTERLIST_MAX) {
 533                         printk("%s,%d: %d uio_iovcnt > SCATTERLIST_MAX",
 534                                __FILE__, __LINE__,
 535                                uiop->uio_iovcnt);
 536                         goto done;
 537                 }
 538         }
 539
 540         /* Go through crypto descriptors, processing as we go */
 541         for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 542             /*
 543              * Find the crypto context.
 544              *
 545              * XXX Note that the logic here prevents us from having
 546              * XXX the same algorithm multiple times in a session
 547              * XXX (or rather, we can but it won't give us the right
 548              * XXX results). To do that, we'd need some way of differentiating
 549              * XXX between the various instances of an algorithm (so we can
 550              * XXX locate the correct crypto context).
 551              */
 552             for (sw = c7108_sessions[lid];
 553                  sw && sw->cri_alg != crd->crd_alg;
 554                  sw = sw->next)
 555                 ;
 556
 557             /* No such context ? */
 558             if (sw == NULL) {
 559                 crp->crp_etype = EINVAL;
 560                 dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 561                 goto done;
 562             }
 563
 564             skip = crd->crd_skip;
 565
 566             /*
 567              * setup the SG list skip from the start of the buffer
 568              */
 569             memset(sg, 0, sizeof(sg));
 570             if (crp->crp_flags & CRYPTO_F_SKBUF) {
 571                 int i, len;
 572                 type = CRYPTO_BUF_SKBUF;
 573
 574                 sg_num = 0;
 575                 sg_len = 0;
 576
 577                 if (skip < skb_headlen(skb)) {
 578                     //sg[sg_num].page   = virt_to_page(skb->data + skip);
 579                         //sg[sg_num].offset = offset_in_page(skb->data + skip);
 580                     len = skb_headlen(skb) - skip;
 581                     if (len + sg_len > crd->crd_len)
 582                         len = crd->crd_len - sg_len;
 583                     //sg[sg_num].length = len;
 584                     sg_set_page(&sg[sg_num], virt_to_page(skb->data + skip), len, offset_in_page(skb->data + skip));
 585                         sg_len += sg[sg_num].length;
 586                     sg_num++;
 587                     skip = 0;
 588                 } else
 589                     skip -= skb_headlen(skb);
 590
 591                 for (i = 0; sg_len < crd->crd_len &&
 592                          i < skb_shinfo(skb)->nr_frags &&
 593                          sg_num < SCATTERLIST_MAX; i++) {
 594                     if (skip < skb_shinfo(skb)->frags[i].size) {
 595                         //sg[sg_num].page   = skb_shinfo(skb)->frags[i].page;
 596                         //sg[sg_num].offset = skb_shinfo(skb)->frags[i].page_offset + skip;
 597                         len = skb_shinfo(skb)->frags[i].size - skip;
 598                         if (len + sg_len > crd->crd_len)
 599                             len = crd->crd_len - sg_len;
 600                         //sg[sg_num].length = len;
 601                         sg_set_page(&sg[sg_num], skb_shinfo(skb)->frags[i].page, len, skb_shinfo(skb)->frags[i].page_offset + skip);
 602                         sg_len += sg[sg_num].length;
 603                         sg_num++;
 604                         skip = 0;
 605                     } else
 606                         skip -= skb_shinfo(skb)->frags[i].size;
 607                 }
 608             } else if (crp->crp_flags & CRYPTO_F_IOV) {
 609                 int len;
 610                 type = CRYPTO_BUF_IOV;
 611                 sg_len = 0;
 612                 for (sg_num = 0; sg_len < crd->crd_len &&
 613                          sg_num < uiop->uio_iovcnt &&
 614                          sg_num < SCATTERLIST_MAX; sg_num++) {
 615                     if (skip < uiop->uio_iov[sg_num].iov_len) {
 616                         //sg[sg_num].page   =                       virt_to_page(uiop->uio_iov[sg_num].iov_base+skip);
 617                         //sg[sg_num].offset =                      offset_in_page(uiop->uio_iov[sg_num].iov_base+skip);
 618                         len = uiop->uio_iov[sg_num].iov_len - skip;
 619                         if (len + sg_len > crd->crd_len)
 620                             len = crd->crd_len - sg_len;
 621                         //sg[sg_num].length = len;
 622                         sg_set_page(&sg[sg_num], virt_to_page(uiop->uio_iov[sg_num].iov_base+skip), len, offset_in_page(uiop->uio_iov[sg_num].iov_base+skip));
 623                         sg_len += sg[sg_num].length;
 624                         skip = 0;
 625                     } else
 626                         skip -= uiop->uio_iov[sg_num].iov_len;
 627                 }
 628             } else {
 629                 type = CRYPTO_BUF_CONTIG;
 630                 //sg[0].page   = virt_to_page(crp->crp_buf + skip);
 631                 //sg[0].offset = offset_in_page(crp->crp_buf + skip);
 632                 sg_len = (crp->crp_ilen - skip);
 633                 if (sg_len > crd->crd_len)
 634                     sg_len = crd->crd_len;
 635                 //sg[0].length = sg_len;
 636                 sg_set_page(&sg[0], virt_to_page(crp->crp_buf + skip), sg_len, offset_in_page(crp->crp_buf + skip));
 637                 sg_num = 1;
 638             }
 639
 640
 641             switch (sw->xfm_type) {
 642
 643             case HW_TYPE_CIPHER: {
 644
 645                 unsigned char iv[64];
 646                 unsigned char *ivp = iv;
 647                 int i;
 648                 int ivsize = 16;    /* fixed for AES */
 649                 int blocksize = 16; /* fixed for AES */
 650
 651                 if (sg_len < blocksize) {
 652                     crp->crp_etype = EINVAL;
 653                     dprintk("%s,%d: EINVAL len %d < %d\n",
 654                             __FILE__, __LINE__,
 655                             sg_len,
 656                             blocksize);
 657                     goto done;
 658                 }
 659
 660                 if (ivsize > sizeof(iv)) {
 661                     crp->crp_etype = EINVAL;
 662                     dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 663                     goto done;
 664                 }
 665
 666                 if (crd->crd_flags & CRD_F_ENCRYPT) { /* encrypt */
 667
 668                     if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
 669                         ivp = crd->crd_iv;
 670                     } else {
 671                         get_random_bytes(ivp, ivsize);
 672                     }
 673                     /*
 674                      * do we have to copy the IV back to the buffer ?
 675                      */
 676                     if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 677                             crypto_copyback(crp->crp_buf,
 678                                           crd->crd_inject,
 679                                           ivsize,
 680                                           (caddr_t)ivp);
 681                     }
 682
 683                     c7108_xlate_key(crd->crd_klen,
 684                                     (u8*)crd->crd_key, (u32*)hwkey);
 685
 686                     /* Encrypt SG list */
 687                     for (i = 0; i < sg_num; i++) {
 688                         sg[i].dma_address =
 689                             dma_map_single(NULL,
 690                                            kmap(sg_page(&sg[i])) + sg[i].offset, sg_len, DMA_BIDIRECTIONAL);
 691 #if 0
 692                         printk("sg[%d]:0x%08x, off 0x%08x "
 693                                "kmap 0x%08x phys 0x%08x\n",
 694                                i, sg[i].page, sg[i].offset,
 695                                kmap(sg[i].page) + sg[i].offset,
 696                                sg[i].dma_address);
 697 #endif
 698                         c7108_aes_cipher(C7108_AES_ENCRYPT,
 699                                          sg[i].dma_address,
 700                                          sg[i].dma_address,
 701                                          sg_len,
 702                                          crd->crd_klen,
 703                                          c7108_crypto_mode,
 704                                          hwkey,
 705                                          ivp);
 706
 707                         if ((c7108_crypto_mode == C7108_AES_CTRL_MODE_CBC)||
 708                             (c7108_crypto_mode == C7108_AES_CTRL_MODE_ECB)) {
 709                             /* Read back expanded key and cache it in key
 710                              * context.
 711                              * NOTE: for ECB/CBC modes only (not CTR, CFB, OFB)
 712                              *       where you set the key once.
 713                              */
 714                             c7108_cache_key(crd->crd_klen,
 715                                             (u32*)hwkey, (u8*)crd->crd_key);
 716 #if 0
 717                             printk("%s expanded key:", __FUNCTION__);
 718                             for (i = 0; i < (crd->crd_klen + 7) / 8; i++)
 719                                 printk("%s0x%02x", (i % 8) ? " " : "\n    ",
 720                                        crd->crd_key[i]);
 721                             printk("\n");
 722 #endif
 723                         }
 724                     }
 725                 }
 726                 else { /*decrypt */
 727
 728                     if (crd->crd_flags & CRD_F_IV_EXPLICIT) {
 729                         ivp = crd->crd_iv;
 730                     } else {
 731                         crypto_copydata(crp->crp_buf, crd->crd_inject,
 732                                   ivsize, (caddr_t)ivp);
 733                     }
 734
 735                     c7108_xlate_key(crd->crd_klen,
 736                                     (u8*)crd->crd_key, (u32*)hwkey);
 737
 738                     /* Decrypt SG list */
 739                     for (i = 0; i < sg_num; i++) {
 740                         sg[i].dma_address =
 741                             dma_map_single(NULL,
 742                                            kmap(sg_page(&sg[i])) + sg[i].offset,
 743                                            sg_len, DMA_BIDIRECTIONAL);
 744
 745 #if 0
 746                         printk("sg[%d]:0x%08x, off 0x%08x "
 747                                "kmap 0x%08x phys 0x%08x\n",
 748                                i, sg[i].page, sg[i].offset,
 749                                kmap(sg[i].page) + sg[i].offset,
 750                                sg[i].dma_address);
 751 #endif
 752                         c7108_aes_cipher(C7108_AES_DECRYPT,
 753                                          sg[i].dma_address,
 754                                          sg[i].dma_address,
 755                                          sg_len,
 756                                          crd->crd_klen,
 757                                          c7108_crypto_mode,
 758                                          hwkey,
 759                                          ivp);
 760                     }
 761                 }
 762             } break;
 763             case SW_TYPE_HMAC:
 764             case SW_TYPE_HASH:
 765                 crp->crp_etype = EINVAL;
 766                 goto done;
 767                 break;
 768
 769             case SW_TYPE_COMP:
 770                 crp->crp_etype = EINVAL;
 771                 goto done;
 772                 break;
 773
 774             default:
 775                 /* Unknown/unsupported algorithm */
 776                 dprintk("%s,%d: EINVAL\n", __FILE__, __LINE__);
 777                 crp->crp_etype = EINVAL;
 778                 goto done;
 779             }
 780         }
 781
 782 done:
 783         crypto_done(crp);
 784         return 0;
 785 }
 786
 787 static struct {
 788         softc_device_decl sc_dev;
 789 } a7108dev;
 790
 791 static device_method_t a7108_methods = {
 792 /* crypto device methods */
 793         DEVMETHOD(cryptodev_newsession, c7108_newsession),
 794         DEVMETHOD(cryptodev_freesession, c7108_freesession),
 795         DEVMETHOD(cryptodev_process, c7108_process),
 796         DEVMETHOD(cryptodev_kprocess, NULL)
 797 };
 798
 799 static int
 800 cypher_7108_crypto_init(void)
 801 {
 802         dprintk("%s(%p)\n", __FUNCTION__, cypher_7108_crypto_init);
 803
 804         iobar = (unsigned long)ioremap(CCU_AES_REG_BASE, 0x4000);
 805         printk("7108: AES @ 0x%08x (0x%08x phys) %s mode\n",
 806                iobar, CCU_AES_REG_BASE,
 807                c7108_crypto_mode & C7108_AES_CTRL_MODE_CBC ? "CBC" :
 808                c7108_crypto_mode & C7108_AES_CTRL_MODE_ECB ? "ECB" :
 809                c7108_crypto_mode & C7108_AES_CTRL_MODE_CTR ? "CTR" :
 810                c7108_crypto_mode & C7108_AES_CTRL_MODE_CFB ? "CFB" :
 811                c7108_crypto_mode & C7108_AES_CTRL_MODE_OFB ? "OFB" : "???");
 812         csr_mutex  = SPIN_LOCK_UNLOCKED;
 813
 814         memset(&a7108dev, 0, sizeof(a7108dev));
 815         softc_device_init(&a7108dev, "aes7108", 0, a7108_methods);
 816
 817         c7108_id = crypto_get_driverid(softc_get_device(&a7108dev), CRYPTOCAP_F_HARDWARE);
 818         if (c7108_id < 0)
 819                 panic("7108: crypto device cannot initialize!");
 820
 821 //      crypto_register(c7108_id, CRYPTO_AES_CBC, 0, 0, c7108_newsession, c7108_freesession, c7108_process, NULL);
 822         crypto_register(c7108_id, CRYPTO_AES_CBC, 0, 0);
 823
 824         return(0);
 825 }
 826
 827 static void
 828 cypher_7108_crypto_exit(void)
 829 {
 830         dprintk("%s()\n", __FUNCTION__);
 831         crypto_unregister_all(c7108_id);
 832         c7108_id = -1;
 833 }
 834
 835 module_init(cypher_7108_crypto_init);
 836 module_exit(cypher_7108_crypto_exit);
 837
 838 MODULE_LICENSE("Dual BSD/GPL");
 839 MODULE_DESCRIPTION("Cypher 7108 Crypto (OCF module for kernel crypto)");