target/linux/generic/files/crypto/ocf/safe/safe.c

   1 /*-
   2  * Linux port done by David McCullough <david_mccullough@mcafee.com>
   3  * Copyright (C) 2004-2010 David McCullough
   4  * The license and original author are listed below.
   5  *
   6  * Copyright (c) 2003 Sam Leffler, Errno Consulting
   7  * Copyright (c) 2003 Global Technology Associates, Inc.
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31 __FBSDID("$FreeBSD: src/sys/dev/safe/safe.c,v 1.18 2007/03/21 03:42:50 sam Exp $");
  32  */
  33
  34 #include <linux/version.h>
  35 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,33))
  36 #include <generated/autoconf.h>
  37 #else
  38 #include <linux/autoconf.h>
  39 #endif
  40 #include <linux/module.h>
  41 #include <linux/kernel.h>
  42 #include <linux/init.h>
  43 #include <linux/list.h>
  44 #include <linux/slab.h>
  45 #include <linux/wait.h>
  46 #include <linux/sched.h>
  47 #include <linux/pci.h>
  48 #include <linux/delay.h>
  49 #include <linux/interrupt.h>
  50 #include <linux/spinlock.h>
  51 #include <linux/random.h>
  52 #include <linux/version.h>
  53 #include <linux/skbuff.h>
  54 #include <asm/io.h>
  55
  56 /*
  57  * SafeNet SafeXcel-1141 hardware crypto accelerator
  58  */
  59
  60 #include <cryptodev.h>
  61 #include <uio.h>
  62 #include <safe/safereg.h>
  63 #include <safe/safevar.h>
  64
  65 #if 1
  66 #define DPRINTF(a)      do { \
  67                                                 if (debug) { \
  68                                                         printk("%s: ", sc ? \
  69                                                                 device_get_nameunit(sc->sc_dev) : "safe"); \
  70                                                         printk a; \
  71                                                 } \
  72                                         } while (0)
  73 #else
  74 #define DPRINTF(a)
  75 #endif
  76
  77 /*
  78  * until we find a cleaner way, include the BSD md5/sha1 code
  79  * here
  80  */
  81 #define HMAC_HACK 1
  82 #ifdef HMAC_HACK
  83 #define LITTLE_ENDIAN 1234
  84 #define BIG_ENDIAN 4321
  85 #ifdef __LITTLE_ENDIAN
  86 #define BYTE_ORDER LITTLE_ENDIAN
  87 #endif
  88 #ifdef __BIG_ENDIAN
  89 #define BYTE_ORDER BIG_ENDIAN
  90 #endif
  91 #include <safe/md5.h>
  92 #include <safe/md5.c>
  93 #include <safe/sha1.h>
  94 #include <safe/sha1.c>
  95
  96 u_int8_t hmac_ipad_buffer[64] = {
  97     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
  98     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
  99     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
 100     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
 101     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
 102     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
 103     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36,
 104     0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36, 0x36
 105 };
 106
 107 u_int8_t hmac_opad_buffer[64] = {
 108     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 109     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 110     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 111     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 112     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 113     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 114     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C,
 115     0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C, 0x5C
 116 };
 117 #endif /* HMAC_HACK */
 118
 119 /* add proc entry for this */
 120 struct safe_stats safestats;
 121
 122 #define debug safe_debug
 123 int safe_debug = 0;
 124 module_param(safe_debug, int, 0644);
 125 MODULE_PARM_DESC(safe_debug, "Enable debug");
 126
 127 static  void safe_callback(struct safe_softc *, struct safe_ringentry *);
 128 static  void safe_feed(struct safe_softc *, struct safe_ringentry *);
 129 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
 130 static  void safe_rng_init(struct safe_softc *);
 131 int safe_rngbufsize = 8;                /* 32 bytes each read  */
 132 module_param(safe_rngbufsize, int, 0644);
 133 MODULE_PARM_DESC(safe_rngbufsize, "RNG polling buffer size (32-bit words)");
 134 int safe_rngmaxalarm = 8;               /* max alarms before reset */
 135 module_param(safe_rngmaxalarm, int, 0644);
 136 MODULE_PARM_DESC(safe_rngmaxalarm, "RNG max alarms before reset");
 137 #endif /* SAFE_NO_RNG */
 138
 139 static void safe_totalreset(struct safe_softc *sc);
 140 static int safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op);
 141 static int safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op);
 142 static int safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re);
 143 static int safe_kprocess(device_t dev, struct cryptkop *krp, int hint);
 144 static int safe_kstart(struct safe_softc *sc);
 145 static int safe_ksigbits(struct safe_softc *sc, struct crparam *cr);
 146 static void safe_kfeed(struct safe_softc *sc);
 147 static void safe_kpoll(unsigned long arg);
 148 static void safe_kload_reg(struct safe_softc *sc, u_int32_t off,
 149                                                                 u_int32_t len, struct crparam *n);
 150
 151 static  int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
 152 static  int safe_freesession(device_t, u_int64_t);
 153 static  int safe_process(device_t, struct cryptop *, int);
 154
 155 static device_method_t safe_methods = {
 156         /* crypto device methods */
 157         DEVMETHOD(cryptodev_newsession, safe_newsession),
 158         DEVMETHOD(cryptodev_freesession,safe_freesession),
 159         DEVMETHOD(cryptodev_process,    safe_process),
 160         DEVMETHOD(cryptodev_kprocess,   safe_kprocess),
 161 };
 162
 163 #define READ_REG(sc,r)                  readl((sc)->sc_base_addr + (r))
 164 #define WRITE_REG(sc,r,val)             writel((val), (sc)->sc_base_addr + (r))
 165
 166 #define SAFE_MAX_CHIPS 8
 167 static struct safe_softc *safe_chip_idx[SAFE_MAX_CHIPS];
 168
 169 /*
 170  * split our buffers up into safe DMAable byte fragments to avoid lockup
 171  * bug in 1141 HW on rev 1.0.
 172  */
 173
 174 static int
 175 pci_map_linear(
 176         struct safe_softc *sc,
 177         struct safe_operand *buf,
 178         void *addr,
 179         int len)
 180 {
 181         dma_addr_t tmp;
 182         int chunk, tlen = len;
 183
 184         tmp = pci_map_single(sc->sc_pcidev, addr, len, PCI_DMA_BIDIRECTIONAL);
 185
 186         buf->mapsize += len;
 187         while (len > 0) {
 188                 chunk = (len > sc->sc_max_dsize) ? sc->sc_max_dsize : len;
 189                 buf->segs[buf->nsegs].ds_addr = tmp;
 190                 buf->segs[buf->nsegs].ds_len  = chunk;
 191                 buf->segs[buf->nsegs].ds_tlen = tlen;
 192                 buf->nsegs++;
 193                 tmp  += chunk;
 194                 len  -= chunk;
 195                 tlen = 0;
 196         }
 197         return 0;
 198 }
 199
 200 /*
 201  * map in a given uio buffer (great on some arches :-)
 202  */
 203
 204 static int
 205 pci_map_uio(struct safe_softc *sc, struct safe_operand *buf, struct uio *uio)
 206 {
 207         struct iovec *iov = uio->uio_iov;
 208         int n;
 209
 210         DPRINTF(("%s()\n", __FUNCTION__));
 211
 212         buf->mapsize = 0;
 213         buf->nsegs = 0;
 214
 215         for (n = 0; n < uio->uio_iovcnt; n++) {
 216                 pci_map_linear(sc, buf, iov->iov_base, iov->iov_len);
 217                 iov++;
 218         }
 219
 220         /* identify this buffer by the first segment */
 221         buf->map = (void *) buf->segs[0].ds_addr;
 222         return(0);
 223 }
 224
 225 /*
 226  * map in a given sk_buff
 227  */
 228
 229 static int
 230 pci_map_skb(struct safe_softc *sc,struct safe_operand *buf,struct sk_buff *skb)
 231 {
 232         int i;
 233
 234         DPRINTF(("%s()\n", __FUNCTION__));
 235
 236         buf->mapsize = 0;
 237         buf->nsegs = 0;
 238
 239         pci_map_linear(sc, buf, skb->data, skb_headlen(skb));
 240
 241         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 242                 pci_map_linear(sc, buf,
 243                                 page_address(skb_shinfo(skb)->frags[i].page) +
 244                                                         skb_shinfo(skb)->frags[i].page_offset,
 245                                 skb_shinfo(skb)->frags[i].size);
 246         }
 247
 248         /* identify this buffer by the first segment */
 249         buf->map = (void *) buf->segs[0].ds_addr;
 250         return(0);
 251 }
 252
 253
 254 #if 0 /* not needed at this time */
 255 static void
 256 pci_sync_operand(struct safe_softc *sc, struct safe_operand *buf)
 257 {
 258         int i;
 259
 260         DPRINTF(("%s()\n", __FUNCTION__));
 261         for (i = 0; i < buf->nsegs; i++)
 262                 pci_dma_sync_single_for_cpu(sc->sc_pcidev, buf->segs[i].ds_addr,
 263                                 buf->segs[i].ds_len, PCI_DMA_BIDIRECTIONAL);
 264 }
 265 #endif
 266
 267 static void
 268 pci_unmap_operand(struct safe_softc *sc, struct safe_operand *buf)
 269 {
 270         int i;
 271         DPRINTF(("%s()\n", __FUNCTION__));
 272         for (i = 0; i < buf->nsegs; i++) {
 273                 if (buf->segs[i].ds_tlen) {
 274                         DPRINTF(("%s - unmap %d 0x%x %d\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
 275                         pci_unmap_single(sc->sc_pcidev, buf->segs[i].ds_addr,
 276                                         buf->segs[i].ds_tlen, PCI_DMA_BIDIRECTIONAL);
 277                         DPRINTF(("%s - unmap %d 0x%x %d done\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
 278                 }
 279                 buf->segs[i].ds_addr = 0;
 280                 buf->segs[i].ds_len = 0;
 281                 buf->segs[i].ds_tlen = 0;
 282         }
 283         buf->nsegs = 0;
 284         buf->mapsize = 0;
 285         buf->map = 0;
 286 }
 287
 288
 289 /*
 290  * SafeXcel Interrupt routine
 291  */
 292 static irqreturn_t
 293 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 294 safe_intr(int irq, void *arg)
 295 #else
 296 safe_intr(int irq, void *arg, struct pt_regs *regs)
 297 #endif
 298 {
 299         struct safe_softc *sc = arg;
 300         int stat;
 301         unsigned long flags;
 302
 303         stat = READ_REG(sc, SAFE_HM_STAT);
 304
 305         DPRINTF(("%s(stat=0x%x)\n", __FUNCTION__, stat));
 306
 307         if (stat == 0)          /* shared irq, not for us */
 308                 return IRQ_NONE;
 309
 310         WRITE_REG(sc, SAFE_HI_CLR, stat);       /* IACK */
 311
 312         if ((stat & SAFE_INT_PE_DDONE)) {
 313                 /*
 314                  * Descriptor(s) done; scan the ring and
 315                  * process completed operations.
 316                  */
 317                 spin_lock_irqsave(&sc->sc_ringmtx, flags);
 318                 while (sc->sc_back != sc->sc_front) {
 319                         struct safe_ringentry *re = sc->sc_back;
 320
 321 #ifdef SAFE_DEBUG
 322                         if (debug) {
 323                                 safe_dump_ringstate(sc, __func__);
 324                                 safe_dump_request(sc, __func__, re);
 325                         }
 326 #endif
 327                         /*
 328                          * safe_process marks ring entries that were allocated
 329                          * but not used with a csr of zero.  This insures the
 330                          * ring front pointer never needs to be set backwards
 331                          * in the event that an entry is allocated but not used
 332                          * because of a setup error.
 333                          */
 334                         DPRINTF(("%s re->re_desc.d_csr=0x%x\n", __FUNCTION__, re->re_desc.d_csr));
 335                         if (re->re_desc.d_csr != 0) {
 336                                 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) {
 337                                         DPRINTF(("%s !CSR_IS_DONE\n", __FUNCTION__));
 338                                         break;
 339                                 }
 340                                 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) {
 341                                         DPRINTF(("%s !LEN_IS_DONE\n", __FUNCTION__));
 342                                         break;
 343                                 }
 344                                 sc->sc_nqchip--;
 345                                 safe_callback(sc, re);
 346                         }
 347                         if (++(sc->sc_back) == sc->sc_ringtop)
 348                                 sc->sc_back = sc->sc_ring;
 349                 }
 350                 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
 351         }
 352
 353         /*
 354          * Check to see if we got any DMA Error
 355          */
 356         if (stat & SAFE_INT_PE_ERROR) {
 357                 printk("%s: dmaerr dmastat %08x\n", device_get_nameunit(sc->sc_dev),
 358                                 (int)READ_REG(sc, SAFE_PE_DMASTAT));
 359                 safestats.st_dmaerr++;
 360                 safe_totalreset(sc);
 361 #if 0
 362                 safe_feed(sc);
 363 #endif
 364         }
 365
 366         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 367                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 368                 DPRINTF(("%s: wakeup crypto %x\n", __func__,
 369                         sc->sc_needwakeup));
 370                 sc->sc_needwakeup &= ~wakeup;
 371                 crypto_unblock(sc->sc_cid, wakeup);
 372         }
 373
 374         return IRQ_HANDLED;
 375 }
 376
 377 /*
 378  * safe_feed() - post a request to chip
 379  */
 380 static void
 381 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
 382 {
 383         DPRINTF(("%s()\n", __FUNCTION__));
 384 #ifdef SAFE_DEBUG
 385         if (debug) {
 386                 safe_dump_ringstate(sc, __func__);
 387                 safe_dump_request(sc, __func__, re);
 388         }
 389 #endif
 390         sc->sc_nqchip++;
 391         if (sc->sc_nqchip > safestats.st_maxqchip)
 392                 safestats.st_maxqchip = sc->sc_nqchip;
 393         /* poke h/w to check descriptor ring, any value can be written */
 394         WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
 395 }
 396
 397 #define N(a)    (sizeof(a) / sizeof (a[0]))
 398 static void
 399 safe_setup_enckey(struct safe_session *ses, caddr_t key)
 400 {
 401         int i;
 402
 403         bcopy(key, ses->ses_key, ses->ses_klen / 8);
 404
 405         /* PE is little-endian, insure proper byte order */
 406         for (i = 0; i < N(ses->ses_key); i++)
 407                 ses->ses_key[i] = htole32(ses->ses_key[i]);
 408 }
 409
 410 static void
 411 safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
 412 {
 413 #ifdef HMAC_HACK
 414         MD5_CTX md5ctx;
 415         SHA1_CTX sha1ctx;
 416         int i;
 417
 418
 419         for (i = 0; i < klen; i++)
 420                 key[i] ^= HMAC_IPAD_VAL;
 421
 422         if (algo == CRYPTO_MD5_HMAC) {
 423                 MD5Init(&md5ctx);
 424                 MD5Update(&md5ctx, key, klen);
 425                 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 426                 bcopy(md5ctx.md5_st8, ses->ses_hminner, sizeof(md5ctx.md5_st8));
 427         } else {
 428                 SHA1Init(&sha1ctx);
 429                 SHA1Update(&sha1ctx, key, klen);
 430                 SHA1Update(&sha1ctx, hmac_ipad_buffer,
 431                     SHA1_HMAC_BLOCK_LEN - klen);
 432                 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 433         }
 434
 435         for (i = 0; i < klen; i++)
 436                 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 437
 438         if (algo == CRYPTO_MD5_HMAC) {
 439                 MD5Init(&md5ctx);
 440                 MD5Update(&md5ctx, key, klen);
 441                 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 442                 bcopy(md5ctx.md5_st8, ses->ses_hmouter, sizeof(md5ctx.md5_st8));
 443         } else {
 444                 SHA1Init(&sha1ctx);
 445                 SHA1Update(&sha1ctx, key, klen);
 446                 SHA1Update(&sha1ctx, hmac_opad_buffer,
 447                     SHA1_HMAC_BLOCK_LEN - klen);
 448                 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 449         }
 450
 451         for (i = 0; i < klen; i++)
 452                 key[i] ^= HMAC_OPAD_VAL;
 453
 454 #if 0
 455         /*
 456          * this code prevents SHA working on a BE host,
 457          * so it is obviously wrong.  I think the byte
 458          * swap setup we do with the chip fixes this for us
 459          */
 460
 461         /* PE is little-endian, insure proper byte order */
 462         for (i = 0; i < N(ses->ses_hminner); i++) {
 463                 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
 464                 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
 465         }
 466 #endif
 467 #else /* HMAC_HACK */
 468         printk("safe: md5/sha not implemented\n");
 469 #endif /* HMAC_HACK */
 470 }
 471 #undef N
 472
 473 /*
 474  * Allocate a new 'session' and return an encoded session id.  'sidp'
 475  * contains our registration id, and should contain an encoded session
 476  * id on successful allocation.
 477  */
 478 static int
 479 safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 480 {
 481         struct safe_softc *sc = device_get_softc(dev);
 482         struct cryptoini *c, *encini = NULL, *macini = NULL;
 483         struct safe_session *ses = NULL;
 484         int sesn;
 485
 486         DPRINTF(("%s()\n", __FUNCTION__));
 487
 488         if (sidp == NULL || cri == NULL || sc == NULL)
 489                 return (EINVAL);
 490
 491         for (c = cri; c != NULL; c = c->cri_next) {
 492                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 493                     c->cri_alg == CRYPTO_SHA1_HMAC ||
 494                     c->cri_alg == CRYPTO_NULL_HMAC) {
 495                         if (macini)
 496                                 return (EINVAL);
 497                         macini = c;
 498                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 499                     c->cri_alg == CRYPTO_3DES_CBC ||
 500                     c->cri_alg == CRYPTO_AES_CBC ||
 501                     c->cri_alg == CRYPTO_NULL_CBC) {
 502                         if (encini)
 503                                 return (EINVAL);
 504                         encini = c;
 505                 } else
 506                         return (EINVAL);
 507         }
 508         if (encini == NULL && macini == NULL)
 509                 return (EINVAL);
 510         if (encini) {                   /* validate key length */
 511                 switch (encini->cri_alg) {
 512                 case CRYPTO_DES_CBC:
 513                         if (encini->cri_klen != 64)
 514                                 return (EINVAL);
 515                         break;
 516                 case CRYPTO_3DES_CBC:
 517                         if (encini->cri_klen != 192)
 518                                 return (EINVAL);
 519                         break;
 520                 case CRYPTO_AES_CBC:
 521                         if (encini->cri_klen != 128 &&
 522                             encini->cri_klen != 192 &&
 523                             encini->cri_klen != 256)
 524                                 return (EINVAL);
 525                         break;
 526                 }
 527         }
 528
 529         if (sc->sc_sessions == NULL) {
 530                 ses = sc->sc_sessions = (struct safe_session *)
 531                         kmalloc(sizeof(struct safe_session), SLAB_ATOMIC);
 532                 if (ses == NULL)
 533                         return (ENOMEM);
 534                 memset(ses, 0, sizeof(struct safe_session));
 535                 sesn = 0;
 536                 sc->sc_nsessions = 1;
 537         } else {
 538                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 539                         if (sc->sc_sessions[sesn].ses_used == 0) {
 540                                 ses = &sc->sc_sessions[sesn];
 541                                 break;
 542                         }
 543                 }
 544
 545                 if (ses == NULL) {
 546                         sesn = sc->sc_nsessions;
 547                         ses = (struct safe_session *)
 548                                 kmalloc((sesn + 1) * sizeof(struct safe_session), SLAB_ATOMIC);
 549                         if (ses == NULL)
 550                                 return (ENOMEM);
 551                         memset(ses, 0, (sesn + 1) * sizeof(struct safe_session));
 552                         bcopy(sc->sc_sessions, ses, sesn *
 553                             sizeof(struct safe_session));
 554                         bzero(sc->sc_sessions, sesn *
 555                             sizeof(struct safe_session));
 556                         kfree(sc->sc_sessions);
 557                         sc->sc_sessions = ses;
 558                         ses = &sc->sc_sessions[sesn];
 559                         sc->sc_nsessions++;
 560                 }
 561         }
 562
 563         bzero(ses, sizeof(struct safe_session));
 564         ses->ses_used = 1;
 565
 566         if (encini) {
 567                 /* get an IV */
 568                 /* XXX may read fewer than requested */
 569                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 570
 571                 ses->ses_klen = encini->cri_klen;
 572                 if (encini->cri_key != NULL)
 573                         safe_setup_enckey(ses, encini->cri_key);
 574         }
 575
 576         if (macini) {
 577                 ses->ses_mlen = macini->cri_mlen;
 578                 if (ses->ses_mlen == 0) {
 579                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 580                                 ses->ses_mlen = MD5_HASH_LEN;
 581                         else
 582                                 ses->ses_mlen = SHA1_HASH_LEN;
 583                 }
 584
 585                 if (macini->cri_key != NULL) {
 586                         safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
 587                             macini->cri_klen / 8);
 588                 }
 589         }
 590
 591         *sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
 592         return (0);
 593 }
 594
 595 /*
 596  * Deallocate a session.
 597  */
 598 static int
 599 safe_freesession(device_t dev, u_int64_t tid)
 600 {
 601         struct safe_softc *sc = device_get_softc(dev);
 602         int session, ret;
 603         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 604
 605         DPRINTF(("%s()\n", __FUNCTION__));
 606
 607         if (sc == NULL)
 608                 return (EINVAL);
 609
 610         session = SAFE_SESSION(sid);
 611         if (session < sc->sc_nsessions) {
 612                 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
 613                 ret = 0;
 614         } else
 615                 ret = EINVAL;
 616         return (ret);
 617 }
 618
 619
 620 static int
 621 safe_process(device_t dev, struct cryptop *crp, int hint)
 622 {
 623         struct safe_softc *sc = device_get_softc(dev);
 624         int err = 0, i, nicealign, uniform;
 625         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 626         int bypass, oplen, ivsize;
 627         caddr_t iv;
 628         int16_t coffset;
 629         struct safe_session *ses;
 630         struct safe_ringentry *re;
 631         struct safe_sarec *sa;
 632         struct safe_pdesc *pd;
 633         u_int32_t cmd0, cmd1, staterec;
 634         unsigned long flags;
 635
 636         DPRINTF(("%s()\n", __FUNCTION__));
 637
 638         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 639                 safestats.st_invalid++;
 640                 return (EINVAL);
 641         }
 642         if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 643                 safestats.st_badsession++;
 644                 return (EINVAL);
 645         }
 646
 647         spin_lock_irqsave(&sc->sc_ringmtx, flags);
 648         if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
 649                 safestats.st_ringfull++;
 650                 sc->sc_needwakeup |= CRYPTO_SYMQ;
 651                 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
 652                 return (ERESTART);
 653         }
 654         re = sc->sc_front;
 655
 656         staterec = re->re_sa.sa_staterec;       /* save */
 657         /* NB: zero everything but the PE descriptor */
 658         bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
 659         re->re_sa.sa_staterec = staterec;       /* restore */
 660
 661         re->re_crp = crp;
 662         re->re_sesn = SAFE_SESSION(crp->crp_sid);
 663
 664         re->re_src.nsegs = 0;
 665         re->re_dst.nsegs = 0;
 666
 667         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 668                 re->re_src_skb = (struct sk_buff *)crp->crp_buf;
 669                 re->re_dst_skb = (struct sk_buff *)crp->crp_buf;
 670         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 671                 re->re_src_io = (struct uio *)crp->crp_buf;
 672                 re->re_dst_io = (struct uio *)crp->crp_buf;
 673         } else {
 674                 safestats.st_badflags++;
 675                 err = EINVAL;
 676                 goto errout;    /* XXX we don't handle contiguous blocks! */
 677         }
 678
 679         sa = &re->re_sa;
 680         ses = &sc->sc_sessions[re->re_sesn];
 681
 682         crd1 = crp->crp_desc;
 683         if (crd1 == NULL) {
 684                 safestats.st_nodesc++;
 685                 err = EINVAL;
 686                 goto errout;
 687         }
 688         crd2 = crd1->crd_next;
 689
 690         cmd0 = SAFE_SA_CMD0_BASIC;              /* basic group operation */
 691         cmd1 = 0;
 692         if (crd2 == NULL) {
 693                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 694                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 695                     crd1->crd_alg == CRYPTO_NULL_HMAC) {
 696                         maccrd = crd1;
 697                         enccrd = NULL;
 698                         cmd0 |= SAFE_SA_CMD0_OP_HASH;
 699                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 700                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 701                     crd1->crd_alg == CRYPTO_AES_CBC ||
 702                     crd1->crd_alg == CRYPTO_NULL_CBC) {
 703                         maccrd = NULL;
 704                         enccrd = crd1;
 705                         cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
 706                 } else {
 707                         safestats.st_badalg++;
 708                         err = EINVAL;
 709                         goto errout;
 710                 }
 711         } else {
 712                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 713                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 714                     crd1->crd_alg == CRYPTO_NULL_HMAC) &&
 715                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 716                         crd2->crd_alg == CRYPTO_3DES_CBC ||
 717                         crd2->crd_alg == CRYPTO_AES_CBC ||
 718                         crd2->crd_alg == CRYPTO_NULL_CBC) &&
 719                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 720                         maccrd = crd1;
 721                         enccrd = crd2;
 722                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 723                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 724                     crd1->crd_alg == CRYPTO_AES_CBC ||
 725                     crd1->crd_alg == CRYPTO_NULL_CBC) &&
 726                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 727                         crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 728                         crd2->crd_alg == CRYPTO_NULL_HMAC) &&
 729                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 730                         enccrd = crd1;
 731                         maccrd = crd2;
 732                 } else {
 733                         safestats.st_badalg++;
 734                         err = EINVAL;
 735                         goto errout;
 736                 }
 737                 cmd0 |= SAFE_SA_CMD0_OP_BOTH;
 738         }
 739
 740         if (enccrd) {
 741                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
 742                         safe_setup_enckey(ses, enccrd->crd_key);
 743
 744                 if (enccrd->crd_alg == CRYPTO_DES_CBC) {
 745                         cmd0 |= SAFE_SA_CMD0_DES;
 746                         cmd1 |= SAFE_SA_CMD1_CBC;
 747                         ivsize = 2*sizeof(u_int32_t);
 748                 } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
 749                         cmd0 |= SAFE_SA_CMD0_3DES;
 750                         cmd1 |= SAFE_SA_CMD1_CBC;
 751                         ivsize = 2*sizeof(u_int32_t);
 752                 } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
 753                         cmd0 |= SAFE_SA_CMD0_AES;
 754                         cmd1 |= SAFE_SA_CMD1_CBC;
 755                         if (ses->ses_klen == 128)
 756                              cmd1 |=  SAFE_SA_CMD1_AES128;
 757                         else if (ses->ses_klen == 192)
 758                              cmd1 |=  SAFE_SA_CMD1_AES192;
 759                         else
 760                              cmd1 |=  SAFE_SA_CMD1_AES256;
 761                         ivsize = 4*sizeof(u_int32_t);
 762                 } else {
 763                         cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
 764                         ivsize = 0;
 765                 }
 766
 767                 /*
 768                  * Setup encrypt/decrypt state.  When using basic ops
 769                  * we can't use an inline IV because hash/crypt offset
 770                  * must be from the end of the IV to the start of the
 771                  * crypt data and this leaves out the preceding header
 772                  * from the hash calculation.  Instead we place the IV
 773                  * in the state record and set the hash/crypt offset to
 774                  * copy both the header+IV.
 775                  */
 776                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 777                         cmd0 |= SAFE_SA_CMD0_OUTBOUND;
 778
 779                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 780                                 iv = enccrd->crd_iv;
 781                         else
 782                                 iv = (caddr_t) ses->ses_iv;
 783                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 784                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
 785                                     enccrd->crd_inject, ivsize, iv);
 786                         }
 787                         bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
 788                         /* make iv LE */
 789                         for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
 790                                 re->re_sastate.sa_saved_iv[i] =
 791                                         cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
 792                         cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
 793                         re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
 794                 } else {
 795                         cmd0 |= SAFE_SA_CMD0_INBOUND;
 796
 797                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 798                                 bcopy(enccrd->crd_iv,
 799                                         re->re_sastate.sa_saved_iv, ivsize);
 800                         } else {
 801                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 802                                     enccrd->crd_inject, ivsize,
 803                                     (caddr_t)re->re_sastate.sa_saved_iv);
 804                         }
 805                         /* make iv LE */
 806                         for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
 807                                 re->re_sastate.sa_saved_iv[i] =
 808                                         cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
 809                         cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
 810                 }
 811                 /*
 812                  * For basic encryption use the zero pad algorithm.
 813                  * This pads results to an 8-byte boundary and
 814                  * suppresses padding verification for inbound (i.e.
 815                  * decrypt) operations.
 816                  *
 817                  * NB: Not sure if the 8-byte pad boundary is a problem.
 818                  */
 819                 cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
 820
 821                 /* XXX assert key bufs have the same size */
 822                 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
 823         }
 824
 825         if (maccrd) {
 826                 if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 827                         safe_setup_mackey(ses, maccrd->crd_alg,
 828                             maccrd->crd_key, maccrd->crd_klen / 8);
 829                 }
 830
 831                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
 832                         cmd0 |= SAFE_SA_CMD0_MD5;
 833                         cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
 834                 } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
 835                         cmd0 |= SAFE_SA_CMD0_SHA1;
 836                         cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
 837                 } else {
 838                         cmd0 |= SAFE_SA_CMD0_HASH_NULL;
 839                 }
 840                 /*
 841                  * Digest data is loaded from the SA and the hash
 842                  * result is saved to the state block where we
 843                  * retrieve it for return to the caller.
 844                  */
 845                 /* XXX assert digest bufs have the same size */
 846                 bcopy(ses->ses_hminner, sa->sa_indigest,
 847                         sizeof(sa->sa_indigest));
 848                 bcopy(ses->ses_hmouter, sa->sa_outdigest,
 849                         sizeof(sa->sa_outdigest));
 850
 851                 cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
 852                 re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
 853         }
 854
 855         if (enccrd && maccrd) {
 856                 /*
 857                  * The offset from hash data to the start of
 858                  * crypt data is the difference in the skips.
 859                  */
 860                 bypass = maccrd->crd_skip;
 861                 coffset = enccrd->crd_skip - maccrd->crd_skip;
 862                 if (coffset < 0) {
 863                         DPRINTF(("%s: hash does not precede crypt; "
 864                                 "mac skip %u enc skip %u\n",
 865                                 __func__, maccrd->crd_skip, enccrd->crd_skip));
 866                         safestats.st_skipmismatch++;
 867                         err = EINVAL;
 868                         goto errout;
 869                 }
 870                 oplen = enccrd->crd_skip + enccrd->crd_len;
 871                 if (maccrd->crd_skip + maccrd->crd_len != oplen) {
 872                         DPRINTF(("%s: hash amount %u != crypt amount %u\n",
 873                                 __func__, maccrd->crd_skip + maccrd->crd_len,
 874                                 oplen));
 875                         safestats.st_lenmismatch++;
 876                         err = EINVAL;
 877                         goto errout;
 878                 }
 879 #ifdef SAFE_DEBUG
 880                 if (debug) {
 881                         printf("mac: skip %d, len %d, inject %d\n",
 882                             maccrd->crd_skip, maccrd->crd_len,
 883                             maccrd->crd_inject);
 884                         printf("enc: skip %d, len %d, inject %d\n",
 885                             enccrd->crd_skip, enccrd->crd_len,
 886                             enccrd->crd_inject);
 887                         printf("bypass %d coffset %d oplen %d\n",
 888                                 bypass, coffset, oplen);
 889                 }
 890 #endif
 891                 if (coffset & 3) {      /* offset must be 32-bit aligned */
 892                         DPRINTF(("%s: coffset %u misaligned\n",
 893                                 __func__, coffset));
 894                         safestats.st_coffmisaligned++;
 895                         err = EINVAL;
 896                         goto errout;
 897                 }
 898                 coffset >>= 2;
 899                 if (coffset > 255) {    /* offset must be <256 dwords */
 900                         DPRINTF(("%s: coffset %u too big\n",
 901                                 __func__, coffset));
 902                         safestats.st_cofftoobig++;
 903                         err = EINVAL;
 904                         goto errout;
 905                 }
 906                 /*
 907                  * Tell the hardware to copy the header to the output.
 908                  * The header is defined as the data from the end of
 909                  * the bypass to the start of data to be encrypted.
 910                  * Typically this is the inline IV.  Note that you need
 911                  * to do this even if src+dst are the same; it appears
 912                  * that w/o this bit the crypted data is written
 913                  * immediately after the bypass data.
 914                  */
 915                 cmd1 |= SAFE_SA_CMD1_HDRCOPY;
 916                 /*
 917                  * Disable IP header mutable bit handling.  This is
 918                  * needed to get correct HMAC calculations.
 919                  */
 920                 cmd1 |= SAFE_SA_CMD1_MUTABLE;
 921         } else {
 922                 if (enccrd) {
 923                         bypass = enccrd->crd_skip;
 924                         oplen = bypass + enccrd->crd_len;
 925                 } else {
 926                         bypass = maccrd->crd_skip;
 927                         oplen = bypass + maccrd->crd_len;
 928                 }
 929                 coffset = 0;
 930         }
 931         /* XXX verify multiple of 4 when using s/g */
 932         if (bypass > 96) {              /* bypass offset must be <= 96 bytes */
 933                 DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
 934                 safestats.st_bypasstoobig++;
 935                 err = EINVAL;
 936                 goto errout;
 937         }
 938
 939         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 940                 if (pci_map_skb(sc, &re->re_src, re->re_src_skb)) {
 941                         safestats.st_noload++;
 942                         err = ENOMEM;
 943                         goto errout;
 944                 }
 945         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 946                 if (pci_map_uio(sc, &re->re_src, re->re_src_io)) {
 947                         safestats.st_noload++;
 948                         err = ENOMEM;
 949                         goto errout;
 950                 }
 951         }
 952         nicealign = safe_dmamap_aligned(sc, &re->re_src);
 953         uniform = safe_dmamap_uniform(sc, &re->re_src);
 954
 955         DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
 956                 nicealign, uniform, re->re_src.nsegs));
 957         if (re->re_src.nsegs > 1) {
 958                 re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
 959                         ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
 960                 for (i = 0; i < re->re_src_nsegs; i++) {
 961                         /* NB: no need to check if there's space */
 962                         pd = sc->sc_spfree;
 963                         if (++(sc->sc_spfree) == sc->sc_springtop)
 964                                 sc->sc_spfree = sc->sc_spring;
 965
 966                         KASSERT((pd->pd_flags&3) == 0 ||
 967                                 (pd->pd_flags&3) == SAFE_PD_DONE,
 968                                 ("bogus source particle descriptor; flags %x",
 969                                 pd->pd_flags));
 970                         pd->pd_addr = re->re_src_segs[i].ds_addr;
 971                         pd->pd_size = re->re_src_segs[i].ds_len;
 972                         pd->pd_flags = SAFE_PD_READY;
 973                 }
 974                 cmd0 |= SAFE_SA_CMD0_IGATHER;
 975         } else {
 976                 /*
 977                  * No need for gather, reference the operand directly.
 978                  */
 979                 re->re_desc.d_src = re->re_src_segs[0].ds_addr;
 980         }
 981
 982         if (enccrd == NULL && maccrd != NULL) {
 983                 /*
 984                  * Hash op; no destination needed.
 985                  */
 986         } else {
 987                 if (crp->crp_flags & (CRYPTO_F_IOV|CRYPTO_F_SKBUF)) {
 988                         if (!nicealign) {
 989                                 safestats.st_iovmisaligned++;
 990                                 err = EINVAL;
 991                                 goto errout;
 992                         }
 993                         if (uniform != 1) {
 994                                 device_printf(sc->sc_dev, "!uniform source\n");
 995                                 if (!uniform) {
 996                                         /*
 997                                          * There's no way to handle the DMA
 998                                          * requirements with this uio.  We
 999                                          * could create a separate DMA area for
1000                                          * the result and then copy it back,
1001                                          * but for now we just bail and return
1002                                          * an error.  Note that uio requests
1003                                          * > SAFE_MAX_DSIZE are handled because
1004                                          * the DMA map and segment list for the
1005                                          * destination wil result in a
1006                                          * destination particle list that does
1007                                          * the necessary scatter DMA.
1008                                          */
1009                                         safestats.st_iovnotuniform++;
1010                                         err = EINVAL;
1011                                         goto errout;
1012                                 }
1013                         } else
1014                                 re->re_dst = re->re_src;
1015                 } else {
1016                         safestats.st_badflags++;
1017                         err = EINVAL;
1018                         goto errout;
1019                 }
1020
1021                 if (re->re_dst.nsegs > 1) {
1022                         re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
1023                             ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
1024                         for (i = 0; i < re->re_dst_nsegs; i++) {
1025                                 pd = sc->sc_dpfree;
1026                                 KASSERT((pd->pd_flags&3) == 0 ||
1027                                         (pd->pd_flags&3) == SAFE_PD_DONE,
1028                                         ("bogus dest particle descriptor; flags %x",
1029                                                 pd->pd_flags));
1030                                 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
1031                                         sc->sc_dpfree = sc->sc_dpring;
1032                                 pd->pd_addr = re->re_dst_segs[i].ds_addr;
1033                                 pd->pd_flags = SAFE_PD_READY;
1034                         }
1035                         cmd0 |= SAFE_SA_CMD0_OSCATTER;
1036                 } else {
1037                         /*
1038                          * No need for scatter, reference the operand directly.
1039                          */
1040                         re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1041                 }
1042         }
1043
1044         /*
1045          * All done with setup; fillin the SA command words
1046          * and the packet engine descriptor.  The operation
1047          * is now ready for submission to the hardware.
1048          */
1049         sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1050         sa->sa_cmd1 = cmd1
1051                     | (coffset << SAFE_SA_CMD1_OFFSET_S)
1052                     | SAFE_SA_CMD1_SAREV1       /* Rev 1 SA data structure */
1053                     | SAFE_SA_CMD1_SRPCI
1054                     ;
1055         /*
1056          * NB: the order of writes is important here.  In case the
1057          * chip is scanning the ring because of an outstanding request
1058          * it might nab this one too.  In that case we need to make
1059          * sure the setup is complete before we write the length
1060          * field of the descriptor as it signals the descriptor is
1061          * ready for processing.
1062          */
1063         re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1064         if (maccrd)
1065                 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1066         wmb();
1067         re->re_desc.d_len = oplen
1068                           | SAFE_PE_LEN_READY
1069                           | (bypass << SAFE_PE_LEN_BYPASS_S)
1070                           ;
1071
1072         safestats.st_ipackets++;
1073         safestats.st_ibytes += oplen;
1074
1075         if (++(sc->sc_front) == sc->sc_ringtop)
1076                 sc->sc_front = sc->sc_ring;
1077
1078         /* XXX honor batching */
1079         safe_feed(sc, re);
1080         spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1081         return (0);
1082
1083 errout:
1084         if (re->re_src.map != re->re_dst.map)
1085                 pci_unmap_operand(sc, &re->re_dst);
1086         if (re->re_src.map)
1087                 pci_unmap_operand(sc, &re->re_src);
1088         spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1089         if (err != ERESTART) {
1090                 crp->crp_etype = err;
1091                 crypto_done(crp);
1092         } else {
1093                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1094         }
1095         return (err);
1096 }
1097
1098 static void
1099 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1100 {
1101         struct cryptop *crp = (struct cryptop *)re->re_crp;
1102         struct cryptodesc *crd;
1103
1104         DPRINTF(("%s()\n", __FUNCTION__));
1105
1106         safestats.st_opackets++;
1107         safestats.st_obytes += re->re_dst.mapsize;
1108
1109         if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1110                 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1111                         re->re_desc.d_csr,
1112                         re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1113                 safestats.st_peoperr++;
1114                 crp->crp_etype = EIO;           /* something more meaningful? */
1115         }
1116
1117         if (re->re_dst.map != NULL && re->re_dst.map != re->re_src.map)
1118                 pci_unmap_operand(sc, &re->re_dst);
1119         pci_unmap_operand(sc, &re->re_src);
1120
1121         /*
1122          * If result was written to a differet mbuf chain, swap
1123          * it in as the return value and reclaim the original.
1124          */
1125         if ((crp->crp_flags & CRYPTO_F_SKBUF) && re->re_src_skb != re->re_dst_skb) {
1126                 device_printf(sc->sc_dev, "no CRYPTO_F_SKBUF swapping support\n");
1127                 /* kfree_skb(skb) */
1128                 /* crp->crp_buf = (caddr_t)re->re_dst_skb */
1129                 return;
1130         }
1131
1132         if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
1133                 /* copy out IV for future use */
1134                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1135                         int i;
1136                         int ivsize;
1137
1138                         if (crd->crd_alg == CRYPTO_DES_CBC ||
1139                             crd->crd_alg == CRYPTO_3DES_CBC) {
1140                                 ivsize = 2*sizeof(u_int32_t);
1141                         } else if (crd->crd_alg == CRYPTO_AES_CBC) {
1142                                 ivsize = 4*sizeof(u_int32_t);
1143                         } else
1144                                 continue;
1145                         crypto_copydata(crp->crp_flags, crp->crp_buf,
1146                             crd->crd_skip + crd->crd_len - ivsize, ivsize,
1147                             (caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
1148                         for (i = 0;
1149                                         i < ivsize/sizeof(sc->sc_sessions[re->re_sesn].ses_iv[0]);
1150                                         i++)
1151                                 sc->sc_sessions[re->re_sesn].ses_iv[i] =
1152                                         cpu_to_le32(sc->sc_sessions[re->re_sesn].ses_iv[i]);
1153                         break;
1154                 }
1155         }
1156
1157         if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1158                 /* copy out ICV result */
1159                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1160                         if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1161                             crd->crd_alg == CRYPTO_SHA1_HMAC ||
1162                             crd->crd_alg == CRYPTO_NULL_HMAC))
1163                                 continue;
1164                         if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1165                                 /*
1166                                  * SHA-1 ICV's are byte-swapped; fix 'em up
1167                                  * before copy them to their destination.
1168                                  */
1169                                 re->re_sastate.sa_saved_indigest[0] =
1170                                         cpu_to_be32(re->re_sastate.sa_saved_indigest[0]);
1171                                 re->re_sastate.sa_saved_indigest[1] =
1172                                         cpu_to_be32(re->re_sastate.sa_saved_indigest[1]);
1173                                 re->re_sastate.sa_saved_indigest[2] =
1174                                         cpu_to_be32(re->re_sastate.sa_saved_indigest[2]);
1175                         } else {
1176                                 re->re_sastate.sa_saved_indigest[0] =
1177                                         cpu_to_le32(re->re_sastate.sa_saved_indigest[0]);
1178                                 re->re_sastate.sa_saved_indigest[1] =
1179                                         cpu_to_le32(re->re_sastate.sa_saved_indigest[1]);
1180                                 re->re_sastate.sa_saved_indigest[2] =
1181                                         cpu_to_le32(re->re_sastate.sa_saved_indigest[2]);
1182                         }
1183                         crypto_copyback(crp->crp_flags, crp->crp_buf,
1184                             crd->crd_inject,
1185                             sc->sc_sessions[re->re_sesn].ses_mlen,
1186                             (caddr_t)re->re_sastate.sa_saved_indigest);
1187                         break;
1188                 }
1189         }
1190         crypto_done(crp);
1191 }
1192
1193
1194 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
1195 #define SAFE_RNG_MAXWAIT        1000
1196
1197 static void
1198 safe_rng_init(struct safe_softc *sc)
1199 {
1200         u_int32_t w, v;
1201         int i;
1202
1203         DPRINTF(("%s()\n", __FUNCTION__));
1204
1205         WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1206         /* use default value according to the manual */
1207         WRITE_REG(sc, SAFE_RNG_CNFG, 0x834);    /* magic from SafeNet */
1208         WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1209
1210         /*
1211          * There is a bug in rev 1.0 of the 1140 that when the RNG
1212          * is brought out of reset the ready status flag does not
1213          * work until the RNG has finished its internal initialization.
1214          *
1215          * So in order to determine the device is through its
1216          * initialization we must read the data register, using the
1217          * status reg in the read in case it is initialized.  Then read
1218          * the data register until it changes from the first read.
1219          * Once it changes read the data register until it changes
1220          * again.  At this time the RNG is considered initialized.
1221          * This could take between 750ms - 1000ms in time.
1222          */
1223         i = 0;
1224         w = READ_REG(sc, SAFE_RNG_OUT);
1225         do {
1226                 v = READ_REG(sc, SAFE_RNG_OUT);
1227                 if (v != w) {
1228                         w = v;
1229                         break;
1230                 }
1231                 DELAY(10);
1232         } while (++i < SAFE_RNG_MAXWAIT);
1233
1234         /* Wait Until data changes again */
1235         i = 0;
1236         do {
1237                 v = READ_REG(sc, SAFE_RNG_OUT);
1238                 if (v != w)
1239                         break;
1240                 DELAY(10);
1241         } while (++i < SAFE_RNG_MAXWAIT);
1242 }
1243
1244 static __inline void
1245 safe_rng_disable_short_cycle(struct safe_softc *sc)
1246 {
1247         DPRINTF(("%s()\n", __FUNCTION__));
1248
1249         WRITE_REG(sc, SAFE_RNG_CTRL,
1250                 READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
1251 }
1252
1253 static __inline void
1254 safe_rng_enable_short_cycle(struct safe_softc *sc)
1255 {
1256         DPRINTF(("%s()\n", __FUNCTION__));
1257
1258         WRITE_REG(sc, SAFE_RNG_CTRL,
1259                 READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1260 }
1261
1262 static __inline u_int32_t
1263 safe_rng_read(struct safe_softc *sc)
1264 {
1265         int i;
1266
1267         i = 0;
1268         while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1269                 ;
1270         return READ_REG(sc, SAFE_RNG_OUT);
1271 }
1272
1273 static int
1274 safe_read_random(void *arg, u_int32_t *buf, int maxwords)
1275 {
1276         struct safe_softc *sc = (struct safe_softc *) arg;
1277         int i, rc;
1278
1279         DPRINTF(("%s()\n", __FUNCTION__));
1280
1281         safestats.st_rng++;
1282         /*
1283          * Fetch the next block of data.
1284          */
1285         if (maxwords > safe_rngbufsize)
1286                 maxwords = safe_rngbufsize;
1287         if (maxwords > SAFE_RNG_MAXBUFSIZ)
1288                 maxwords = SAFE_RNG_MAXBUFSIZ;
1289 retry:
1290         /* read as much as we can */
1291         for (rc = 0; rc < maxwords; rc++) {
1292                 if (READ_REG(sc, SAFE_RNG_STAT) != 0)
1293                         break;
1294                 buf[rc] = READ_REG(sc, SAFE_RNG_OUT);
1295         }
1296         if (rc == 0)
1297                 return 0;
1298         /*
1299          * Check the comparator alarm count and reset the h/w if
1300          * it exceeds our threshold.  This guards against the
1301          * hardware oscillators resonating with external signals.
1302          */
1303         if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1304                 u_int32_t freq_inc, w;
1305
1306                 DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1307                         (unsigned)READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1308                 safestats.st_rngalarm++;
1309                 safe_rng_enable_short_cycle(sc);
1310                 freq_inc = 18;
1311                 for (i = 0; i < 64; i++) {
1312                         w = READ_REG(sc, SAFE_RNG_CNFG);
1313                         freq_inc = ((w + freq_inc) & 0x3fL);
1314                         w = ((w & ~0x3fL) | freq_inc);
1315                         WRITE_REG(sc, SAFE_RNG_CNFG, w);
1316
1317                         WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1318
1319                         (void) safe_rng_read(sc);
1320                         DELAY(25);
1321
1322                         if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1323                                 safe_rng_disable_short_cycle(sc);
1324                                 goto retry;
1325                         }
1326                         freq_inc = 1;
1327                 }
1328                 safe_rng_disable_short_cycle(sc);
1329         } else
1330                 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1331
1332         return(rc);
1333 }
1334 #endif /* defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG) */
1335
1336
1337 /*
1338  * Resets the board.  Values in the regesters are left as is
1339  * from the reset (i.e. initial values are assigned elsewhere).
1340  */
1341 static void
1342 safe_reset_board(struct safe_softc *sc)
1343 {
1344         u_int32_t v;
1345         /*
1346          * Reset the device.  The manual says no delay
1347          * is needed between marking and clearing reset.
1348          */
1349         DPRINTF(("%s()\n", __FUNCTION__));
1350
1351         v = READ_REG(sc, SAFE_PE_DMACFG) &~
1352                 (SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1353                  SAFE_PE_DMACFG_SGRESET);
1354         WRITE_REG(sc, SAFE_PE_DMACFG, v
1355                                     | SAFE_PE_DMACFG_PERESET
1356                                     | SAFE_PE_DMACFG_PDRRESET
1357                                     | SAFE_PE_DMACFG_SGRESET);
1358         WRITE_REG(sc, SAFE_PE_DMACFG, v);
1359 }
1360
1361 /*
1362  * Initialize registers we need to touch only once.
1363  */
1364 static void
1365 safe_init_board(struct safe_softc *sc)
1366 {
1367         u_int32_t v, dwords;
1368
1369         DPRINTF(("%s()\n", __FUNCTION__));
1370
1371         v = READ_REG(sc, SAFE_PE_DMACFG);
1372         v &=~ (   SAFE_PE_DMACFG_PEMODE
1373                         | SAFE_PE_DMACFG_FSENA          /* failsafe enable */
1374                         | SAFE_PE_DMACFG_GPRPCI         /* gather ring on PCI */
1375                         | SAFE_PE_DMACFG_SPRPCI         /* scatter ring on PCI */
1376                         | SAFE_PE_DMACFG_ESDESC         /* endian-swap descriptors */
1377                         | SAFE_PE_DMACFG_ESPDESC        /* endian-swap part. desc's */
1378                         | SAFE_PE_DMACFG_ESSA           /* endian-swap SA's */
1379                         | SAFE_PE_DMACFG_ESPACKET       /* swap the packet data */
1380                   );
1381         v |= SAFE_PE_DMACFG_FSENA               /* failsafe enable */
1382           |  SAFE_PE_DMACFG_GPRPCI              /* gather ring on PCI */
1383           |  SAFE_PE_DMACFG_SPRPCI              /* scatter ring on PCI */
1384           |  SAFE_PE_DMACFG_ESDESC              /* endian-swap descriptors */
1385           |  SAFE_PE_DMACFG_ESPDESC             /* endian-swap part. desc's */
1386           |  SAFE_PE_DMACFG_ESSA                /* endian-swap SA's */
1387 #if 0
1388           |  SAFE_PE_DMACFG_ESPACKET    /* swap the packet data */
1389 #endif
1390           ;
1391         WRITE_REG(sc, SAFE_PE_DMACFG, v);
1392
1393 #ifdef __BIG_ENDIAN
1394         /* tell the safenet that we are 4321 and not 1234 */
1395         WRITE_REG(sc, SAFE_ENDIAN, 0xe4e41b1b);
1396 #endif
1397
1398         if (sc->sc_chiprev == SAFE_REV(1,0)) {
1399                 /*
1400                  * Avoid large PCI DMA transfers.  Rev 1.0 has a bug where
1401                  * "target mode transfers" done while the chip is DMA'ing
1402                  * >1020 bytes cause the hardware to lockup.  To avoid this
1403                  * we reduce the max PCI transfer size and use small source
1404                  * particle descriptors (<= 256 bytes).
1405                  */
1406                 WRITE_REG(sc, SAFE_DMA_CFG, 256);
1407                 device_printf(sc->sc_dev,
1408                         "Reduce max DMA size to %u words for rev %u.%u WAR\n",
1409                         (unsigned) ((READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff),
1410                         (unsigned) SAFE_REV_MAJ(sc->sc_chiprev),
1411                         (unsigned) SAFE_REV_MIN(sc->sc_chiprev));
1412                 sc->sc_max_dsize = 256;
1413         } else {
1414                 sc->sc_max_dsize = SAFE_MAX_DSIZE;
1415         }
1416
1417         /* NB: operands+results are overlaid */
1418         WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1419         WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1420         /*
1421          * Configure ring entry size and number of items in the ring.
1422          */
1423         KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1424                 ("PE ring entry not 32-bit aligned!"));
1425         dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1426         WRITE_REG(sc, SAFE_PE_RINGCFG,
1427                 (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1428         WRITE_REG(sc, SAFE_PE_RINGPOLL, 0);     /* disable polling */
1429
1430         WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1431         WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1432         WRITE_REG(sc, SAFE_PE_PARTSIZE,
1433                 (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1434         /*
1435          * NB: destination particles are fixed size.  We use
1436          *     an mbuf cluster and require all results go to
1437          *     clusters or smaller.
1438          */
1439         WRITE_REG(sc, SAFE_PE_PARTCFG, sc->sc_max_dsize);
1440
1441         /* it's now safe to enable PE mode, do it */
1442         WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1443
1444         /*
1445          * Configure hardware to use level-triggered interrupts and
1446          * to interrupt after each descriptor is processed.
1447          */
1448         WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1449         WRITE_REG(sc, SAFE_HI_CLR, 0xffffffff);
1450         WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1451         WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1452 }
1453
1454
1455 /*
1456  * Clean up after a chip crash.
1457  * It is assumed that the caller in splimp()
1458  */
1459 static void
1460 safe_cleanchip(struct safe_softc *sc)
1461 {
1462         DPRINTF(("%s()\n", __FUNCTION__));
1463
1464         if (sc->sc_nqchip != 0) {
1465                 struct safe_ringentry *re = sc->sc_back;
1466
1467                 while (re != sc->sc_front) {
1468                         if (re->re_desc.d_csr != 0)
1469                                 safe_free_entry(sc, re);
1470                         if (++re == sc->sc_ringtop)
1471                                 re = sc->sc_ring;
1472                 }
1473                 sc->sc_back = re;
1474                 sc->sc_nqchip = 0;
1475         }
1476 }
1477
1478 /*
1479  * free a safe_q
1480  * It is assumed that the caller is within splimp().
1481  */
1482 static int
1483 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1484 {
1485         struct cryptop *crp;
1486
1487         DPRINTF(("%s()\n", __FUNCTION__));
1488
1489         /*
1490          * Free header MCR
1491          */
1492         if ((re->re_dst_skb != NULL) && (re->re_src_skb != re->re_dst_skb))
1493 #ifdef NOTYET
1494                 m_freem(re->re_dst_m);
1495 #else
1496                 printk("%s,%d: SKB not supported\n", __FILE__, __LINE__);
1497 #endif
1498
1499         crp = (struct cryptop *)re->re_crp;
1500
1501         re->re_desc.d_csr = 0;
1502
1503         crp->crp_etype = EFAULT;
1504         crypto_done(crp);
1505         return(0);
1506 }
1507
1508 /*
1509  * Routine to reset the chip and clean up.
1510  * It is assumed that the caller is in splimp()
1511  */
1512 static void
1513 safe_totalreset(struct safe_softc *sc)
1514 {
1515         DPRINTF(("%s()\n", __FUNCTION__));
1516
1517         safe_reset_board(sc);
1518         safe_init_board(sc);
1519         safe_cleanchip(sc);
1520 }
1521
1522 /*
1523  * Is the operand suitable aligned for direct DMA.  Each
1524  * segment must be aligned on a 32-bit boundary and all
1525  * but the last segment must be a multiple of 4 bytes.
1526  */
1527 static int
1528 safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op)
1529 {
1530         int i;
1531
1532         DPRINTF(("%s()\n", __FUNCTION__));
1533
1534         for (i = 0; i < op->nsegs; i++) {
1535                 if (op->segs[i].ds_addr & 3)
1536                         return (0);
1537                 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1538                         return (0);
1539         }
1540         return (1);
1541 }
1542
1543 /*
1544  * Is the operand suitable for direct DMA as the destination
1545  * of an operation.  The hardware requires that each ``particle''
1546  * but the last in an operation result have the same size.  We
1547  * fix that size at SAFE_MAX_DSIZE bytes.  This routine returns
1548  * 0 if some segment is not a multiple of of this size, 1 if all
1549  * segments are exactly this size, or 2 if segments are at worst
1550  * a multple of this size.
1551  */
1552 static int
1553 safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op)
1554 {
1555         int result = 1;
1556
1557         DPRINTF(("%s()\n", __FUNCTION__));
1558
1559         if (op->nsegs > 0) {
1560                 int i;
1561
1562                 for (i = 0; i < op->nsegs-1; i++) {
1563                         if (op->segs[i].ds_len % sc->sc_max_dsize)
1564                                 return (0);
1565                         if (op->segs[i].ds_len != sc->sc_max_dsize)
1566                                 result = 2;
1567                 }
1568         }
1569         return (result);
1570 }
1571
1572 static int
1573 safe_kprocess(device_t dev, struct cryptkop *krp, int hint)
1574 {
1575         struct safe_softc *sc = device_get_softc(dev);
1576         struct safe_pkq *q;
1577         unsigned long flags;
1578
1579         DPRINTF(("%s()\n", __FUNCTION__));
1580
1581         if (sc == NULL) {
1582                 krp->krp_status = EINVAL;
1583                 goto err;
1584         }
1585
1586         if (krp->krp_op != CRK_MOD_EXP) {
1587                 krp->krp_status = EOPNOTSUPP;
1588                 goto err;
1589         }
1590
1591         q = (struct safe_pkq *) kmalloc(sizeof(*q), GFP_KERNEL);
1592         if (q == NULL) {
1593                 krp->krp_status = ENOMEM;
1594                 goto err;
1595         }
1596         memset(q, 0, sizeof(*q));
1597         q->pkq_krp = krp;
1598         INIT_LIST_HEAD(&q->pkq_list);
1599
1600         spin_lock_irqsave(&sc->sc_pkmtx, flags);
1601         list_add_tail(&q->pkq_list, &sc->sc_pkq);
1602         safe_kfeed(sc);
1603         spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1604         return (0);
1605
1606 err:
1607         crypto_kdone(krp);
1608         return (0);
1609 }
1610
1611 #define SAFE_CRK_PARAM_BASE     0
1612 #define SAFE_CRK_PARAM_EXP      1
1613 #define SAFE_CRK_PARAM_MOD      2
1614
1615 static int
1616 safe_kstart(struct safe_softc *sc)
1617 {
1618         struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
1619         int exp_bits, mod_bits, base_bits;
1620         u_int32_t op, a_off, b_off, c_off, d_off;
1621
1622         DPRINTF(("%s()\n", __FUNCTION__));
1623
1624         if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
1625                 krp->krp_status = EINVAL;
1626                 return (1);
1627         }
1628
1629         base_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1630         if (base_bits > 2048)
1631                 goto too_big;
1632         if (base_bits <= 0)             /* 5. base not zero */
1633                 goto too_small;
1634
1635         exp_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1636         if (exp_bits > 2048)
1637                 goto too_big;
1638         if (exp_bits <= 0)              /* 1. exponent word length > 0 */
1639                 goto too_small;         /* 4. exponent not zero */
1640
1641         mod_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1642         if (mod_bits > 2048)
1643                 goto too_big;
1644         if (mod_bits <= 32)             /* 2. modulus word length > 1 */
1645                 goto too_small;         /* 8. MSW of modulus != zero */
1646         if (mod_bits < exp_bits)        /* 3 modulus len >= exponent len */
1647                 goto too_small;
1648         if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
1649                 goto bad_domain;        /* 6. modulus is odd */
1650         if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
1651                 goto too_small;         /* make sure result will fit */
1652
1653         /* 7. modulus > base */
1654         if (mod_bits < base_bits)
1655                 goto too_small;
1656         if (mod_bits == base_bits) {
1657                 u_int8_t *basep, *modp;
1658                 int i;
1659
1660                 basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
1661                     ((base_bits + 7) / 8) - 1;
1662                 modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
1663                     ((mod_bits + 7) / 8) - 1;
1664
1665                 for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
1666                         if (*modp < *basep)
1667                                 goto too_small;
1668                         if (*modp > *basep)
1669                                 break;
1670                 }
1671         }
1672
1673         /* And on the 9th step, he rested. */
1674
1675         WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
1676         WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
1677         if (mod_bits > 1024) {
1678                 op = SAFE_PK_FUNC_EXP4;
1679                 a_off = 0x000;
1680                 b_off = 0x100;
1681                 c_off = 0x200;
1682                 d_off = 0x300;
1683         } else {
1684                 op = SAFE_PK_FUNC_EXP16;
1685                 a_off = 0x000;
1686                 b_off = 0x080;
1687                 c_off = 0x100;
1688                 d_off = 0x180;
1689         }
1690         sc->sc_pk_reslen = b_off - a_off;
1691         sc->sc_pk_resoff = d_off;
1692
1693         /* A is exponent, B is modulus, C is base, D is result */
1694         safe_kload_reg(sc, a_off, b_off - a_off,
1695             &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1696         WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
1697         safe_kload_reg(sc, b_off, b_off - a_off,
1698             &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1699         WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
1700         safe_kload_reg(sc, c_off, b_off - a_off,
1701             &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1702         WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
1703         WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
1704
1705         WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
1706
1707         return (0);
1708
1709 too_big:
1710         krp->krp_status = E2BIG;
1711         return (1);
1712 too_small:
1713         krp->krp_status = ERANGE;
1714         return (1);
1715 bad_domain:
1716         krp->krp_status = EDOM;
1717         return (1);
1718 }
1719
1720 static int
1721 safe_ksigbits(struct safe_softc *sc, struct crparam *cr)
1722 {
1723         u_int plen = (cr->crp_nbits + 7) / 8;
1724         int i, sig = plen * 8;
1725         u_int8_t c, *p = cr->crp_p;
1726
1727         DPRINTF(("%s()\n", __FUNCTION__));
1728
1729         for (i = plen - 1; i >= 0; i--) {
1730                 c = p[i];
1731                 if (c != 0) {
1732                         while ((c & 0x80) == 0) {
1733                                 sig--;
1734                                 c <<= 1;
1735                         }
1736                         break;
1737                 }
1738                 sig -= 8;
1739         }
1740         return (sig);
1741 }
1742
1743 static void
1744 safe_kfeed(struct safe_softc *sc)
1745 {
1746         struct safe_pkq *q, *tmp;
1747
1748         DPRINTF(("%s()\n", __FUNCTION__));
1749
1750         if (list_empty(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
1751                 return;
1752         if (sc->sc_pkq_cur != NULL)
1753                 return;
1754         list_for_each_entry_safe(q, tmp, &sc->sc_pkq, pkq_list) {
1755                 sc->sc_pkq_cur = q;
1756                 list_del(&q->pkq_list);
1757                 if (safe_kstart(sc) != 0) {
1758                         crypto_kdone(q->pkq_krp);
1759                         kfree(q);
1760                         sc->sc_pkq_cur = NULL;
1761                 } else {
1762                         /* op started, start polling */
1763                         mod_timer(&sc->sc_pkto, jiffies + 1);
1764                         break;
1765                 }
1766         }
1767 }
1768
1769 static void
1770 safe_kpoll(unsigned long arg)
1771 {
1772         struct safe_softc *sc = NULL;
1773         struct safe_pkq *q;
1774         struct crparam *res;
1775         int i;
1776         u_int32_t buf[64];
1777         unsigned long flags;
1778
1779         DPRINTF(("%s()\n", __FUNCTION__));
1780
1781         if (arg >= SAFE_MAX_CHIPS)
1782                 return;
1783         sc = safe_chip_idx[arg];
1784         if (!sc) {
1785                 DPRINTF(("%s() - bad callback\n", __FUNCTION__));
1786                 return;
1787         }
1788
1789         spin_lock_irqsave(&sc->sc_pkmtx, flags);
1790         if (sc->sc_pkq_cur == NULL)
1791                 goto out;
1792         if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
1793                 /* still running, check back later */
1794                 mod_timer(&sc->sc_pkto, jiffies + 1);
1795                 goto out;
1796         }
1797
1798         q = sc->sc_pkq_cur;
1799         res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
1800         bzero(buf, sizeof(buf));
1801         bzero(res->crp_p, (res->crp_nbits + 7) / 8);
1802         for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
1803                 buf[i] = le32_to_cpu(READ_REG(sc, SAFE_PK_RAM_START +
1804                     sc->sc_pk_resoff + (i << 2)));
1805         bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
1806         /*
1807          * reduce the bits that need copying if possible
1808          */
1809         res->crp_nbits = min(res->crp_nbits,sc->sc_pk_reslen * 8);
1810         res->crp_nbits = safe_ksigbits(sc, res);
1811
1812         for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
1813                 WRITE_REG(sc, i, 0);
1814
1815         crypto_kdone(q->pkq_krp);
1816         kfree(q);
1817         sc->sc_pkq_cur = NULL;
1818
1819         safe_kfeed(sc);
1820 out:
1821         spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1822 }
1823
1824 static void
1825 safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
1826     struct crparam *n)
1827 {
1828         u_int32_t buf[64], i;
1829
1830         DPRINTF(("%s()\n", __FUNCTION__));
1831
1832         bzero(buf, sizeof(buf));
1833         bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
1834
1835         for (i = 0; i < len >> 2; i++)
1836                 WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
1837                     cpu_to_le32(buf[i]));
1838 }
1839
1840 #ifdef SAFE_DEBUG
1841 static void
1842 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1843 {
1844         printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
1845                 , tag
1846                 , READ_REG(sc, SAFE_DMA_ENDIAN)
1847                 , READ_REG(sc, SAFE_DMA_SRCADDR)
1848                 , READ_REG(sc, SAFE_DMA_DSTADDR)
1849                 , READ_REG(sc, SAFE_DMA_STAT)
1850         );
1851 }
1852
1853 static void
1854 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1855 {
1856         printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
1857                 , tag
1858                 , READ_REG(sc, SAFE_HI_CFG)
1859                 , READ_REG(sc, SAFE_HI_MASK)
1860                 , READ_REG(sc, SAFE_HI_DESC_CNT)
1861                 , READ_REG(sc, SAFE_HU_STAT)
1862                 , READ_REG(sc, SAFE_HM_STAT)
1863         );
1864 }
1865
1866 static void
1867 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1868 {
1869         u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1870
1871         /* NB: assume caller has lock on ring */
1872         printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
1873                 tag,
1874                 estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1875                 (unsigned long)(sc->sc_back - sc->sc_ring),
1876                 (unsigned long)(sc->sc_front - sc->sc_ring));
1877 }
1878
1879 static void
1880 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1881 {
1882         int ix, nsegs;
1883
1884         ix = re - sc->sc_ring;
1885         printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
1886                 , tag
1887                 , re, ix
1888                 , re->re_desc.d_csr
1889                 , re->re_desc.d_src
1890                 , re->re_desc.d_dst
1891                 , re->re_desc.d_sa
1892                 , re->re_desc.d_len
1893         );
1894         if (re->re_src.nsegs > 1) {
1895                 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1896                         sizeof(struct safe_pdesc);
1897                 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1898                         printf(" spd[%u] %p: %p size %u flags %x"
1899                                 , ix, &sc->sc_spring[ix]
1900                                 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1901                                 , sc->sc_spring[ix].pd_size
1902                                 , sc->sc_spring[ix].pd_flags
1903                         );
1904                         if (sc->sc_spring[ix].pd_size == 0)
1905                                 printf(" (zero!)");
1906                         printf("\n");
1907                         if (++ix == SAFE_TOTAL_SPART)
1908                                 ix = 0;
1909                 }
1910         }
1911         if (re->re_dst.nsegs > 1) {
1912                 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1913                         sizeof(struct safe_pdesc);
1914                 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1915                         printf(" dpd[%u] %p: %p flags %x\n"
1916                                 , ix, &sc->sc_dpring[ix]
1917                                 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1918                                 , sc->sc_dpring[ix].pd_flags
1919                         );
1920                         if (++ix == SAFE_TOTAL_DPART)
1921                                 ix = 0;
1922                 }
1923         }
1924         printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1925                 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1926         printf("sa: key %x %x %x %x %x %x %x %x\n"
1927                 , re->re_sa.sa_key[0]
1928                 , re->re_sa.sa_key[1]
1929                 , re->re_sa.sa_key[2]
1930                 , re->re_sa.sa_key[3]
1931                 , re->re_sa.sa_key[4]
1932                 , re->re_sa.sa_key[5]
1933                 , re->re_sa.sa_key[6]
1934                 , re->re_sa.sa_key[7]
1935         );
1936         printf("sa: indigest %x %x %x %x %x\n"
1937                 , re->re_sa.sa_indigest[0]
1938                 , re->re_sa.sa_indigest[1]
1939                 , re->re_sa.sa_indigest[2]
1940                 , re->re_sa.sa_indigest[3]
1941                 , re->re_sa.sa_indigest[4]
1942         );
1943         printf("sa: outdigest %x %x %x %x %x\n"
1944                 , re->re_sa.sa_outdigest[0]
1945                 , re->re_sa.sa_outdigest[1]
1946                 , re->re_sa.sa_outdigest[2]
1947                 , re->re_sa.sa_outdigest[3]
1948                 , re->re_sa.sa_outdigest[4]
1949         );
1950         printf("sr: iv %x %x %x %x\n"
1951                 , re->re_sastate.sa_saved_iv[0]
1952                 , re->re_sastate.sa_saved_iv[1]
1953                 , re->re_sastate.sa_saved_iv[2]
1954                 , re->re_sastate.sa_saved_iv[3]
1955         );
1956         printf("sr: hashbc %u indigest %x %x %x %x %x\n"
1957                 , re->re_sastate.sa_saved_hashbc
1958                 , re->re_sastate.sa_saved_indigest[0]
1959                 , re->re_sastate.sa_saved_indigest[1]
1960                 , re->re_sastate.sa_saved_indigest[2]
1961                 , re->re_sastate.sa_saved_indigest[3]
1962                 , re->re_sastate.sa_saved_indigest[4]
1963         );
1964 }
1965
1966 static void
1967 safe_dump_ring(struct safe_softc *sc, const char *tag)
1968 {
1969         unsigned long flags;
1970
1971         spin_lock_irqsave(&sc->sc_ringmtx, flags);
1972         printf("\nSafeNet Ring State:\n");
1973         safe_dump_intrstate(sc, tag);
1974         safe_dump_dmastatus(sc, tag);
1975         safe_dump_ringstate(sc, tag);
1976         if (sc->sc_nqchip) {
1977                 struct safe_ringentry *re = sc->sc_back;
1978                 do {
1979                         safe_dump_request(sc, tag, re);
1980                         if (++re == sc->sc_ringtop)
1981                                 re = sc->sc_ring;
1982                 } while (re != sc->sc_front);
1983         }
1984         spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1985 }
1986 #endif /* SAFE_DEBUG */
1987
1988
1989 static int safe_probe(struct pci_dev *dev, const struct pci_device_id *ent)
1990 {
1991         struct safe_softc *sc = NULL;
1992         u32 mem_start, mem_len, cmd;
1993         int i, rc, devinfo;
1994         dma_addr_t raddr;
1995         static int num_chips = 0;
1996
1997         DPRINTF(("%s()\n", __FUNCTION__));
1998
1999         if (pci_enable_device(dev) < 0)
2000                 return(-ENODEV);
2001
2002         if (!dev->irq) {
2003                 printk("safe: found device with no IRQ assigned. check BIOS settings!");
2004                 pci_disable_device(dev);
2005                 return(-ENODEV);
2006         }
2007
2008         if (pci_set_mwi(dev)) {
2009                 printk("safe: pci_set_mwi failed!");
2010                 return(-ENODEV);
2011         }
2012
2013         sc = (struct safe_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
2014         if (!sc)
2015                 return(-ENOMEM);
2016         memset(sc, 0, sizeof(*sc));
2017
2018         softc_device_init(sc, "safe", num_chips, safe_methods);
2019
2020         sc->sc_irq = -1;
2021         sc->sc_cid = -1;
2022         sc->sc_pcidev = dev;
2023         if (num_chips < SAFE_MAX_CHIPS) {
2024                 safe_chip_idx[device_get_unit(sc->sc_dev)] = sc;
2025                 num_chips++;
2026         }
2027
2028         INIT_LIST_HEAD(&sc->sc_pkq);
2029         spin_lock_init(&sc->sc_pkmtx);
2030
2031         pci_set_drvdata(sc->sc_pcidev, sc);
2032
2033         /* we read its hardware registers as memory */
2034         mem_start = pci_resource_start(sc->sc_pcidev, 0);
2035         mem_len   = pci_resource_len(sc->sc_pcidev, 0);
2036
2037         sc->sc_base_addr = (ocf_iomem_t) ioremap(mem_start, mem_len);
2038         if (!sc->sc_base_addr) {
2039                 device_printf(sc->sc_dev, "failed to ioremap 0x%x-0x%x\n",
2040                                 mem_start, mem_start + mem_len - 1);
2041                 goto out;
2042         }
2043
2044         /* fix up the bus size */
2045         if (pci_set_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
2046                 device_printf(sc->sc_dev, "No usable DMA configuration, aborting.\n");
2047                 goto out;
2048         }
2049         if (pci_set_consistent_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
2050                 device_printf(sc->sc_dev, "No usable consistent DMA configuration, aborting.\n");
2051                 goto out;
2052         }
2053
2054         pci_set_master(sc->sc_pcidev);
2055
2056         pci_read_config_dword(sc->sc_pcidev, PCI_COMMAND, &cmd);
2057
2058         if (!(cmd & PCI_COMMAND_MEMORY)) {
2059                 device_printf(sc->sc_dev, "failed to enable memory mapping\n");
2060                 goto out;
2061         }
2062
2063         if (!(cmd & PCI_COMMAND_MASTER)) {
2064                 device_printf(sc->sc_dev, "failed to enable bus mastering\n");
2065                 goto out;
2066         }
2067
2068         rc = request_irq(dev->irq, safe_intr, IRQF_SHARED, "safe", sc);
2069         if (rc) {
2070                 device_printf(sc->sc_dev, "failed to hook irq %d\n", sc->sc_irq);
2071                 goto out;
2072         }
2073         sc->sc_irq = dev->irq;
2074
2075         sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
2076                         (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
2077
2078         /*
2079          * Allocate packet engine descriptors.
2080          */
2081         sc->sc_ringalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2082                         SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2083                         &sc->sc_ringalloc.dma_paddr);
2084         if (!sc->sc_ringalloc.dma_vaddr) {
2085                 device_printf(sc->sc_dev, "cannot allocate PE descriptor ring\n");
2086                 goto out;
2087         }
2088
2089         /*
2090          * Hookup the static portion of all our data structures.
2091          */
2092         sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
2093         sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
2094         sc->sc_front = sc->sc_ring;
2095         sc->sc_back = sc->sc_ring;
2096         raddr = sc->sc_ringalloc.dma_paddr;
2097         bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
2098         for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
2099                 struct safe_ringentry *re = &sc->sc_ring[i];
2100
2101                 re->re_desc.d_sa = raddr +
2102                         offsetof(struct safe_ringentry, re_sa);
2103                 re->re_sa.sa_staterec = raddr +
2104                         offsetof(struct safe_ringentry, re_sastate);
2105
2106                 raddr += sizeof (struct safe_ringentry);
2107         }
2108         spin_lock_init(&sc->sc_ringmtx);
2109
2110         /*
2111          * Allocate scatter and gather particle descriptors.
2112          */
2113         sc->sc_spalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2114                         SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
2115                         &sc->sc_spalloc.dma_paddr);
2116         if (!sc->sc_spalloc.dma_vaddr) {
2117                 device_printf(sc->sc_dev, "cannot allocate source particle descriptor ring\n");
2118                 goto out;
2119         }
2120         sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
2121         sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
2122         sc->sc_spfree = sc->sc_spring;
2123         bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
2124
2125         sc->sc_dpalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2126                         SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2127                         &sc->sc_dpalloc.dma_paddr);
2128         if (!sc->sc_dpalloc.dma_vaddr) {
2129                 device_printf(sc->sc_dev, "cannot allocate destination particle descriptor ring\n");
2130                 goto out;
2131         }
2132         sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
2133         sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
2134         sc->sc_dpfree = sc->sc_dpring;
2135         bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
2136
2137         sc->sc_cid = crypto_get_driverid(softc_get_device(sc), CRYPTOCAP_F_HARDWARE);
2138         if (sc->sc_cid < 0) {
2139                 device_printf(sc->sc_dev, "could not get crypto driver id\n");
2140                 goto out;
2141         }
2142
2143         printf("%s:", device_get_nameunit(sc->sc_dev));
2144
2145         devinfo = READ_REG(sc, SAFE_DEVINFO);
2146         if (devinfo & SAFE_DEVINFO_RNG) {
2147                 sc->sc_flags |= SAFE_FLAGS_RNG;
2148                 printf(" rng");
2149         }
2150         if (devinfo & SAFE_DEVINFO_PKEY) {
2151                 printf(" key");
2152                 sc->sc_flags |= SAFE_FLAGS_KEY;
2153                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
2154 #if 0
2155                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
2156 #endif
2157                 init_timer(&sc->sc_pkto);
2158                 sc->sc_pkto.function = safe_kpoll;
2159                 sc->sc_pkto.data = (unsigned long) device_get_unit(sc->sc_dev);
2160         }
2161         if (devinfo & SAFE_DEVINFO_DES) {
2162                 printf(" des/3des");
2163                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
2164                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
2165         }
2166         if (devinfo & SAFE_DEVINFO_AES) {
2167                 printf(" aes");
2168                 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
2169         }
2170         if (devinfo & SAFE_DEVINFO_MD5) {
2171                 printf(" md5");
2172                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
2173         }
2174         if (devinfo & SAFE_DEVINFO_SHA1) {
2175                 printf(" sha1");
2176                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
2177         }
2178         printf(" null");
2179         crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
2180         crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
2181         /* XXX other supported algorithms */
2182         printf("\n");
2183
2184         safe_reset_board(sc);           /* reset h/w */
2185         safe_init_board(sc);            /* init h/w */
2186
2187 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
2188         if (sc->sc_flags & SAFE_FLAGS_RNG) {
2189                 safe_rng_init(sc);
2190                 crypto_rregister(sc->sc_cid, safe_read_random, sc);
2191         }
2192 #endif /* SAFE_NO_RNG */
2193
2194         return (0);
2195
2196 out:
2197         if (sc->sc_cid >= 0)
2198                 crypto_unregister_all(sc->sc_cid);
2199         if (sc->sc_irq != -1)
2200                 free_irq(sc->sc_irq, sc);
2201         if (sc->sc_ringalloc.dma_vaddr)
2202                 pci_free_consistent(sc->sc_pcidev,
2203                                 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2204                                 sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2205         if (sc->sc_spalloc.dma_vaddr)
2206                 pci_free_consistent(sc->sc_pcidev,
2207                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2208                                 sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2209         if (sc->sc_dpalloc.dma_vaddr)
2210                 pci_free_consistent(sc->sc_pcidev,
2211                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2212                                 sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2213         kfree(sc);
2214         return(-ENODEV);
2215 }
2216
2217 static void safe_remove(struct pci_dev *dev)
2218 {
2219         struct safe_softc *sc = pci_get_drvdata(dev);
2220
2221         DPRINTF(("%s()\n", __FUNCTION__));
2222
2223         /* XXX wait/abort active ops */
2224
2225         WRITE_REG(sc, SAFE_HI_MASK, 0);         /* disable interrupts */
2226
2227         del_timer_sync(&sc->sc_pkto);
2228
2229         crypto_unregister_all(sc->sc_cid);
2230
2231         safe_cleanchip(sc);
2232
2233         if (sc->sc_irq != -1)
2234                 free_irq(sc->sc_irq, sc);
2235         if (sc->sc_ringalloc.dma_vaddr)
2236                 pci_free_consistent(sc->sc_pcidev,
2237                                 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2238                                 sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2239         if (sc->sc_spalloc.dma_vaddr)
2240                 pci_free_consistent(sc->sc_pcidev,
2241                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2242                                 sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2243         if (sc->sc_dpalloc.dma_vaddr)
2244                 pci_free_consistent(sc->sc_pcidev,
2245                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2246                                 sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2247         sc->sc_irq = -1;
2248         sc->sc_ringalloc.dma_vaddr = NULL;
2249         sc->sc_spalloc.dma_vaddr = NULL;
2250         sc->sc_dpalloc.dma_vaddr = NULL;
2251 }
2252
2253 static struct pci_device_id safe_pci_tbl[] = {
2254         { PCI_VENDOR_SAFENET, PCI_PRODUCT_SAFEXCEL,
2255           PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
2256         { },
2257 };
2258 MODULE_DEVICE_TABLE(pci, safe_pci_tbl);
2259
2260 static struct pci_driver safe_driver = {
2261         .name         = "safe",
2262         .id_table     = safe_pci_tbl,
2263         .probe        = safe_probe,
2264         .remove       = safe_remove,
2265         /* add PM stuff here one day */
2266 };
2267
2268 static int __init safe_init (void)
2269 {
2270         struct safe_softc *sc = NULL;
2271         int rc;
2272
2273         DPRINTF(("%s(%p)\n", __FUNCTION__, safe_init));
2274
2275         rc = pci_register_driver(&safe_driver);
2276         pci_register_driver_compat(&safe_driver, rc);
2277
2278         return rc;
2279 }
2280
2281 static void __exit safe_exit (void)
2282 {
2283         pci_unregister_driver(&safe_driver);
2284 }
2285
2286 module_init(safe_init);
2287 module_exit(safe_exit);
2288
2289 MODULE_LICENSE("BSD");
2290 MODULE_AUTHOR("David McCullough <david_mccullough@mcafee.com>");
2291 MODULE_DESCRIPTION("OCF driver for safenet PCI crypto devices");