target/linux/generic/files/crypto/ocf/safe/safe.c

   1 /*-
   2  * Linux port done by David McCullough <david_mccullough@mcafee.com>
   3  * Copyright (C) 2004-2010 David McCullough
   4  * The license and original author are listed below.
   5  *
   6  * Copyright (c) 2003 Sam Leffler, Errno Consulting
   7  * Copyright (c) 2003 Global Technology Associates, Inc.
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  * 1. Redistributions of source code must retain the above copyright
  14  *    notice, this list of conditions and the following disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  29  * SUCH DAMAGE.
  30  *
  31 __FBSDID("$FreeBSD: src/sys/dev/safe/safe.c,v 1.18 2007/03/21 03:42:50 sam Exp $");
  32  */
  33
  34 #include <linux/version.h>
  35 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) && !defined(AUTOCONF_INCLUDED)
  36 #include <linux/config.h>
  37 #endif
  38 #include <linux/module.h>
  39 #include <linux/kernel.h>
  40 #include <linux/init.h>
  41 #include <linux/list.h>
  42 #include <linux/slab.h>
  43 #include <linux/wait.h>
  44 #include <linux/sched.h>
  45 #include <linux/pci.h>
  46 #include <linux/delay.h>
  47 #include <linux/interrupt.h>
  48 #include <linux/spinlock.h>
  49 #include <linux/random.h>
  50 #include <linux/skbuff.h>
  51 #include <asm/io.h>
  52
  53 /*
  54  * SafeNet SafeXcel-1141 hardware crypto accelerator
  55  */
  56
  57 #include <cryptodev.h>
  58 #include <uio.h>
  59 #include <safe/safereg.h>
  60 #include <safe/safevar.h>
  61
  62 #if 1
  63 #define DPRINTF(a)      do { \
  64                                                 if (debug) { \
  65                                                         printk("%s: ", sc ? \
  66                                                                 device_get_nameunit(sc->sc_dev) : "safe"); \
  67                                                         printk a; \
  68                                                 } \
  69                                         } while (0)
  70 #else
  71 #define DPRINTF(a)
  72 #endif
  73
  74 /*
  75  * until we find a cleaner way, include the BSD md5/sha1 code
  76  * here
  77  */
  78 #define HMAC_HACK 1
  79 #ifdef HMAC_HACK
  80 #include <safe/hmachack.h>
  81 #include <safe/md5.h>
  82 #include <safe/md5.c>
  83 #include <safe/sha1.h>
  84 #include <safe/sha1.c>
  85 #endif /* HMAC_HACK */
  86
  87 /* add proc entry for this */
  88 struct safe_stats safestats;
  89
  90 #define debug safe_debug
  91 int safe_debug = 0;
  92 module_param(safe_debug, int, 0644);
  93 MODULE_PARM_DESC(safe_debug, "Enable debug");
  94
  95 static  void safe_callback(struct safe_softc *, struct safe_ringentry *);
  96 static  void safe_feed(struct safe_softc *, struct safe_ringentry *);
  97 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
  98 static  void safe_rng_init(struct safe_softc *);
  99 int safe_rngbufsize = 8;                /* 32 bytes each read  */
 100 module_param(safe_rngbufsize, int, 0644);
 101 MODULE_PARM_DESC(safe_rngbufsize, "RNG polling buffer size (32-bit words)");
 102 int safe_rngmaxalarm = 8;               /* max alarms before reset */
 103 module_param(safe_rngmaxalarm, int, 0644);
 104 MODULE_PARM_DESC(safe_rngmaxalarm, "RNG max alarms before reset");
 105 #endif /* SAFE_NO_RNG */
 106
 107 static void safe_totalreset(struct safe_softc *sc);
 108 static int safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op);
 109 static int safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op);
 110 static int safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re);
 111 static int safe_kprocess(device_t dev, struct cryptkop *krp, int hint);
 112 static int safe_kstart(struct safe_softc *sc);
 113 static int safe_ksigbits(struct safe_softc *sc, struct crparam *cr);
 114 static void safe_kfeed(struct safe_softc *sc);
 115 static void safe_kpoll(unsigned long arg);
 116 static void safe_kload_reg(struct safe_softc *sc, u_int32_t off,
 117                                                                 u_int32_t len, struct crparam *n);
 118
 119 static  int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
 120 static  int safe_freesession(device_t, u_int64_t);
 121 static  int safe_process(device_t, struct cryptop *, int);
 122
 123 static device_method_t safe_methods = {
 124         /* crypto device methods */
 125         DEVMETHOD(cryptodev_newsession, safe_newsession),
 126         DEVMETHOD(cryptodev_freesession,safe_freesession),
 127         DEVMETHOD(cryptodev_process,    safe_process),
 128         DEVMETHOD(cryptodev_kprocess,   safe_kprocess),
 129 };
 130
 131 #define READ_REG(sc,r)                  readl((sc)->sc_base_addr + (r))
 132 #define WRITE_REG(sc,r,val)             writel((val), (sc)->sc_base_addr + (r))
 133
 134 #define SAFE_MAX_CHIPS 8
 135 static struct safe_softc *safe_chip_idx[SAFE_MAX_CHIPS];
 136
 137 /*
 138  * split our buffers up into safe DMAable byte fragments to avoid lockup
 139  * bug in 1141 HW on rev 1.0.
 140  */
 141
 142 static int
 143 pci_map_linear(
 144         struct safe_softc *sc,
 145         struct safe_operand *buf,
 146         void *addr,
 147         int len)
 148 {
 149         dma_addr_t tmp;
 150         int chunk, tlen = len;
 151
 152         tmp = pci_map_single(sc->sc_pcidev, addr, len, PCI_DMA_BIDIRECTIONAL);
 153
 154         buf->mapsize += len;
 155         while (len > 0) {
 156                 chunk = (len > sc->sc_max_dsize) ? sc->sc_max_dsize : len;
 157                 buf->segs[buf->nsegs].ds_addr = tmp;
 158                 buf->segs[buf->nsegs].ds_len  = chunk;
 159                 buf->segs[buf->nsegs].ds_tlen = tlen;
 160                 buf->nsegs++;
 161                 tmp  += chunk;
 162                 len  -= chunk;
 163                 tlen = 0;
 164         }
 165         return 0;
 166 }
 167
 168 /*
 169  * map in a given uio buffer (great on some arches :-)
 170  */
 171
 172 static int
 173 pci_map_uio(struct safe_softc *sc, struct safe_operand *buf, struct uio *uio)
 174 {
 175         struct iovec *iov = uio->uio_iov;
 176         int n;
 177
 178         DPRINTF(("%s()\n", __FUNCTION__));
 179
 180         buf->mapsize = 0;
 181         buf->nsegs = 0;
 182
 183         for (n = 0; n < uio->uio_iovcnt; n++) {
 184                 pci_map_linear(sc, buf, iov->iov_base, iov->iov_len);
 185                 iov++;
 186         }
 187
 188         /* identify this buffer by the first segment */
 189         buf->map = (void *) buf->segs[0].ds_addr;
 190         return(0);
 191 }
 192
 193 /*
 194  * map in a given sk_buff
 195  */
 196
 197 static int
 198 pci_map_skb(struct safe_softc *sc,struct safe_operand *buf,struct sk_buff *skb)
 199 {
 200         int i;
 201
 202         DPRINTF(("%s()\n", __FUNCTION__));
 203
 204         buf->mapsize = 0;
 205         buf->nsegs = 0;
 206
 207         pci_map_linear(sc, buf, skb->data, skb_headlen(skb));
 208
 209         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 210                 pci_map_linear(sc, buf,
 211                                 page_address(skb_frag_page(&skb_shinfo(skb)->frags[i])) +
 212                                                         skb_shinfo(skb)->frags[i].page_offset,
 213                                 skb_shinfo(skb)->frags[i].size);
 214         }
 215
 216         /* identify this buffer by the first segment */
 217         buf->map = (void *) buf->segs[0].ds_addr;
 218         return(0);
 219 }
 220
 221
 222 #if 0 /* not needed at this time */
 223 static void
 224 pci_sync_operand(struct safe_softc *sc, struct safe_operand *buf)
 225 {
 226         int i;
 227
 228         DPRINTF(("%s()\n", __FUNCTION__));
 229         for (i = 0; i < buf->nsegs; i++)
 230                 pci_dma_sync_single_for_cpu(sc->sc_pcidev, buf->segs[i].ds_addr,
 231                                 buf->segs[i].ds_len, PCI_DMA_BIDIRECTIONAL);
 232 }
 233 #endif
 234
 235 static void
 236 pci_unmap_operand(struct safe_softc *sc, struct safe_operand *buf)
 237 {
 238         int i;
 239         DPRINTF(("%s()\n", __FUNCTION__));
 240         for (i = 0; i < buf->nsegs; i++) {
 241                 if (buf->segs[i].ds_tlen) {
 242                         DPRINTF(("%s - unmap %d 0x%x %d\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
 243                         pci_unmap_single(sc->sc_pcidev, buf->segs[i].ds_addr,
 244                                         buf->segs[i].ds_tlen, PCI_DMA_BIDIRECTIONAL);
 245                         DPRINTF(("%s - unmap %d 0x%x %d done\n", __FUNCTION__, i, buf->segs[i].ds_addr, buf->segs[i].ds_tlen));
 246                 }
 247                 buf->segs[i].ds_addr = 0;
 248                 buf->segs[i].ds_len = 0;
 249                 buf->segs[i].ds_tlen = 0;
 250         }
 251         buf->nsegs = 0;
 252         buf->mapsize = 0;
 253         buf->map = 0;
 254 }
 255
 256
 257 /*
 258  * SafeXcel Interrupt routine
 259  */
 260 static irqreturn_t
 261 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 262 safe_intr(int irq, void *arg)
 263 #else
 264 safe_intr(int irq, void *arg, struct pt_regs *regs)
 265 #endif
 266 {
 267         struct safe_softc *sc = arg;
 268         int stat;
 269         unsigned long flags;
 270
 271         stat = READ_REG(sc, SAFE_HM_STAT);
 272
 273         DPRINTF(("%s(stat=0x%x)\n", __FUNCTION__, stat));
 274
 275         if (stat == 0)          /* shared irq, not for us */
 276                 return IRQ_NONE;
 277
 278         WRITE_REG(sc, SAFE_HI_CLR, stat);       /* IACK */
 279
 280         if ((stat & SAFE_INT_PE_DDONE)) {
 281                 /*
 282                  * Descriptor(s) done; scan the ring and
 283                  * process completed operations.
 284                  */
 285                 spin_lock_irqsave(&sc->sc_ringmtx, flags);
 286                 while (sc->sc_back != sc->sc_front) {
 287                         struct safe_ringentry *re = sc->sc_back;
 288
 289 #ifdef SAFE_DEBUG
 290                         if (debug) {
 291                                 safe_dump_ringstate(sc, __func__);
 292                                 safe_dump_request(sc, __func__, re);
 293                         }
 294 #endif
 295                         /*
 296                          * safe_process marks ring entries that were allocated
 297                          * but not used with a csr of zero.  This insures the
 298                          * ring front pointer never needs to be set backwards
 299                          * in the event that an entry is allocated but not used
 300                          * because of a setup error.
 301                          */
 302                         DPRINTF(("%s re->re_desc.d_csr=0x%x\n", __FUNCTION__, re->re_desc.d_csr));
 303                         if (re->re_desc.d_csr != 0) {
 304                                 if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr)) {
 305                                         DPRINTF(("%s !CSR_IS_DONE\n", __FUNCTION__));
 306                                         break;
 307                                 }
 308                                 if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len)) {
 309                                         DPRINTF(("%s !LEN_IS_DONE\n", __FUNCTION__));
 310                                         break;
 311                                 }
 312                                 sc->sc_nqchip--;
 313                                 safe_callback(sc, re);
 314                         }
 315                         if (++(sc->sc_back) == sc->sc_ringtop)
 316                                 sc->sc_back = sc->sc_ring;
 317                 }
 318                 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
 319         }
 320
 321         /*
 322          * Check to see if we got any DMA Error
 323          */
 324         if (stat & SAFE_INT_PE_ERROR) {
 325                 printk("%s: dmaerr dmastat %08x\n", device_get_nameunit(sc->sc_dev),
 326                                 (int)READ_REG(sc, SAFE_PE_DMASTAT));
 327                 safestats.st_dmaerr++;
 328                 safe_totalreset(sc);
 329 #if 0
 330                 safe_feed(sc);
 331 #endif
 332         }
 333
 334         if (sc->sc_needwakeup) {                /* XXX check high watermark */
 335                 int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 336                 DPRINTF(("%s: wakeup crypto %x\n", __func__,
 337                         sc->sc_needwakeup));
 338                 sc->sc_needwakeup &= ~wakeup;
 339                 crypto_unblock(sc->sc_cid, wakeup);
 340         }
 341
 342         return IRQ_HANDLED;
 343 }
 344
 345 /*
 346  * safe_feed() - post a request to chip
 347  */
 348 static void
 349 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
 350 {
 351         DPRINTF(("%s()\n", __FUNCTION__));
 352 #ifdef SAFE_DEBUG
 353         if (debug) {
 354                 safe_dump_ringstate(sc, __func__);
 355                 safe_dump_request(sc, __func__, re);
 356         }
 357 #endif
 358         sc->sc_nqchip++;
 359         if (sc->sc_nqchip > safestats.st_maxqchip)
 360                 safestats.st_maxqchip = sc->sc_nqchip;
 361         /* poke h/w to check descriptor ring, any value can be written */
 362         WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
 363 }
 364
 365 #define N(a)    (sizeof(a) / sizeof (a[0]))
 366 static void
 367 safe_setup_enckey(struct safe_session *ses, caddr_t key)
 368 {
 369         int i;
 370
 371         bcopy(key, ses->ses_key, ses->ses_klen / 8);
 372
 373         /* PE is little-endian, insure proper byte order */
 374         for (i = 0; i < N(ses->ses_key); i++)
 375                 ses->ses_key[i] = htole32(ses->ses_key[i]);
 376 }
 377
 378 static void
 379 safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
 380 {
 381 #ifdef HMAC_HACK
 382         MD5_CTX md5ctx;
 383         SHA1_CTX sha1ctx;
 384         int i;
 385
 386
 387         for (i = 0; i < klen; i++)
 388                 key[i] ^= HMAC_IPAD_VAL;
 389
 390         if (algo == CRYPTO_MD5_HMAC) {
 391                 MD5Init(&md5ctx);
 392                 MD5Update(&md5ctx, key, klen);
 393                 MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 394                 bcopy(md5ctx.md5_st8, ses->ses_hminner, sizeof(md5ctx.md5_st8));
 395         } else {
 396                 SHA1Init(&sha1ctx);
 397                 SHA1Update(&sha1ctx, key, klen);
 398                 SHA1Update(&sha1ctx, hmac_ipad_buffer,
 399                     SHA1_HMAC_BLOCK_LEN - klen);
 400                 bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 401         }
 402
 403         for (i = 0; i < klen; i++)
 404                 key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 405
 406         if (algo == CRYPTO_MD5_HMAC) {
 407                 MD5Init(&md5ctx);
 408                 MD5Update(&md5ctx, key, klen);
 409                 MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 410                 bcopy(md5ctx.md5_st8, ses->ses_hmouter, sizeof(md5ctx.md5_st8));
 411         } else {
 412                 SHA1Init(&sha1ctx);
 413                 SHA1Update(&sha1ctx, key, klen);
 414                 SHA1Update(&sha1ctx, hmac_opad_buffer,
 415                     SHA1_HMAC_BLOCK_LEN - klen);
 416                 bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 417         }
 418
 419         for (i = 0; i < klen; i++)
 420                 key[i] ^= HMAC_OPAD_VAL;
 421
 422 #if 0
 423         /*
 424          * this code prevents SHA working on a BE host,
 425          * so it is obviously wrong.  I think the byte
 426          * swap setup we do with the chip fixes this for us
 427          */
 428
 429         /* PE is little-endian, insure proper byte order */
 430         for (i = 0; i < N(ses->ses_hminner); i++) {
 431                 ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
 432                 ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
 433         }
 434 #endif
 435 #else /* HMAC_HACK */
 436         printk("safe: md5/sha not implemented\n");
 437 #endif /* HMAC_HACK */
 438 }
 439 #undef N
 440
 441 /*
 442  * Allocate a new 'session' and return an encoded session id.  'sidp'
 443  * contains our registration id, and should contain an encoded session
 444  * id on successful allocation.
 445  */
 446 static int
 447 safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 448 {
 449         struct safe_softc *sc = device_get_softc(dev);
 450         struct cryptoini *c, *encini = NULL, *macini = NULL;
 451         struct safe_session *ses = NULL;
 452         int sesn;
 453
 454         DPRINTF(("%s()\n", __FUNCTION__));
 455
 456         if (sidp == NULL || cri == NULL || sc == NULL)
 457                 return (EINVAL);
 458
 459         for (c = cri; c != NULL; c = c->cri_next) {
 460                 if (c->cri_alg == CRYPTO_MD5_HMAC ||
 461                     c->cri_alg == CRYPTO_SHA1_HMAC ||
 462                     c->cri_alg == CRYPTO_NULL_HMAC) {
 463                         if (macini)
 464                                 return (EINVAL);
 465                         macini = c;
 466                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 467                     c->cri_alg == CRYPTO_3DES_CBC ||
 468                     c->cri_alg == CRYPTO_AES_CBC ||
 469                     c->cri_alg == CRYPTO_NULL_CBC) {
 470                         if (encini)
 471                                 return (EINVAL);
 472                         encini = c;
 473                 } else
 474                         return (EINVAL);
 475         }
 476         if (encini == NULL && macini == NULL)
 477                 return (EINVAL);
 478         if (encini) {                   /* validate key length */
 479                 switch (encini->cri_alg) {
 480                 case CRYPTO_DES_CBC:
 481                         if (encini->cri_klen != 64)
 482                                 return (EINVAL);
 483                         break;
 484                 case CRYPTO_3DES_CBC:
 485                         if (encini->cri_klen != 192)
 486                                 return (EINVAL);
 487                         break;
 488                 case CRYPTO_AES_CBC:
 489                         if (encini->cri_klen != 128 &&
 490                             encini->cri_klen != 192 &&
 491                             encini->cri_klen != 256)
 492                                 return (EINVAL);
 493                         break;
 494                 }
 495         }
 496
 497         if (sc->sc_sessions == NULL) {
 498                 ses = sc->sc_sessions = (struct safe_session *)
 499                         kmalloc(sizeof(struct safe_session), SLAB_ATOMIC);
 500                 if (ses == NULL)
 501                         return (ENOMEM);
 502                 memset(ses, 0, sizeof(struct safe_session));
 503                 sesn = 0;
 504                 sc->sc_nsessions = 1;
 505         } else {
 506                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 507                         if (sc->sc_sessions[sesn].ses_used == 0) {
 508                                 ses = &sc->sc_sessions[sesn];
 509                                 break;
 510                         }
 511                 }
 512
 513                 if (ses == NULL) {
 514                         sesn = sc->sc_nsessions;
 515                         ses = (struct safe_session *)
 516                                 kmalloc((sesn + 1) * sizeof(struct safe_session), SLAB_ATOMIC);
 517                         if (ses == NULL)
 518                                 return (ENOMEM);
 519                         memset(ses, 0, (sesn + 1) * sizeof(struct safe_session));
 520                         bcopy(sc->sc_sessions, ses, sesn *
 521                             sizeof(struct safe_session));
 522                         bzero(sc->sc_sessions, sesn *
 523                             sizeof(struct safe_session));
 524                         kfree(sc->sc_sessions);
 525                         sc->sc_sessions = ses;
 526                         ses = &sc->sc_sessions[sesn];
 527                         sc->sc_nsessions++;
 528                 }
 529         }
 530
 531         bzero(ses, sizeof(struct safe_session));
 532         ses->ses_used = 1;
 533
 534         if (encini) {
 535                 ses->ses_klen = encini->cri_klen;
 536                 if (encini->cri_key != NULL)
 537                         safe_setup_enckey(ses, encini->cri_key);
 538         }
 539
 540         if (macini) {
 541                 ses->ses_mlen = macini->cri_mlen;
 542                 if (ses->ses_mlen == 0) {
 543                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 544                                 ses->ses_mlen = MD5_HASH_LEN;
 545                         else
 546                                 ses->ses_mlen = SHA1_HASH_LEN;
 547                 }
 548
 549                 if (macini->cri_key != NULL) {
 550                         safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
 551                             macini->cri_klen / 8);
 552                 }
 553         }
 554
 555         *sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
 556         return (0);
 557 }
 558
 559 /*
 560  * Deallocate a session.
 561  */
 562 static int
 563 safe_freesession(device_t dev, u_int64_t tid)
 564 {
 565         struct safe_softc *sc = device_get_softc(dev);
 566         int session, ret;
 567         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 568
 569         DPRINTF(("%s()\n", __FUNCTION__));
 570
 571         if (sc == NULL)
 572                 return (EINVAL);
 573
 574         session = SAFE_SESSION(sid);
 575         if (session < sc->sc_nsessions) {
 576                 bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
 577                 ret = 0;
 578         } else
 579                 ret = EINVAL;
 580         return (ret);
 581 }
 582
 583
 584 static int
 585 safe_process(device_t dev, struct cryptop *crp, int hint)
 586 {
 587         struct safe_softc *sc = device_get_softc(dev);
 588         int err = 0, i, nicealign, uniform;
 589         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 590         int bypass, oplen, ivsize;
 591         caddr_t iv;
 592         int16_t coffset;
 593         struct safe_session *ses;
 594         struct safe_ringentry *re;
 595         struct safe_sarec *sa;
 596         struct safe_pdesc *pd;
 597         u_int32_t cmd0, cmd1, staterec, rand_iv[4];
 598         unsigned long flags;
 599
 600         DPRINTF(("%s()\n", __FUNCTION__));
 601
 602         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 603                 safestats.st_invalid++;
 604                 return (EINVAL);
 605         }
 606         if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 607                 safestats.st_badsession++;
 608                 return (EINVAL);
 609         }
 610
 611         spin_lock_irqsave(&sc->sc_ringmtx, flags);
 612         if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
 613                 safestats.st_ringfull++;
 614                 sc->sc_needwakeup |= CRYPTO_SYMQ;
 615                 spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
 616                 return (ERESTART);
 617         }
 618         re = sc->sc_front;
 619
 620         staterec = re->re_sa.sa_staterec;       /* save */
 621         /* NB: zero everything but the PE descriptor */
 622         bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
 623         re->re_sa.sa_staterec = staterec;       /* restore */
 624
 625         re->re_crp = crp;
 626         re->re_sesn = SAFE_SESSION(crp->crp_sid);
 627
 628         re->re_src.nsegs = 0;
 629         re->re_dst.nsegs = 0;
 630
 631         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 632                 re->re_src_skb = (struct sk_buff *)crp->crp_buf;
 633                 re->re_dst_skb = (struct sk_buff *)crp->crp_buf;
 634         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 635                 re->re_src_io = (struct uio *)crp->crp_buf;
 636                 re->re_dst_io = (struct uio *)crp->crp_buf;
 637         } else {
 638                 safestats.st_badflags++;
 639                 err = EINVAL;
 640                 goto errout;    /* XXX we don't handle contiguous blocks! */
 641         }
 642
 643         sa = &re->re_sa;
 644         ses = &sc->sc_sessions[re->re_sesn];
 645
 646         crd1 = crp->crp_desc;
 647         if (crd1 == NULL) {
 648                 safestats.st_nodesc++;
 649                 err = EINVAL;
 650                 goto errout;
 651         }
 652         crd2 = crd1->crd_next;
 653
 654         cmd0 = SAFE_SA_CMD0_BASIC;              /* basic group operation */
 655         cmd1 = 0;
 656         if (crd2 == NULL) {
 657                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 658                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 659                     crd1->crd_alg == CRYPTO_NULL_HMAC) {
 660                         maccrd = crd1;
 661                         enccrd = NULL;
 662                         cmd0 |= SAFE_SA_CMD0_OP_HASH;
 663                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 664                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 665                     crd1->crd_alg == CRYPTO_AES_CBC ||
 666                     crd1->crd_alg == CRYPTO_NULL_CBC) {
 667                         maccrd = NULL;
 668                         enccrd = crd1;
 669                         cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
 670                 } else {
 671                         safestats.st_badalg++;
 672                         err = EINVAL;
 673                         goto errout;
 674                 }
 675         } else {
 676                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 677                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 678                     crd1->crd_alg == CRYPTO_NULL_HMAC) &&
 679                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 680                         crd2->crd_alg == CRYPTO_3DES_CBC ||
 681                         crd2->crd_alg == CRYPTO_AES_CBC ||
 682                         crd2->crd_alg == CRYPTO_NULL_CBC) &&
 683                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 684                         maccrd = crd1;
 685                         enccrd = crd2;
 686                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 687                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 688                     crd1->crd_alg == CRYPTO_AES_CBC ||
 689                     crd1->crd_alg == CRYPTO_NULL_CBC) &&
 690                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 691                         crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 692                         crd2->crd_alg == CRYPTO_NULL_HMAC) &&
 693                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 694                         enccrd = crd1;
 695                         maccrd = crd2;
 696                 } else {
 697                         safestats.st_badalg++;
 698                         err = EINVAL;
 699                         goto errout;
 700                 }
 701                 cmd0 |= SAFE_SA_CMD0_OP_BOTH;
 702         }
 703
 704         if (enccrd) {
 705                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
 706                         safe_setup_enckey(ses, enccrd->crd_key);
 707
 708                 if (enccrd->crd_alg == CRYPTO_DES_CBC) {
 709                         cmd0 |= SAFE_SA_CMD0_DES;
 710                         cmd1 |= SAFE_SA_CMD1_CBC;
 711                         ivsize = 2*sizeof(u_int32_t);
 712                 } else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
 713                         cmd0 |= SAFE_SA_CMD0_3DES;
 714                         cmd1 |= SAFE_SA_CMD1_CBC;
 715                         ivsize = 2*sizeof(u_int32_t);
 716                 } else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
 717                         cmd0 |= SAFE_SA_CMD0_AES;
 718                         cmd1 |= SAFE_SA_CMD1_CBC;
 719                         if (ses->ses_klen == 128)
 720                              cmd1 |=  SAFE_SA_CMD1_AES128;
 721                         else if (ses->ses_klen == 192)
 722                              cmd1 |=  SAFE_SA_CMD1_AES192;
 723                         else
 724                              cmd1 |=  SAFE_SA_CMD1_AES256;
 725                         ivsize = 4*sizeof(u_int32_t);
 726                 } else {
 727                         cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
 728                         ivsize = 0;
 729                 }
 730
 731                 /*
 732                  * Setup encrypt/decrypt state.  When using basic ops
 733                  * we can't use an inline IV because hash/crypt offset
 734                  * must be from the end of the IV to the start of the
 735                  * crypt data and this leaves out the preceding header
 736                  * from the hash calculation.  Instead we place the IV
 737                  * in the state record and set the hash/crypt offset to
 738                  * copy both the header+IV.
 739                  */
 740                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 741                         cmd0 |= SAFE_SA_CMD0_OUTBOUND;
 742
 743                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 744                                 iv = enccrd->crd_iv;
 745                         else
 746                                 read_random((iv = (caddr_t) &rand_iv[0]), sizeof(rand_iv));
 747                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 748                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
 749                                     enccrd->crd_inject, ivsize, iv);
 750                         }
 751                         bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
 752                         /* make iv LE */
 753                         for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
 754                                 re->re_sastate.sa_saved_iv[i] =
 755                                         cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
 756                         cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
 757                         re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
 758                 } else {
 759                         cmd0 |= SAFE_SA_CMD0_INBOUND;
 760
 761                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 762                                 bcopy(enccrd->crd_iv,
 763                                         re->re_sastate.sa_saved_iv, ivsize);
 764                         } else {
 765                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 766                                     enccrd->crd_inject, ivsize,
 767                                     (caddr_t)re->re_sastate.sa_saved_iv);
 768                         }
 769                         /* make iv LE */
 770                         for (i = 0; i < ivsize/sizeof(re->re_sastate.sa_saved_iv[0]); i++)
 771                                 re->re_sastate.sa_saved_iv[i] =
 772                                         cpu_to_le32(re->re_sastate.sa_saved_iv[i]);
 773                         cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
 774                 }
 775                 /*
 776                  * For basic encryption use the zero pad algorithm.
 777                  * This pads results to an 8-byte boundary and
 778                  * suppresses padding verification for inbound (i.e.
 779                  * decrypt) operations.
 780                  *
 781                  * NB: Not sure if the 8-byte pad boundary is a problem.
 782                  */
 783                 cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
 784
 785                 /* XXX assert key bufs have the same size */
 786                 bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
 787         }
 788
 789         if (maccrd) {
 790                 if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 791                         safe_setup_mackey(ses, maccrd->crd_alg,
 792                             maccrd->crd_key, maccrd->crd_klen / 8);
 793                 }
 794
 795                 if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
 796                         cmd0 |= SAFE_SA_CMD0_MD5;
 797                         cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
 798                 } else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
 799                         cmd0 |= SAFE_SA_CMD0_SHA1;
 800                         cmd1 |= SAFE_SA_CMD1_HMAC;      /* NB: enable HMAC */
 801                 } else {
 802                         cmd0 |= SAFE_SA_CMD0_HASH_NULL;
 803                 }
 804                 /*
 805                  * Digest data is loaded from the SA and the hash
 806                  * result is saved to the state block where we
 807                  * retrieve it for return to the caller.
 808                  */
 809                 /* XXX assert digest bufs have the same size */
 810                 bcopy(ses->ses_hminner, sa->sa_indigest,
 811                         sizeof(sa->sa_indigest));
 812                 bcopy(ses->ses_hmouter, sa->sa_outdigest,
 813                         sizeof(sa->sa_outdigest));
 814
 815                 cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
 816                 re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
 817         }
 818
 819         if (enccrd && maccrd) {
 820                 /*
 821                  * The offset from hash data to the start of
 822                  * crypt data is the difference in the skips.
 823                  */
 824                 bypass = maccrd->crd_skip;
 825                 coffset = enccrd->crd_skip - maccrd->crd_skip;
 826                 if (coffset < 0) {
 827                         DPRINTF(("%s: hash does not precede crypt; "
 828                                 "mac skip %u enc skip %u\n",
 829                                 __func__, maccrd->crd_skip, enccrd->crd_skip));
 830                         safestats.st_skipmismatch++;
 831                         err = EINVAL;
 832                         goto errout;
 833                 }
 834                 oplen = enccrd->crd_skip + enccrd->crd_len;
 835                 if (maccrd->crd_skip + maccrd->crd_len != oplen) {
 836                         DPRINTF(("%s: hash amount %u != crypt amount %u\n",
 837                                 __func__, maccrd->crd_skip + maccrd->crd_len,
 838                                 oplen));
 839                         safestats.st_lenmismatch++;
 840                         err = EINVAL;
 841                         goto errout;
 842                 }
 843 #ifdef SAFE_DEBUG
 844                 if (debug) {
 845                         printf("mac: skip %d, len %d, inject %d\n",
 846                             maccrd->crd_skip, maccrd->crd_len,
 847                             maccrd->crd_inject);
 848                         printf("enc: skip %d, len %d, inject %d\n",
 849                             enccrd->crd_skip, enccrd->crd_len,
 850                             enccrd->crd_inject);
 851                         printf("bypass %d coffset %d oplen %d\n",
 852                                 bypass, coffset, oplen);
 853                 }
 854 #endif
 855                 if (coffset & 3) {      /* offset must be 32-bit aligned */
 856                         DPRINTF(("%s: coffset %u misaligned\n",
 857                                 __func__, coffset));
 858                         safestats.st_coffmisaligned++;
 859                         err = EINVAL;
 860                         goto errout;
 861                 }
 862                 coffset >>= 2;
 863                 if (coffset > 255) {    /* offset must be <256 dwords */
 864                         DPRINTF(("%s: coffset %u too big\n",
 865                                 __func__, coffset));
 866                         safestats.st_cofftoobig++;
 867                         err = EINVAL;
 868                         goto errout;
 869                 }
 870                 /*
 871                  * Tell the hardware to copy the header to the output.
 872                  * The header is defined as the data from the end of
 873                  * the bypass to the start of data to be encrypted.
 874                  * Typically this is the inline IV.  Note that you need
 875                  * to do this even if src+dst are the same; it appears
 876                  * that w/o this bit the crypted data is written
 877                  * immediately after the bypass data.
 878                  */
 879                 cmd1 |= SAFE_SA_CMD1_HDRCOPY;
 880                 /*
 881                  * Disable IP header mutable bit handling.  This is
 882                  * needed to get correct HMAC calculations.
 883                  */
 884                 cmd1 |= SAFE_SA_CMD1_MUTABLE;
 885         } else {
 886                 if (enccrd) {
 887                         bypass = enccrd->crd_skip;
 888                         oplen = bypass + enccrd->crd_len;
 889                 } else {
 890                         bypass = maccrd->crd_skip;
 891                         oplen = bypass + maccrd->crd_len;
 892                 }
 893                 coffset = 0;
 894         }
 895         /* XXX verify multiple of 4 when using s/g */
 896         if (bypass > 96) {              /* bypass offset must be <= 96 bytes */
 897                 DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
 898                 safestats.st_bypasstoobig++;
 899                 err = EINVAL;
 900                 goto errout;
 901         }
 902
 903         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 904                 if (pci_map_skb(sc, &re->re_src, re->re_src_skb)) {
 905                         safestats.st_noload++;
 906                         err = ENOMEM;
 907                         goto errout;
 908                 }
 909         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 910                 if (pci_map_uio(sc, &re->re_src, re->re_src_io)) {
 911                         safestats.st_noload++;
 912                         err = ENOMEM;
 913                         goto errout;
 914                 }
 915         }
 916         nicealign = safe_dmamap_aligned(sc, &re->re_src);
 917         uniform = safe_dmamap_uniform(sc, &re->re_src);
 918
 919         DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
 920                 nicealign, uniform, re->re_src.nsegs));
 921         if (re->re_src.nsegs > 1) {
 922                 re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
 923                         ((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
 924                 for (i = 0; i < re->re_src_nsegs; i++) {
 925                         /* NB: no need to check if there's space */
 926                         pd = sc->sc_spfree;
 927                         if (++(sc->sc_spfree) == sc->sc_springtop)
 928                                 sc->sc_spfree = sc->sc_spring;
 929
 930                         KASSERT((pd->pd_flags&3) == 0 ||
 931                                 (pd->pd_flags&3) == SAFE_PD_DONE,
 932                                 ("bogus source particle descriptor; flags %x",
 933                                 pd->pd_flags));
 934                         pd->pd_addr = re->re_src_segs[i].ds_addr;
 935                         pd->pd_size = re->re_src_segs[i].ds_len;
 936                         pd->pd_flags = SAFE_PD_READY;
 937                 }
 938                 cmd0 |= SAFE_SA_CMD0_IGATHER;
 939         } else {
 940                 /*
 941                  * No need for gather, reference the operand directly.
 942                  */
 943                 re->re_desc.d_src = re->re_src_segs[0].ds_addr;
 944         }
 945
 946         if (enccrd == NULL && maccrd != NULL) {
 947                 /*
 948                  * Hash op; no destination needed.
 949                  */
 950         } else {
 951                 if (crp->crp_flags & (CRYPTO_F_IOV|CRYPTO_F_SKBUF)) {
 952                         if (!nicealign) {
 953                                 safestats.st_iovmisaligned++;
 954                                 err = EINVAL;
 955                                 goto errout;
 956                         }
 957                         if (uniform != 1) {
 958                                 device_printf(sc->sc_dev, "!uniform source\n");
 959                                 if (!uniform) {
 960                                         /*
 961                                          * There's no way to handle the DMA
 962                                          * requirements with this uio.  We
 963                                          * could create a separate DMA area for
 964                                          * the result and then copy it back,
 965                                          * but for now we just bail and return
 966                                          * an error.  Note that uio requests
 967                                          * > SAFE_MAX_DSIZE are handled because
 968                                          * the DMA map and segment list for the
 969                                          * destination wil result in a
 970                                          * destination particle list that does
 971                                          * the necessary scatter DMA.
 972                                          */
 973                                         safestats.st_iovnotuniform++;
 974                                         err = EINVAL;
 975                                         goto errout;
 976                                 }
 977                         } else
 978                                 re->re_dst = re->re_src;
 979                 } else {
 980                         safestats.st_badflags++;
 981                         err = EINVAL;
 982                         goto errout;
 983                 }
 984
 985                 if (re->re_dst.nsegs > 1) {
 986                         re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
 987                             ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
 988                         for (i = 0; i < re->re_dst_nsegs; i++) {
 989                                 pd = sc->sc_dpfree;
 990                                 KASSERT((pd->pd_flags&3) == 0 ||
 991                                         (pd->pd_flags&3) == SAFE_PD_DONE,
 992                                         ("bogus dest particle descriptor; flags %x",
 993                                                 pd->pd_flags));
 994                                 if (++(sc->sc_dpfree) == sc->sc_dpringtop)
 995                                         sc->sc_dpfree = sc->sc_dpring;
 996                                 pd->pd_addr = re->re_dst_segs[i].ds_addr;
 997                                 pd->pd_flags = SAFE_PD_READY;
 998                         }
 999                         cmd0 |= SAFE_SA_CMD0_OSCATTER;
1000                 } else {
1001                         /*
1002                          * No need for scatter, reference the operand directly.
1003                          */
1004                         re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
1005                 }
1006         }
1007
1008         /*
1009          * All done with setup; fillin the SA command words
1010          * and the packet engine descriptor.  The operation
1011          * is now ready for submission to the hardware.
1012          */
1013         sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
1014         sa->sa_cmd1 = cmd1
1015                     | (coffset << SAFE_SA_CMD1_OFFSET_S)
1016                     | SAFE_SA_CMD1_SAREV1       /* Rev 1 SA data structure */
1017                     | SAFE_SA_CMD1_SRPCI
1018                     ;
1019         /*
1020          * NB: the order of writes is important here.  In case the
1021          * chip is scanning the ring because of an outstanding request
1022          * it might nab this one too.  In that case we need to make
1023          * sure the setup is complete before we write the length
1024          * field of the descriptor as it signals the descriptor is
1025          * ready for processing.
1026          */
1027         re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
1028         if (maccrd)
1029                 re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
1030         wmb();
1031         re->re_desc.d_len = oplen
1032                           | SAFE_PE_LEN_READY
1033                           | (bypass << SAFE_PE_LEN_BYPASS_S)
1034                           ;
1035
1036         safestats.st_ipackets++;
1037         safestats.st_ibytes += oplen;
1038
1039         if (++(sc->sc_front) == sc->sc_ringtop)
1040                 sc->sc_front = sc->sc_ring;
1041
1042         /* XXX honor batching */
1043         safe_feed(sc, re);
1044         spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1045         return (0);
1046
1047 errout:
1048         if (re->re_src.map != re->re_dst.map)
1049                 pci_unmap_operand(sc, &re->re_dst);
1050         if (re->re_src.map)
1051                 pci_unmap_operand(sc, &re->re_src);
1052         spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1053         if (err != ERESTART) {
1054                 crp->crp_etype = err;
1055                 crypto_done(crp);
1056         } else {
1057                 sc->sc_needwakeup |= CRYPTO_SYMQ;
1058         }
1059         return (err);
1060 }
1061
1062 static void
1063 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
1064 {
1065         struct cryptop *crp = (struct cryptop *)re->re_crp;
1066         struct cryptodesc *crd;
1067
1068         DPRINTF(("%s()\n", __FUNCTION__));
1069
1070         safestats.st_opackets++;
1071         safestats.st_obytes += re->re_dst.mapsize;
1072
1073         if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
1074                 device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
1075                         re->re_desc.d_csr,
1076                         re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
1077                 safestats.st_peoperr++;
1078                 crp->crp_etype = EIO;           /* something more meaningful? */
1079         }
1080
1081         if (re->re_dst.map != NULL && re->re_dst.map != re->re_src.map)
1082                 pci_unmap_operand(sc, &re->re_dst);
1083         pci_unmap_operand(sc, &re->re_src);
1084
1085         /*
1086          * If result was written to a differet mbuf chain, swap
1087          * it in as the return value and reclaim the original.
1088          */
1089         if ((crp->crp_flags & CRYPTO_F_SKBUF) && re->re_src_skb != re->re_dst_skb) {
1090                 device_printf(sc->sc_dev, "no CRYPTO_F_SKBUF swapping support\n");
1091                 /* kfree_skb(skb) */
1092                 /* crp->crp_buf = (caddr_t)re->re_dst_skb */
1093                 return;
1094         }
1095
1096         if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
1097                 /* copy out ICV result */
1098                 for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
1099                         if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
1100                             crd->crd_alg == CRYPTO_SHA1_HMAC ||
1101                             crd->crd_alg == CRYPTO_NULL_HMAC))
1102                                 continue;
1103                         if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
1104                                 /*
1105                                  * SHA-1 ICV's are byte-swapped; fix 'em up
1106                                  * before copy them to their destination.
1107                                  */
1108                                 re->re_sastate.sa_saved_indigest[0] =
1109                                         cpu_to_be32(re->re_sastate.sa_saved_indigest[0]);
1110                                 re->re_sastate.sa_saved_indigest[1] =
1111                                         cpu_to_be32(re->re_sastate.sa_saved_indigest[1]);
1112                                 re->re_sastate.sa_saved_indigest[2] =
1113                                         cpu_to_be32(re->re_sastate.sa_saved_indigest[2]);
1114                         } else {
1115                                 re->re_sastate.sa_saved_indigest[0] =
1116                                         cpu_to_le32(re->re_sastate.sa_saved_indigest[0]);
1117                                 re->re_sastate.sa_saved_indigest[1] =
1118                                         cpu_to_le32(re->re_sastate.sa_saved_indigest[1]);
1119                                 re->re_sastate.sa_saved_indigest[2] =
1120                                         cpu_to_le32(re->re_sastate.sa_saved_indigest[2]);
1121                         }
1122                         crypto_copyback(crp->crp_flags, crp->crp_buf,
1123                             crd->crd_inject,
1124                             sc->sc_sessions[re->re_sesn].ses_mlen,
1125                             (caddr_t)re->re_sastate.sa_saved_indigest);
1126                         break;
1127                 }
1128         }
1129         crypto_done(crp);
1130 }
1131
1132
1133 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
1134 #define SAFE_RNG_MAXWAIT        1000
1135
1136 static void
1137 safe_rng_init(struct safe_softc *sc)
1138 {
1139         u_int32_t w, v;
1140         int i;
1141
1142         DPRINTF(("%s()\n", __FUNCTION__));
1143
1144         WRITE_REG(sc, SAFE_RNG_CTRL, 0);
1145         /* use default value according to the manual */
1146         WRITE_REG(sc, SAFE_RNG_CNFG, 0x834);    /* magic from SafeNet */
1147         WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1148
1149         /*
1150          * There is a bug in rev 1.0 of the 1140 that when the RNG
1151          * is brought out of reset the ready status flag does not
1152          * work until the RNG has finished its internal initialization.
1153          *
1154          * So in order to determine the device is through its
1155          * initialization we must read the data register, using the
1156          * status reg in the read in case it is initialized.  Then read
1157          * the data register until it changes from the first read.
1158          * Once it changes read the data register until it changes
1159          * again.  At this time the RNG is considered initialized.
1160          * This could take between 750ms - 1000ms in time.
1161          */
1162         i = 0;
1163         w = READ_REG(sc, SAFE_RNG_OUT);
1164         do {
1165                 v = READ_REG(sc, SAFE_RNG_OUT);
1166                 if (v != w) {
1167                         w = v;
1168                         break;
1169                 }
1170                 DELAY(10);
1171         } while (++i < SAFE_RNG_MAXWAIT);
1172
1173         /* Wait Until data changes again */
1174         i = 0;
1175         do {
1176                 v = READ_REG(sc, SAFE_RNG_OUT);
1177                 if (v != w)
1178                         break;
1179                 DELAY(10);
1180         } while (++i < SAFE_RNG_MAXWAIT);
1181 }
1182
1183 static __inline void
1184 safe_rng_disable_short_cycle(struct safe_softc *sc)
1185 {
1186         DPRINTF(("%s()\n", __FUNCTION__));
1187
1188         WRITE_REG(sc, SAFE_RNG_CTRL,
1189                 READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
1190 }
1191
1192 static __inline void
1193 safe_rng_enable_short_cycle(struct safe_softc *sc)
1194 {
1195         DPRINTF(("%s()\n", __FUNCTION__));
1196
1197         WRITE_REG(sc, SAFE_RNG_CTRL,
1198                 READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
1199 }
1200
1201 static __inline u_int32_t
1202 safe_rng_read(struct safe_softc *sc)
1203 {
1204         int i;
1205
1206         i = 0;
1207         while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
1208                 ;
1209         return READ_REG(sc, SAFE_RNG_OUT);
1210 }
1211
1212 static int
1213 safe_read_random(void *arg, u_int32_t *buf, int maxwords)
1214 {
1215         struct safe_softc *sc = (struct safe_softc *) arg;
1216         int i, rc;
1217
1218         DPRINTF(("%s()\n", __FUNCTION__));
1219
1220         safestats.st_rng++;
1221         /*
1222          * Fetch the next block of data.
1223          */
1224         if (maxwords > safe_rngbufsize)
1225                 maxwords = safe_rngbufsize;
1226         if (maxwords > SAFE_RNG_MAXBUFSIZ)
1227                 maxwords = SAFE_RNG_MAXBUFSIZ;
1228 retry:
1229         /* read as much as we can */
1230         for (rc = 0; rc < maxwords; rc++) {
1231                 if (READ_REG(sc, SAFE_RNG_STAT) != 0)
1232                         break;
1233                 buf[rc] = READ_REG(sc, SAFE_RNG_OUT);
1234         }
1235         if (rc == 0)
1236                 return 0;
1237         /*
1238          * Check the comparator alarm count and reset the h/w if
1239          * it exceeds our threshold.  This guards against the
1240          * hardware oscillators resonating with external signals.
1241          */
1242         if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
1243                 u_int32_t freq_inc, w;
1244
1245                 DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
1246                         (unsigned)READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
1247                 safestats.st_rngalarm++;
1248                 safe_rng_enable_short_cycle(sc);
1249                 freq_inc = 18;
1250                 for (i = 0; i < 64; i++) {
1251                         w = READ_REG(sc, SAFE_RNG_CNFG);
1252                         freq_inc = ((w + freq_inc) & 0x3fL);
1253                         w = ((w & ~0x3fL) | freq_inc);
1254                         WRITE_REG(sc, SAFE_RNG_CNFG, w);
1255
1256                         WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1257
1258                         (void) safe_rng_read(sc);
1259                         DELAY(25);
1260
1261                         if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
1262                                 safe_rng_disable_short_cycle(sc);
1263                                 goto retry;
1264                         }
1265                         freq_inc = 1;
1266                 }
1267                 safe_rng_disable_short_cycle(sc);
1268         } else
1269                 WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
1270
1271         return(rc);
1272 }
1273 #endif /* defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG) */
1274
1275
1276 /*
1277  * Resets the board.  Values in the regesters are left as is
1278  * from the reset (i.e. initial values are assigned elsewhere).
1279  */
1280 static void
1281 safe_reset_board(struct safe_softc *sc)
1282 {
1283         u_int32_t v;
1284         /*
1285          * Reset the device.  The manual says no delay
1286          * is needed between marking and clearing reset.
1287          */
1288         DPRINTF(("%s()\n", __FUNCTION__));
1289
1290         v = READ_REG(sc, SAFE_PE_DMACFG) &~
1291                 (SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
1292                  SAFE_PE_DMACFG_SGRESET);
1293         WRITE_REG(sc, SAFE_PE_DMACFG, v
1294                                     | SAFE_PE_DMACFG_PERESET
1295                                     | SAFE_PE_DMACFG_PDRRESET
1296                                     | SAFE_PE_DMACFG_SGRESET);
1297         WRITE_REG(sc, SAFE_PE_DMACFG, v);
1298 }
1299
1300 /*
1301  * Initialize registers we need to touch only once.
1302  */
1303 static void
1304 safe_init_board(struct safe_softc *sc)
1305 {
1306         u_int32_t v, dwords;
1307
1308         DPRINTF(("%s()\n", __FUNCTION__));
1309
1310         v = READ_REG(sc, SAFE_PE_DMACFG);
1311         v &=~ (   SAFE_PE_DMACFG_PEMODE
1312                         | SAFE_PE_DMACFG_FSENA          /* failsafe enable */
1313                         | SAFE_PE_DMACFG_GPRPCI         /* gather ring on PCI */
1314                         | SAFE_PE_DMACFG_SPRPCI         /* scatter ring on PCI */
1315                         | SAFE_PE_DMACFG_ESDESC         /* endian-swap descriptors */
1316                         | SAFE_PE_DMACFG_ESPDESC        /* endian-swap part. desc's */
1317                         | SAFE_PE_DMACFG_ESSA           /* endian-swap SA's */
1318                         | SAFE_PE_DMACFG_ESPACKET       /* swap the packet data */
1319                   );
1320         v |= SAFE_PE_DMACFG_FSENA               /* failsafe enable */
1321           |  SAFE_PE_DMACFG_GPRPCI              /* gather ring on PCI */
1322           |  SAFE_PE_DMACFG_SPRPCI              /* scatter ring on PCI */
1323           |  SAFE_PE_DMACFG_ESDESC              /* endian-swap descriptors */
1324           |  SAFE_PE_DMACFG_ESPDESC             /* endian-swap part. desc's */
1325           |  SAFE_PE_DMACFG_ESSA                /* endian-swap SA's */
1326 #if 0
1327           |  SAFE_PE_DMACFG_ESPACKET    /* swap the packet data */
1328 #endif
1329           ;
1330         WRITE_REG(sc, SAFE_PE_DMACFG, v);
1331
1332 #ifdef __BIG_ENDIAN
1333         /* tell the safenet that we are 4321 and not 1234 */
1334         WRITE_REG(sc, SAFE_ENDIAN, 0xe4e41b1b);
1335 #endif
1336
1337         if (sc->sc_chiprev == SAFE_REV(1,0)) {
1338                 /*
1339                  * Avoid large PCI DMA transfers.  Rev 1.0 has a bug where
1340                  * "target mode transfers" done while the chip is DMA'ing
1341                  * >1020 bytes cause the hardware to lockup.  To avoid this
1342                  * we reduce the max PCI transfer size and use small source
1343                  * particle descriptors (<= 256 bytes).
1344                  */
1345                 WRITE_REG(sc, SAFE_DMA_CFG, 256);
1346                 device_printf(sc->sc_dev,
1347                         "Reduce max DMA size to %u words for rev %u.%u WAR\n",
1348                         (unsigned) ((READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff),
1349                         (unsigned) SAFE_REV_MAJ(sc->sc_chiprev),
1350                         (unsigned) SAFE_REV_MIN(sc->sc_chiprev));
1351                 sc->sc_max_dsize = 256;
1352         } else {
1353                 sc->sc_max_dsize = SAFE_MAX_DSIZE;
1354         }
1355
1356         /* NB: operands+results are overlaid */
1357         WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
1358         WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
1359         /*
1360          * Configure ring entry size and number of items in the ring.
1361          */
1362         KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
1363                 ("PE ring entry not 32-bit aligned!"));
1364         dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
1365         WRITE_REG(sc, SAFE_PE_RINGCFG,
1366                 (dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
1367         WRITE_REG(sc, SAFE_PE_RINGPOLL, 0);     /* disable polling */
1368
1369         WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
1370         WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
1371         WRITE_REG(sc, SAFE_PE_PARTSIZE,
1372                 (SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
1373         /*
1374          * NB: destination particles are fixed size.  We use
1375          *     an mbuf cluster and require all results go to
1376          *     clusters or smaller.
1377          */
1378         WRITE_REG(sc, SAFE_PE_PARTCFG, sc->sc_max_dsize);
1379
1380         /* it's now safe to enable PE mode, do it */
1381         WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
1382
1383         /*
1384          * Configure hardware to use level-triggered interrupts and
1385          * to interrupt after each descriptor is processed.
1386          */
1387         WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
1388         WRITE_REG(sc, SAFE_HI_CLR, 0xffffffff);
1389         WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
1390         WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
1391 }
1392
1393
1394 /*
1395  * Clean up after a chip crash.
1396  * It is assumed that the caller in splimp()
1397  */
1398 static void
1399 safe_cleanchip(struct safe_softc *sc)
1400 {
1401         DPRINTF(("%s()\n", __FUNCTION__));
1402
1403         if (sc->sc_nqchip != 0) {
1404                 struct safe_ringentry *re = sc->sc_back;
1405
1406                 while (re != sc->sc_front) {
1407                         if (re->re_desc.d_csr != 0)
1408                                 safe_free_entry(sc, re);
1409                         if (++re == sc->sc_ringtop)
1410                                 re = sc->sc_ring;
1411                 }
1412                 sc->sc_back = re;
1413                 sc->sc_nqchip = 0;
1414         }
1415 }
1416
1417 /*
1418  * free a safe_q
1419  * It is assumed that the caller is within splimp().
1420  */
1421 static int
1422 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
1423 {
1424         struct cryptop *crp;
1425
1426         DPRINTF(("%s()\n", __FUNCTION__));
1427
1428         /*
1429          * Free header MCR
1430          */
1431         if ((re->re_dst_skb != NULL) && (re->re_src_skb != re->re_dst_skb))
1432 #ifdef NOTYET
1433                 m_freem(re->re_dst_m);
1434 #else
1435                 printk("%s,%d: SKB not supported\n", __FILE__, __LINE__);
1436 #endif
1437
1438         crp = (struct cryptop *)re->re_crp;
1439
1440         re->re_desc.d_csr = 0;
1441
1442         crp->crp_etype = EFAULT;
1443         crypto_done(crp);
1444         return(0);
1445 }
1446
1447 /*
1448  * Routine to reset the chip and clean up.
1449  * It is assumed that the caller is in splimp()
1450  */
1451 static void
1452 safe_totalreset(struct safe_softc *sc)
1453 {
1454         DPRINTF(("%s()\n", __FUNCTION__));
1455
1456         safe_reset_board(sc);
1457         safe_init_board(sc);
1458         safe_cleanchip(sc);
1459 }
1460
1461 /*
1462  * Is the operand suitable aligned for direct DMA.  Each
1463  * segment must be aligned on a 32-bit boundary and all
1464  * but the last segment must be a multiple of 4 bytes.
1465  */
1466 static int
1467 safe_dmamap_aligned(struct safe_softc *sc, const struct safe_operand *op)
1468 {
1469         int i;
1470
1471         DPRINTF(("%s()\n", __FUNCTION__));
1472
1473         for (i = 0; i < op->nsegs; i++) {
1474                 if (op->segs[i].ds_addr & 3)
1475                         return (0);
1476                 if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
1477                         return (0);
1478         }
1479         return (1);
1480 }
1481
1482 /*
1483  * Is the operand suitable for direct DMA as the destination
1484  * of an operation.  The hardware requires that each ``particle''
1485  * but the last in an operation result have the same size.  We
1486  * fix that size at SAFE_MAX_DSIZE bytes.  This routine returns
1487  * 0 if some segment is not a multiple of of this size, 1 if all
1488  * segments are exactly this size, or 2 if segments are at worst
1489  * a multple of this size.
1490  */
1491 static int
1492 safe_dmamap_uniform(struct safe_softc *sc, const struct safe_operand *op)
1493 {
1494         int result = 1;
1495
1496         DPRINTF(("%s()\n", __FUNCTION__));
1497
1498         if (op->nsegs > 0) {
1499                 int i;
1500
1501                 for (i = 0; i < op->nsegs-1; i++) {
1502                         if (op->segs[i].ds_len % sc->sc_max_dsize)
1503                                 return (0);
1504                         if (op->segs[i].ds_len != sc->sc_max_dsize)
1505                                 result = 2;
1506                 }
1507         }
1508         return (result);
1509 }
1510
1511 static int
1512 safe_kprocess(device_t dev, struct cryptkop *krp, int hint)
1513 {
1514         struct safe_softc *sc = device_get_softc(dev);
1515         struct safe_pkq *q;
1516         unsigned long flags;
1517
1518         DPRINTF(("%s()\n", __FUNCTION__));
1519
1520         if (sc == NULL) {
1521                 krp->krp_status = EINVAL;
1522                 goto err;
1523         }
1524
1525         if (krp->krp_op != CRK_MOD_EXP) {
1526                 krp->krp_status = EOPNOTSUPP;
1527                 goto err;
1528         }
1529
1530         q = (struct safe_pkq *) kmalloc(sizeof(*q), GFP_KERNEL);
1531         if (q == NULL) {
1532                 krp->krp_status = ENOMEM;
1533                 goto err;
1534         }
1535         memset(q, 0, sizeof(*q));
1536         q->pkq_krp = krp;
1537         INIT_LIST_HEAD(&q->pkq_list);
1538
1539         spin_lock_irqsave(&sc->sc_pkmtx, flags);
1540         list_add_tail(&q->pkq_list, &sc->sc_pkq);
1541         safe_kfeed(sc);
1542         spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1543         return (0);
1544
1545 err:
1546         crypto_kdone(krp);
1547         return (0);
1548 }
1549
1550 #define SAFE_CRK_PARAM_BASE     0
1551 #define SAFE_CRK_PARAM_EXP      1
1552 #define SAFE_CRK_PARAM_MOD      2
1553
1554 static int
1555 safe_kstart(struct safe_softc *sc)
1556 {
1557         struct cryptkop *krp = sc->sc_pkq_cur->pkq_krp;
1558         int exp_bits, mod_bits, base_bits;
1559         u_int32_t op, a_off, b_off, c_off, d_off;
1560
1561         DPRINTF(("%s()\n", __FUNCTION__));
1562
1563         if (krp->krp_iparams < 3 || krp->krp_oparams != 1) {
1564                 krp->krp_status = EINVAL;
1565                 return (1);
1566         }
1567
1568         base_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1569         if (base_bits > 2048)
1570                 goto too_big;
1571         if (base_bits <= 0)             /* 5. base not zero */
1572                 goto too_small;
1573
1574         exp_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1575         if (exp_bits > 2048)
1576                 goto too_big;
1577         if (exp_bits <= 0)              /* 1. exponent word length > 0 */
1578                 goto too_small;         /* 4. exponent not zero */
1579
1580         mod_bits = safe_ksigbits(sc, &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1581         if (mod_bits > 2048)
1582                 goto too_big;
1583         if (mod_bits <= 32)             /* 2. modulus word length > 1 */
1584                 goto too_small;         /* 8. MSW of modulus != zero */
1585         if (mod_bits < exp_bits)        /* 3 modulus len >= exponent len */
1586                 goto too_small;
1587         if ((krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p[0] & 1) == 0)
1588                 goto bad_domain;        /* 6. modulus is odd */
1589         if (mod_bits > krp->krp_param[krp->krp_iparams].crp_nbits)
1590                 goto too_small;         /* make sure result will fit */
1591
1592         /* 7. modulus > base */
1593         if (mod_bits < base_bits)
1594                 goto too_small;
1595         if (mod_bits == base_bits) {
1596                 u_int8_t *basep, *modp;
1597                 int i;
1598
1599                 basep = krp->krp_param[SAFE_CRK_PARAM_BASE].crp_p +
1600                     ((base_bits + 7) / 8) - 1;
1601                 modp = krp->krp_param[SAFE_CRK_PARAM_MOD].crp_p +
1602                     ((mod_bits + 7) / 8) - 1;
1603
1604                 for (i = 0; i < (mod_bits + 7) / 8; i++, basep--, modp--) {
1605                         if (*modp < *basep)
1606                                 goto too_small;
1607                         if (*modp > *basep)
1608                                 break;
1609                 }
1610         }
1611
1612         /* And on the 9th step, he rested. */
1613
1614         WRITE_REG(sc, SAFE_PK_A_LEN, (exp_bits + 31) / 32);
1615         WRITE_REG(sc, SAFE_PK_B_LEN, (mod_bits + 31) / 32);
1616         if (mod_bits > 1024) {
1617                 op = SAFE_PK_FUNC_EXP4;
1618                 a_off = 0x000;
1619                 b_off = 0x100;
1620                 c_off = 0x200;
1621                 d_off = 0x300;
1622         } else {
1623                 op = SAFE_PK_FUNC_EXP16;
1624                 a_off = 0x000;
1625                 b_off = 0x080;
1626                 c_off = 0x100;
1627                 d_off = 0x180;
1628         }
1629         sc->sc_pk_reslen = b_off - a_off;
1630         sc->sc_pk_resoff = d_off;
1631
1632         /* A is exponent, B is modulus, C is base, D is result */
1633         safe_kload_reg(sc, a_off, b_off - a_off,
1634             &krp->krp_param[SAFE_CRK_PARAM_EXP]);
1635         WRITE_REG(sc, SAFE_PK_A_ADDR, a_off >> 2);
1636         safe_kload_reg(sc, b_off, b_off - a_off,
1637             &krp->krp_param[SAFE_CRK_PARAM_MOD]);
1638         WRITE_REG(sc, SAFE_PK_B_ADDR, b_off >> 2);
1639         safe_kload_reg(sc, c_off, b_off - a_off,
1640             &krp->krp_param[SAFE_CRK_PARAM_BASE]);
1641         WRITE_REG(sc, SAFE_PK_C_ADDR, c_off >> 2);
1642         WRITE_REG(sc, SAFE_PK_D_ADDR, d_off >> 2);
1643
1644         WRITE_REG(sc, SAFE_PK_FUNC, op | SAFE_PK_FUNC_RUN);
1645
1646         return (0);
1647
1648 too_big:
1649         krp->krp_status = E2BIG;
1650         return (1);
1651 too_small:
1652         krp->krp_status = ERANGE;
1653         return (1);
1654 bad_domain:
1655         krp->krp_status = EDOM;
1656         return (1);
1657 }
1658
1659 static int
1660 safe_ksigbits(struct safe_softc *sc, struct crparam *cr)
1661 {
1662         u_int plen = (cr->crp_nbits + 7) / 8;
1663         int i, sig = plen * 8;
1664         u_int8_t c, *p = cr->crp_p;
1665
1666         DPRINTF(("%s()\n", __FUNCTION__));
1667
1668         for (i = plen - 1; i >= 0; i--) {
1669                 c = p[i];
1670                 if (c != 0) {
1671                         while ((c & 0x80) == 0) {
1672                                 sig--;
1673                                 c <<= 1;
1674                         }
1675                         break;
1676                 }
1677                 sig -= 8;
1678         }
1679         return (sig);
1680 }
1681
1682 static void
1683 safe_kfeed(struct safe_softc *sc)
1684 {
1685         struct safe_pkq *q, *tmp;
1686
1687         DPRINTF(("%s()\n", __FUNCTION__));
1688
1689         if (list_empty(&sc->sc_pkq) && sc->sc_pkq_cur == NULL)
1690                 return;
1691         if (sc->sc_pkq_cur != NULL)
1692                 return;
1693         list_for_each_entry_safe(q, tmp, &sc->sc_pkq, pkq_list) {
1694                 sc->sc_pkq_cur = q;
1695                 list_del(&q->pkq_list);
1696                 if (safe_kstart(sc) != 0) {
1697                         crypto_kdone(q->pkq_krp);
1698                         kfree(q);
1699                         sc->sc_pkq_cur = NULL;
1700                 } else {
1701                         /* op started, start polling */
1702                         mod_timer(&sc->sc_pkto, jiffies + 1);
1703                         break;
1704                 }
1705         }
1706 }
1707
1708 static void
1709 safe_kpoll(unsigned long arg)
1710 {
1711         struct safe_softc *sc = NULL;
1712         struct safe_pkq *q;
1713         struct crparam *res;
1714         int i;
1715         u_int32_t buf[64];
1716         unsigned long flags;
1717
1718         DPRINTF(("%s()\n", __FUNCTION__));
1719
1720         if (arg >= SAFE_MAX_CHIPS)
1721                 return;
1722         sc = safe_chip_idx[arg];
1723         if (!sc) {
1724                 DPRINTF(("%s() - bad callback\n", __FUNCTION__));
1725                 return;
1726         }
1727
1728         spin_lock_irqsave(&sc->sc_pkmtx, flags);
1729         if (sc->sc_pkq_cur == NULL)
1730                 goto out;
1731         if (READ_REG(sc, SAFE_PK_FUNC) & SAFE_PK_FUNC_RUN) {
1732                 /* still running, check back later */
1733                 mod_timer(&sc->sc_pkto, jiffies + 1);
1734                 goto out;
1735         }
1736
1737         q = sc->sc_pkq_cur;
1738         res = &q->pkq_krp->krp_param[q->pkq_krp->krp_iparams];
1739         bzero(buf, sizeof(buf));
1740         bzero(res->crp_p, (res->crp_nbits + 7) / 8);
1741         for (i = 0; i < sc->sc_pk_reslen >> 2; i++)
1742                 buf[i] = le32_to_cpu(READ_REG(sc, SAFE_PK_RAM_START +
1743                     sc->sc_pk_resoff + (i << 2)));
1744         bcopy(buf, res->crp_p, (res->crp_nbits + 7) / 8);
1745         /*
1746          * reduce the bits that need copying if possible
1747          */
1748         res->crp_nbits = min(res->crp_nbits,sc->sc_pk_reslen * 8);
1749         res->crp_nbits = safe_ksigbits(sc, res);
1750
1751         for (i = SAFE_PK_RAM_START; i < SAFE_PK_RAM_END; i += 4)
1752                 WRITE_REG(sc, i, 0);
1753
1754         crypto_kdone(q->pkq_krp);
1755         kfree(q);
1756         sc->sc_pkq_cur = NULL;
1757
1758         safe_kfeed(sc);
1759 out:
1760         spin_unlock_irqrestore(&sc->sc_pkmtx, flags);
1761 }
1762
1763 static void
1764 safe_kload_reg(struct safe_softc *sc, u_int32_t off, u_int32_t len,
1765     struct crparam *n)
1766 {
1767         u_int32_t buf[64], i;
1768
1769         DPRINTF(("%s()\n", __FUNCTION__));
1770
1771         bzero(buf, sizeof(buf));
1772         bcopy(n->crp_p, buf, (n->crp_nbits + 7) / 8);
1773
1774         for (i = 0; i < len >> 2; i++)
1775                 WRITE_REG(sc, SAFE_PK_RAM_START + off + (i << 2),
1776                     cpu_to_le32(buf[i]));
1777 }
1778
1779 #ifdef SAFE_DEBUG
1780 static void
1781 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
1782 {
1783         printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
1784                 , tag
1785                 , READ_REG(sc, SAFE_DMA_ENDIAN)
1786                 , READ_REG(sc, SAFE_DMA_SRCADDR)
1787                 , READ_REG(sc, SAFE_DMA_DSTADDR)
1788                 , READ_REG(sc, SAFE_DMA_STAT)
1789         );
1790 }
1791
1792 static void
1793 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
1794 {
1795         printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
1796                 , tag
1797                 , READ_REG(sc, SAFE_HI_CFG)
1798                 , READ_REG(sc, SAFE_HI_MASK)
1799                 , READ_REG(sc, SAFE_HI_DESC_CNT)
1800                 , READ_REG(sc, SAFE_HU_STAT)
1801                 , READ_REG(sc, SAFE_HM_STAT)
1802         );
1803 }
1804
1805 static void
1806 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
1807 {
1808         u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
1809
1810         /* NB: assume caller has lock on ring */
1811         printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
1812                 tag,
1813                 estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
1814                 (unsigned long)(sc->sc_back - sc->sc_ring),
1815                 (unsigned long)(sc->sc_front - sc->sc_ring));
1816 }
1817
1818 static void
1819 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
1820 {
1821         int ix, nsegs;
1822
1823         ix = re - sc->sc_ring;
1824         printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
1825                 , tag
1826                 , re, ix
1827                 , re->re_desc.d_csr
1828                 , re->re_desc.d_src
1829                 , re->re_desc.d_dst
1830                 , re->re_desc.d_sa
1831                 , re->re_desc.d_len
1832         );
1833         if (re->re_src.nsegs > 1) {
1834                 ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
1835                         sizeof(struct safe_pdesc);
1836                 for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
1837                         printf(" spd[%u] %p: %p size %u flags %x"
1838                                 , ix, &sc->sc_spring[ix]
1839                                 , (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
1840                                 , sc->sc_spring[ix].pd_size
1841                                 , sc->sc_spring[ix].pd_flags
1842                         );
1843                         if (sc->sc_spring[ix].pd_size == 0)
1844                                 printf(" (zero!)");
1845                         printf("\n");
1846                         if (++ix == SAFE_TOTAL_SPART)
1847                                 ix = 0;
1848                 }
1849         }
1850         if (re->re_dst.nsegs > 1) {
1851                 ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
1852                         sizeof(struct safe_pdesc);
1853                 for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
1854                         printf(" dpd[%u] %p: %p flags %x\n"
1855                                 , ix, &sc->sc_dpring[ix]
1856                                 , (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
1857                                 , sc->sc_dpring[ix].pd_flags
1858                         );
1859                         if (++ix == SAFE_TOTAL_DPART)
1860                                 ix = 0;
1861                 }
1862         }
1863         printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
1864                 re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
1865         printf("sa: key %x %x %x %x %x %x %x %x\n"
1866                 , re->re_sa.sa_key[0]
1867                 , re->re_sa.sa_key[1]
1868                 , re->re_sa.sa_key[2]
1869                 , re->re_sa.sa_key[3]
1870                 , re->re_sa.sa_key[4]
1871                 , re->re_sa.sa_key[5]
1872                 , re->re_sa.sa_key[6]
1873                 , re->re_sa.sa_key[7]
1874         );
1875         printf("sa: indigest %x %x %x %x %x\n"
1876                 , re->re_sa.sa_indigest[0]
1877                 , re->re_sa.sa_indigest[1]
1878                 , re->re_sa.sa_indigest[2]
1879                 , re->re_sa.sa_indigest[3]
1880                 , re->re_sa.sa_indigest[4]
1881         );
1882         printf("sa: outdigest %x %x %x %x %x\n"
1883                 , re->re_sa.sa_outdigest[0]
1884                 , re->re_sa.sa_outdigest[1]
1885                 , re->re_sa.sa_outdigest[2]
1886                 , re->re_sa.sa_outdigest[3]
1887                 , re->re_sa.sa_outdigest[4]
1888         );
1889         printf("sr: iv %x %x %x %x\n"
1890                 , re->re_sastate.sa_saved_iv[0]
1891                 , re->re_sastate.sa_saved_iv[1]
1892                 , re->re_sastate.sa_saved_iv[2]
1893                 , re->re_sastate.sa_saved_iv[3]
1894         );
1895         printf("sr: hashbc %u indigest %x %x %x %x %x\n"
1896                 , re->re_sastate.sa_saved_hashbc
1897                 , re->re_sastate.sa_saved_indigest[0]
1898                 , re->re_sastate.sa_saved_indigest[1]
1899                 , re->re_sastate.sa_saved_indigest[2]
1900                 , re->re_sastate.sa_saved_indigest[3]
1901                 , re->re_sastate.sa_saved_indigest[4]
1902         );
1903 }
1904
1905 static void
1906 safe_dump_ring(struct safe_softc *sc, const char *tag)
1907 {
1908         unsigned long flags;
1909
1910         spin_lock_irqsave(&sc->sc_ringmtx, flags);
1911         printf("\nSafeNet Ring State:\n");
1912         safe_dump_intrstate(sc, tag);
1913         safe_dump_dmastatus(sc, tag);
1914         safe_dump_ringstate(sc, tag);
1915         if (sc->sc_nqchip) {
1916                 struct safe_ringentry *re = sc->sc_back;
1917                 do {
1918                         safe_dump_request(sc, tag, re);
1919                         if (++re == sc->sc_ringtop)
1920                                 re = sc->sc_ring;
1921                 } while (re != sc->sc_front);
1922         }
1923         spin_unlock_irqrestore(&sc->sc_ringmtx, flags);
1924 }
1925 #endif /* SAFE_DEBUG */
1926
1927
1928 static int safe_probe(struct pci_dev *dev, const struct pci_device_id *ent)
1929 {
1930         struct safe_softc *sc = NULL;
1931         u32 mem_start, mem_len, cmd;
1932         int i, rc, devinfo;
1933         dma_addr_t raddr;
1934         static int num_chips = 0;
1935
1936         DPRINTF(("%s()\n", __FUNCTION__));
1937
1938         if (pci_enable_device(dev) < 0)
1939                 return(-ENODEV);
1940
1941         if (!dev->irq) {
1942                 printk("safe: found device with no IRQ assigned. check BIOS settings!");
1943                 pci_disable_device(dev);
1944                 return(-ENODEV);
1945         }
1946
1947         if (pci_set_mwi(dev)) {
1948                 printk("safe: pci_set_mwi failed!");
1949                 return(-ENODEV);
1950         }
1951
1952         sc = (struct safe_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
1953         if (!sc)
1954                 return(-ENOMEM);
1955         memset(sc, 0, sizeof(*sc));
1956
1957         softc_device_init(sc, "safe", num_chips, safe_methods);
1958
1959         sc->sc_irq = -1;
1960         sc->sc_cid = -1;
1961         sc->sc_pcidev = dev;
1962         if (num_chips < SAFE_MAX_CHIPS) {
1963                 safe_chip_idx[device_get_unit(sc->sc_dev)] = sc;
1964                 num_chips++;
1965         }
1966
1967         INIT_LIST_HEAD(&sc->sc_pkq);
1968         spin_lock_init(&sc->sc_pkmtx);
1969
1970         pci_set_drvdata(sc->sc_pcidev, sc);
1971
1972         /* we read its hardware registers as memory */
1973         mem_start = pci_resource_start(sc->sc_pcidev, 0);
1974         mem_len   = pci_resource_len(sc->sc_pcidev, 0);
1975
1976         sc->sc_base_addr = (ocf_iomem_t) ioremap(mem_start, mem_len);
1977         if (!sc->sc_base_addr) {
1978                 device_printf(sc->sc_dev, "failed to ioremap 0x%x-0x%x\n",
1979                                 mem_start, mem_start + mem_len - 1);
1980                 goto out;
1981         }
1982
1983         /* fix up the bus size */
1984         if (pci_set_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
1985                 device_printf(sc->sc_dev, "No usable DMA configuration, aborting.\n");
1986                 goto out;
1987         }
1988         if (pci_set_consistent_dma_mask(sc->sc_pcidev, DMA_32BIT_MASK)) {
1989                 device_printf(sc->sc_dev, "No usable consistent DMA configuration, aborting.\n");
1990                 goto out;
1991         }
1992
1993         pci_set_master(sc->sc_pcidev);
1994
1995         pci_read_config_dword(sc->sc_pcidev, PCI_COMMAND, &cmd);
1996
1997         if (!(cmd & PCI_COMMAND_MEMORY)) {
1998                 device_printf(sc->sc_dev, "failed to enable memory mapping\n");
1999                 goto out;
2000         }
2001
2002         if (!(cmd & PCI_COMMAND_MASTER)) {
2003                 device_printf(sc->sc_dev, "failed to enable bus mastering\n");
2004                 goto out;
2005         }
2006
2007         rc = request_irq(dev->irq, safe_intr, IRQF_SHARED, "safe", sc);
2008         if (rc) {
2009                 device_printf(sc->sc_dev, "failed to hook irq %d\n", sc->sc_irq);
2010                 goto out;
2011         }
2012         sc->sc_irq = dev->irq;
2013
2014         sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
2015                         (SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
2016
2017         /*
2018          * Allocate packet engine descriptors.
2019          */
2020         sc->sc_ringalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2021                         SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2022                         &sc->sc_ringalloc.dma_paddr);
2023         if (!sc->sc_ringalloc.dma_vaddr) {
2024                 device_printf(sc->sc_dev, "cannot allocate PE descriptor ring\n");
2025                 goto out;
2026         }
2027
2028         /*
2029          * Hookup the static portion of all our data structures.
2030          */
2031         sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
2032         sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
2033         sc->sc_front = sc->sc_ring;
2034         sc->sc_back = sc->sc_ring;
2035         raddr = sc->sc_ringalloc.dma_paddr;
2036         bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
2037         for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
2038                 struct safe_ringentry *re = &sc->sc_ring[i];
2039
2040                 re->re_desc.d_sa = raddr +
2041                         offsetof(struct safe_ringentry, re_sa);
2042                 re->re_sa.sa_staterec = raddr +
2043                         offsetof(struct safe_ringentry, re_sastate);
2044
2045                 raddr += sizeof (struct safe_ringentry);
2046         }
2047         spin_lock_init(&sc->sc_ringmtx);
2048
2049         /*
2050          * Allocate scatter and gather particle descriptors.
2051          */
2052         sc->sc_spalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2053                         SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
2054                         &sc->sc_spalloc.dma_paddr);
2055         if (!sc->sc_spalloc.dma_vaddr) {
2056                 device_printf(sc->sc_dev, "cannot allocate source particle descriptor ring\n");
2057                 goto out;
2058         }
2059         sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
2060         sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
2061         sc->sc_spfree = sc->sc_spring;
2062         bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
2063
2064         sc->sc_dpalloc.dma_vaddr = pci_alloc_consistent(sc->sc_pcidev,
2065                         SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2066                         &sc->sc_dpalloc.dma_paddr);
2067         if (!sc->sc_dpalloc.dma_vaddr) {
2068                 device_printf(sc->sc_dev, "cannot allocate destination particle descriptor ring\n");
2069                 goto out;
2070         }
2071         sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
2072         sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
2073         sc->sc_dpfree = sc->sc_dpring;
2074         bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
2075
2076         sc->sc_cid = crypto_get_driverid(softc_get_device(sc), CRYPTOCAP_F_HARDWARE);
2077         if (sc->sc_cid < 0) {
2078                 device_printf(sc->sc_dev, "could not get crypto driver id\n");
2079                 goto out;
2080         }
2081
2082         printf("%s:", device_get_nameunit(sc->sc_dev));
2083
2084         devinfo = READ_REG(sc, SAFE_DEVINFO);
2085         if (devinfo & SAFE_DEVINFO_RNG) {
2086                 sc->sc_flags |= SAFE_FLAGS_RNG;
2087                 printf(" rng");
2088         }
2089         if (devinfo & SAFE_DEVINFO_PKEY) {
2090                 printf(" key");
2091                 sc->sc_flags |= SAFE_FLAGS_KEY;
2092                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
2093 #if 0
2094                 crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
2095 #endif
2096                 init_timer(&sc->sc_pkto);
2097                 sc->sc_pkto.function = safe_kpoll;
2098                 sc->sc_pkto.data = (unsigned long) device_get_unit(sc->sc_dev);
2099         }
2100         if (devinfo & SAFE_DEVINFO_DES) {
2101                 printf(" des/3des");
2102                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
2103                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
2104         }
2105         if (devinfo & SAFE_DEVINFO_AES) {
2106                 printf(" aes");
2107                 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
2108         }
2109         if (devinfo & SAFE_DEVINFO_MD5) {
2110                 printf(" md5");
2111                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
2112         }
2113         if (devinfo & SAFE_DEVINFO_SHA1) {
2114                 printf(" sha1");
2115                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
2116         }
2117         printf(" null");
2118         crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
2119         crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
2120         /* XXX other supported algorithms */
2121         printf("\n");
2122
2123         safe_reset_board(sc);           /* reset h/w */
2124         safe_init_board(sc);            /* init h/w */
2125
2126 #if defined(CONFIG_OCF_RANDOMHARVEST) && !defined(SAFE_NO_RNG)
2127         if (sc->sc_flags & SAFE_FLAGS_RNG) {
2128                 safe_rng_init(sc);
2129                 crypto_rregister(sc->sc_cid, safe_read_random, sc);
2130         }
2131 #endif /* SAFE_NO_RNG */
2132
2133         return (0);
2134
2135 out:
2136         if (sc->sc_cid >= 0)
2137                 crypto_unregister_all(sc->sc_cid);
2138         if (sc->sc_irq != -1)
2139                 free_irq(sc->sc_irq, sc);
2140         if (sc->sc_ringalloc.dma_vaddr)
2141                 pci_free_consistent(sc->sc_pcidev,
2142                                 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2143                                 sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2144         if (sc->sc_spalloc.dma_vaddr)
2145                 pci_free_consistent(sc->sc_pcidev,
2146                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2147                                 sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2148         if (sc->sc_dpalloc.dma_vaddr)
2149                 pci_free_consistent(sc->sc_pcidev,
2150                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2151                                 sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2152         kfree(sc);
2153         return(-ENODEV);
2154 }
2155
2156 static void safe_remove(struct pci_dev *dev)
2157 {
2158         struct safe_softc *sc = pci_get_drvdata(dev);
2159
2160         DPRINTF(("%s()\n", __FUNCTION__));
2161
2162         /* XXX wait/abort active ops */
2163
2164         WRITE_REG(sc, SAFE_HI_MASK, 0);         /* disable interrupts */
2165
2166         del_timer_sync(&sc->sc_pkto);
2167
2168         crypto_unregister_all(sc->sc_cid);
2169
2170         safe_cleanchip(sc);
2171
2172         if (sc->sc_irq != -1)
2173                 free_irq(sc->sc_irq, sc);
2174         if (sc->sc_ringalloc.dma_vaddr)
2175                 pci_free_consistent(sc->sc_pcidev,
2176                                 SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
2177                                 sc->sc_ringalloc.dma_vaddr, sc->sc_ringalloc.dma_paddr);
2178         if (sc->sc_spalloc.dma_vaddr)
2179                 pci_free_consistent(sc->sc_pcidev,
2180                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2181                                 sc->sc_spalloc.dma_vaddr, sc->sc_spalloc.dma_paddr);
2182         if (sc->sc_dpalloc.dma_vaddr)
2183                 pci_free_consistent(sc->sc_pcidev,
2184                                 SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
2185                                 sc->sc_dpalloc.dma_vaddr, sc->sc_dpalloc.dma_paddr);
2186         sc->sc_irq = -1;
2187         sc->sc_ringalloc.dma_vaddr = NULL;
2188         sc->sc_spalloc.dma_vaddr = NULL;
2189         sc->sc_dpalloc.dma_vaddr = NULL;
2190 }
2191
2192 static struct pci_device_id safe_pci_tbl[] = {
2193         { PCI_VENDOR_SAFENET, PCI_PRODUCT_SAFEXCEL,
2194           PCI_ANY_ID, PCI_ANY_ID, 0, 0, },
2195         { },
2196 };
2197 MODULE_DEVICE_TABLE(pci, safe_pci_tbl);
2198
2199 static struct pci_driver safe_driver = {
2200         .name         = "safe",
2201         .id_table     = safe_pci_tbl,
2202         .probe        = safe_probe,
2203         .remove       = safe_remove,
2204         /* add PM stuff here one day */
2205 };
2206
2207 static int __init safe_init (void)
2208 {
2209         struct safe_softc *sc = NULL;
2210         int rc;
2211
2212         DPRINTF(("%s(%p)\n", __FUNCTION__, safe_init));
2213
2214         rc = pci_register_driver(&safe_driver);
2215         pci_register_driver_compat(&safe_driver, rc);
2216
2217         return rc;
2218 }
2219
2220 static void __exit safe_exit (void)
2221 {
2222         pci_unregister_driver(&safe_driver);
2223 }
2224
2225 module_init(safe_init);
2226 module_exit(safe_exit);
2227
2228 MODULE_LICENSE("BSD");
2229 MODULE_AUTHOR("David McCullough <david_mccullough@mcafee.com>");
2230 MODULE_DESCRIPTION("OCF driver for safenet PCI crypto devices");