target/linux/generic/files/crypto/ocf/talitos/talitos.c

   1 /*
   2  * crypto/ocf/talitos/talitos.c
   3  *
   4  * An OCF-Linux module that uses Freescale's SEC to do the crypto.
   5  * Based on crypto/ocf/hifn and crypto/ocf/safe OCF drivers
   6  *
   7  * Copyright (c) 2006 Freescale Semiconductor, Inc.
   8  *
   9  * This code written by Kim A. B. Phillips <kim.phillips@freescale.com>
  10  * some code copied from files with the following:
  11  * Copyright (C) 2004-2007 David McCullough <david_mccullough@mcafee.com>
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. The name of the author may not be used to endorse or promote products
  23  *    derived from this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  28  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  34  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * ---------------------------------------------------------------------------
  37  *
  38  * NOTES:
  39  *
  40  * The Freescale SEC (also known as 'talitos') resides on the
  41  * internal bus, and runs asynchronous to the processor core.  It has
  42  * a wide gamut of cryptographic acceleration features, including single-
  43  * pass IPsec (also known as algorithm chaining).  To properly utilize
  44  * all of the SEC's performance enhancing features, further reworking
  45  * of higher level code (framework, applications) will be necessary.
  46  *
  47  * The following table shows which SEC version is present in which devices:
  48  *
  49  * Devices       SEC version
  50  *
  51  * 8272, 8248    SEC 1.0
  52  * 885, 875      SEC 1.2
  53  * 8555E, 8541E  SEC 2.0
  54  * 8349E         SEC 2.01
  55  * 8548E         SEC 2.1
  56  *
  57  * The following table shows the features offered by each SEC version:
  58  *
  59  *                             Max.   chan-
  60  * version  Bus I/F       Clock  nels  DEU AESU AFEU MDEU PKEU RNG KEU
  61  *
  62  * SEC 1.0  internal 64b  100MHz   4     1    1    1    1    1   1   0
  63  * SEC 1.2  internal 32b   66MHz   1     1    1    0    1    0   0   0
  64  * SEC 2.0  internal 64b  166MHz   4     1    1    1    1    1   1   0
  65  * SEC 2.01 internal 64b  166MHz   4     1    1    1    1    1   1   0
  66  * SEC 2.1  internal 64b  333MHz   4     1    1    1    1    1   1   1
  67  *
  68  * Each execution unit in the SEC has two modes of execution; channel and
  69  * slave/debug.  This driver employs the channel infrastructure in the
  70  * device for convenience.  Only the RNG is directly accessed due to the
  71  * convenience of its random fifo pool.  The relationship between the
  72  * channels and execution units is depicted in the following diagram:
  73  *
  74  *    -------   ------------
  75  * ---| ch0 |---|          |
  76  *    -------   |          |
  77  *              |          |------+-------+-------+-------+------------
  78  *    -------   |          |      |       |       |       |           |
  79  * ---| ch1 |---|          |      |       |       |       |           |
  80  *    -------   |          |   ------  ------  ------  ------      ------
  81  *              |controller|   |DEU |  |AESU|  |MDEU|  |PKEU| ...  |RNG |
  82  *    -------   |          |   ------  ------  ------  ------      ------
  83  * ---| ch2 |---|          |      |       |       |       |           |
  84  *    -------   |          |      |       |       |       |           |
  85  *              |          |------+-------+-------+-------+------------
  86  *    -------   |          |
  87  * ---| ch3 |---|          |
  88  *    -------   ------------
  89  *
  90  * Channel ch0 may drive an aes operation to the aes unit (AESU),
  91  * and, at the same time, ch1 may drive a message digest operation
  92  * to the mdeu. Each channel has an input descriptor FIFO, and the
  93  * FIFO can contain, e.g. on the 8541E, up to 24 entries, before a
  94  * a buffer overrun error is triggered. The controller is responsible
  95  * for fetching the data from descriptor pointers, and passing the
  96  * data to the appropriate EUs. The controller also writes the
  97  * cryptographic operation's result to memory. The SEC notifies
  98  * completion by triggering an interrupt and/or setting the 1st byte
  99  * of the hdr field to 0xff.
 100  *
 101  * TODO:
 102  * o support more algorithms
 103  * o support more versions of the SEC
 104  * o add support for linux 2.4
 105  * o scatter-gather (sg) support
 106  * o add support for public key ops (PKEU)
 107  * o add statistics
 108  */
 109
 110 #include <linux/version.h>
 111 #if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,38) && !defined(AUTOCONF_INCLUDED)
 112 #include <linux/config.h>
 113 #endif
 114 #include <linux/module.h>
 115 #include <linux/init.h>
 116 #include <linux/interrupt.h>
 117 #include <linux/spinlock.h>
 118 #include <linux/random.h>
 119 #include <linux/skbuff.h>
 120 #include <asm/scatterlist.h>
 121 #include <linux/dma-mapping.h>  /* dma_map_single() */
 122 #include <linux/moduleparam.h>
 123
 124 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15)
 125 #include <linux/platform_device.h>
 126 #endif
 127
 128 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 129 #include <linux/of_platform.h>
 130 #endif
 131
 132 #include <cryptodev.h>
 133 #include <uio.h>
 134
 135 #define DRV_NAME "talitos"
 136
 137 #include "talitos_dev.h"
 138 #include "talitos_soft.h"
 139
 140 #define read_random(p,l) get_random_bytes(p,l)
 141
 142 const char talitos_driver_name[] = "Talitos OCF";
 143 const char talitos_driver_version[] = "0.2";
 144
 145 static int talitos_newsession(device_t dev, u_int32_t *sidp,
 146                                                                 struct cryptoini *cri);
 147 static int talitos_freesession(device_t dev, u_int64_t tid);
 148 static int talitos_process(device_t dev, struct cryptop *crp, int hint);
 149 static void dump_talitos_status(struct talitos_softc *sc);
 150 static int talitos_submit(struct talitos_softc *sc, struct talitos_desc *td,
 151                                                                 int chsel);
 152 static void talitos_doneprocessing(struct talitos_softc *sc);
 153 static void talitos_init_device(struct talitos_softc *sc);
 154 static void talitos_reset_device_master(struct talitos_softc *sc);
 155 static void talitos_reset_device(struct talitos_softc *sc);
 156 static void talitos_errorprocessing(struct talitos_softc *sc);
 157 #ifdef CONFIG_PPC_MERGE
 158 static int talitos_probe(struct of_device *ofdev, const struct of_device_id *match);
 159 static int talitos_remove(struct of_device *ofdev);
 160 #else
 161 static int talitos_probe(struct platform_device *pdev);
 162 static int talitos_remove(struct platform_device *pdev);
 163 #endif
 164 #ifdef CONFIG_OCF_RANDOMHARVEST
 165 static int talitos_read_random(void *arg, u_int32_t *buf, int maxwords);
 166 static void talitos_rng_init(struct talitos_softc *sc);
 167 #endif
 168
 169 static device_method_t talitos_methods = {
 170         /* crypto device methods */
 171         DEVMETHOD(cryptodev_newsession, talitos_newsession),
 172         DEVMETHOD(cryptodev_freesession,talitos_freesession),
 173         DEVMETHOD(cryptodev_process,    talitos_process),
 174 };
 175
 176 #define debug talitos_debug
 177 int talitos_debug = 0;
 178 module_param(talitos_debug, int, 0644);
 179 MODULE_PARM_DESC(talitos_debug, "Enable debug");
 180
 181 static inline void talitos_write(volatile unsigned *addr, u32 val)
 182 {
 183         out_be32(addr, val);
 184 }
 185
 186 static inline u32 talitos_read(volatile unsigned *addr)
 187 {
 188         u32 val;
 189         val = in_be32(addr);
 190         return val;
 191 }
 192
 193 static void dump_talitos_status(struct talitos_softc *sc)
 194 {
 195         unsigned int v, v_hi, i, *ptr;
 196         v = talitos_read(sc->sc_base_addr + TALITOS_MCR);
 197         v_hi = talitos_read(sc->sc_base_addr + TALITOS_MCR_HI);
 198         printk(KERN_INFO "%s: MCR          0x%08x_%08x\n",
 199                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 200         v = talitos_read(sc->sc_base_addr + TALITOS_IMR);
 201         v_hi = talitos_read(sc->sc_base_addr + TALITOS_IMR_HI);
 202         printk(KERN_INFO "%s: IMR          0x%08x_%08x\n",
 203                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 204         v = talitos_read(sc->sc_base_addr + TALITOS_ISR);
 205         v_hi = talitos_read(sc->sc_base_addr + TALITOS_ISR_HI);
 206         printk(KERN_INFO "%s: ISR          0x%08x_%08x\n",
 207                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 208         for (i = 0; i < sc->sc_num_channels; i++) {
 209                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 210                         TALITOS_CH_CDPR);
 211                 v_hi = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 212                         TALITOS_CH_CDPR_HI);
 213                 printk(KERN_INFO "%s: CDPR     ch%d 0x%08x_%08x\n",
 214                                 device_get_nameunit(sc->sc_cdev), i, v, v_hi);
 215         }
 216         for (i = 0; i < sc->sc_num_channels; i++) {
 217                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 218                         TALITOS_CH_CCPSR);
 219                 v_hi = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 220                         TALITOS_CH_CCPSR_HI);
 221                 printk(KERN_INFO "%s: CCPSR    ch%d 0x%08x_%08x\n",
 222                                 device_get_nameunit(sc->sc_cdev), i, v, v_hi);
 223         }
 224         ptr = sc->sc_base_addr + TALITOS_CH_DESCBUF;
 225         for (i = 0; i < 16; i++) {
 226                 v = talitos_read(ptr++); v_hi = talitos_read(ptr++);
 227                 printk(KERN_INFO "%s: DESCBUF  ch0 0x%08x_%08x (tdp%02d)\n",
 228                                 device_get_nameunit(sc->sc_cdev), v, v_hi, i);
 229         }
 230         return;
 231 }
 232
 233
 234 #ifdef CONFIG_OCF_RANDOMHARVEST
 235 /*
 236  * pull random numbers off the RNG FIFO, not exceeding amount available
 237  */
 238 static int
 239 talitos_read_random(void *arg, u_int32_t *buf, int maxwords)
 240 {
 241         struct talitos_softc *sc = (struct talitos_softc *) arg;
 242         int rc;
 243         u_int32_t v;
 244
 245         DPRINTF("%s()\n", __FUNCTION__);
 246
 247         /* check for things like FIFO underflow */
 248         v = talitos_read(sc->sc_base_addr + TALITOS_RNGISR_HI);
 249         if (unlikely(v)) {
 250                 printk(KERN_ERR "%s: RNGISR_HI error %08x\n",
 251                                 device_get_nameunit(sc->sc_cdev), v);
 252                 return 0;
 253         }
 254         /*
 255          * OFL is number of available 64-bit words,
 256          * shift and convert to a 32-bit word count
 257          */
 258         v = talitos_read(sc->sc_base_addr + TALITOS_RNGSR_HI);
 259         v = (v & TALITOS_RNGSR_HI_OFL) >> (16 - 1);
 260         if (maxwords > v)
 261                 maxwords = v;
 262         for (rc = 0; rc < maxwords; rc++) {
 263                 buf[rc] = talitos_read(sc->sc_base_addr +
 264                         TALITOS_RNG_FIFO + rc*sizeof(u_int32_t));
 265         }
 266         if (maxwords & 1) {
 267                 /*
 268                  * RNG will complain with an AE in the RNGISR
 269                  * if we don't complete the pairs of 32-bit reads
 270                  * to its 64-bit register based FIFO
 271                  */
 272                 v = talitos_read(sc->sc_base_addr +
 273                         TALITOS_RNG_FIFO + rc*sizeof(u_int32_t));
 274         }
 275
 276         return rc;
 277 }
 278
 279 static void
 280 talitos_rng_init(struct talitos_softc *sc)
 281 {
 282         u_int32_t v;
 283
 284         DPRINTF("%s()\n", __FUNCTION__);
 285         /* reset RNG EU */
 286         v = talitos_read(sc->sc_base_addr + TALITOS_RNGRCR_HI);
 287         v |= TALITOS_RNGRCR_HI_SR;
 288         talitos_write(sc->sc_base_addr + TALITOS_RNGRCR_HI, v);
 289         while ((talitos_read(sc->sc_base_addr + TALITOS_RNGSR_HI)
 290                 & TALITOS_RNGSR_HI_RD) == 0)
 291                         cpu_relax();
 292         /*
 293          * we tell the RNG to start filling the RNG FIFO
 294          * by writing the RNGDSR
 295          */
 296         v = talitos_read(sc->sc_base_addr + TALITOS_RNGDSR_HI);
 297         talitos_write(sc->sc_base_addr + TALITOS_RNGDSR_HI, v);
 298         /*
 299          * 64 bits of data will be pushed onto the FIFO every
 300          * 256 SEC cycles until the FIFO is full.  The RNG then
 301          * attempts to keep the FIFO full.
 302          */
 303         v = talitos_read(sc->sc_base_addr + TALITOS_RNGISR_HI);
 304         if (v) {
 305                 printk(KERN_ERR "%s: RNGISR_HI error %08x\n",
 306                         device_get_nameunit(sc->sc_cdev), v);
 307                 return;
 308         }
 309         /*
 310          * n.b. we need to add a FIPS test here - if the RNG is going
 311          * to fail, it's going to fail at reset time
 312          */
 313         return;
 314 }
 315 #endif /* CONFIG_OCF_RANDOMHARVEST */
 316
 317 /*
 318  * Generate a new software session.
 319  */
 320 static int
 321 talitos_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 322 {
 323         struct cryptoini *c, *encini = NULL, *macini = NULL;
 324         struct talitos_softc *sc = device_get_softc(dev);
 325         struct talitos_session *ses = NULL;
 326         int sesn;
 327
 328         DPRINTF("%s()\n", __FUNCTION__);
 329         if (sidp == NULL || cri == NULL || sc == NULL) {
 330                 DPRINTF("%s,%d - EINVAL\n", __FILE__, __LINE__);
 331                 return EINVAL;
 332         }
 333         for (c = cri; c != NULL; c = c->cri_next) {
 334                 if (c->cri_alg == CRYPTO_MD5 ||
 335                     c->cri_alg == CRYPTO_MD5_HMAC ||
 336                     c->cri_alg == CRYPTO_SHA1 ||
 337                     c->cri_alg == CRYPTO_SHA1_HMAC ||
 338                     c->cri_alg == CRYPTO_NULL_HMAC) {
 339                         if (macini)
 340                                 return EINVAL;
 341                         macini = c;
 342                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 343                     c->cri_alg == CRYPTO_3DES_CBC ||
 344                     c->cri_alg == CRYPTO_AES_CBC ||
 345                     c->cri_alg == CRYPTO_NULL_CBC) {
 346                         if (encini)
 347                                 return EINVAL;
 348                         encini = c;
 349                 } else {
 350                         DPRINTF("UNKNOWN c->cri_alg %d\n", encini->cri_alg);
 351                         return EINVAL;
 352                 }
 353         }
 354         if (encini == NULL && macini == NULL)
 355                 return EINVAL;
 356         if (encini) {
 357                 /* validate key length */
 358                 switch (encini->cri_alg) {
 359                 case CRYPTO_DES_CBC:
 360                         if (encini->cri_klen != 64)
 361                                 return EINVAL;
 362                         break;
 363                 case CRYPTO_3DES_CBC:
 364                         if (encini->cri_klen != 192) {
 365                                 return EINVAL;
 366                         }
 367                         break;
 368                 case CRYPTO_AES_CBC:
 369                         if (encini->cri_klen != 128 &&
 370                             encini->cri_klen != 192 &&
 371                             encini->cri_klen != 256)
 372                                 return EINVAL;
 373                         break;
 374                 default:
 375                         DPRINTF("UNKNOWN encini->cri_alg %d\n",
 376                                 encini->cri_alg);
 377                         return EINVAL;
 378                 }
 379         }
 380
 381         if (sc->sc_sessions == NULL) {
 382                 ses = sc->sc_sessions = (struct talitos_session *)
 383                         kmalloc(sizeof(struct talitos_session), SLAB_ATOMIC);
 384                 if (ses == NULL)
 385                         return ENOMEM;
 386                 memset(ses, 0, sizeof(struct talitos_session));
 387                 sesn = 0;
 388                 sc->sc_nsessions = 1;
 389         } else {
 390                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 391                         if (sc->sc_sessions[sesn].ses_used == 0) {
 392                                 ses = &sc->sc_sessions[sesn];
 393                                 break;
 394                         }
 395                 }
 396
 397                 if (ses == NULL) {
 398                         /* allocating session */
 399                         sesn = sc->sc_nsessions;
 400                         ses = (struct talitos_session *) kmalloc(
 401                                 (sesn + 1) * sizeof(struct talitos_session),
 402                                 SLAB_ATOMIC);
 403                         if (ses == NULL)
 404                                 return ENOMEM;
 405                         memset(ses, 0,
 406                                 (sesn + 1) * sizeof(struct talitos_session));
 407                         memcpy(ses, sc->sc_sessions,
 408                                 sesn * sizeof(struct talitos_session));
 409                         memset(sc->sc_sessions, 0,
 410                                 sesn * sizeof(struct talitos_session));
 411                         kfree(sc->sc_sessions);
 412                         sc->sc_sessions = ses;
 413                         ses = &sc->sc_sessions[sesn];
 414                         sc->sc_nsessions++;
 415                 }
 416         }
 417
 418         ses->ses_used = 1;
 419
 420         if (encini) {
 421                 ses->ses_klen = (encini->cri_klen + 7) / 8;
 422                 memcpy(ses->ses_key, encini->cri_key, ses->ses_klen);
 423                 if (macini) {
 424                         /* doing hash on top of cipher */
 425                         ses->ses_hmac_len = (macini->cri_klen + 7) / 8;
 426                         memcpy(ses->ses_hmac, macini->cri_key,
 427                                 ses->ses_hmac_len);
 428                 }
 429         } else if (macini) {
 430                 /* doing hash */
 431                 ses->ses_klen = (macini->cri_klen + 7) / 8;
 432                 memcpy(ses->ses_key, macini->cri_key, ses->ses_klen);
 433         }
 434
 435         /* back compat way of determining MSC result len */
 436         if (macini) {
 437                 ses->ses_mlen = macini->cri_mlen;
 438                 if (ses->ses_mlen == 0) {
 439                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 440                                 ses->ses_mlen = MD5_HASH_LEN;
 441                         else
 442                                 ses->ses_mlen = SHA1_HASH_LEN;
 443                 }
 444         }
 445
 446         /* really should make up a template td here,
 447          * and only fill things like i/o and direction in process() */
 448
 449         /* assign session ID */
 450         *sidp = TALITOS_SID(sc->sc_num, sesn);
 451         return 0;
 452 }
 453
 454 /*
 455  * Deallocate a session.
 456  */
 457 static int
 458 talitos_freesession(device_t dev, u_int64_t tid)
 459 {
 460         struct talitos_softc *sc = device_get_softc(dev);
 461         int session, ret;
 462         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 463
 464         if (sc == NULL)
 465                 return EINVAL;
 466         session = TALITOS_SESSION(sid);
 467         if (session < sc->sc_nsessions) {
 468                 memset(&sc->sc_sessions[session], 0,
 469                         sizeof(sc->sc_sessions[session]));
 470                 ret = 0;
 471         } else
 472                 ret = EINVAL;
 473         return ret;
 474 }
 475
 476 /*
 477  * launch device processing - it will come back with done notification
 478  * in the form of an interrupt and/or HDR_DONE_BITS in header
 479  */
 480 static int
 481 talitos_submit(
 482         struct talitos_softc *sc,
 483         struct talitos_desc *td,
 484         int chsel)
 485 {
 486         u_int32_t v;
 487
 488         v = dma_map_single(NULL, td, sizeof(*td), DMA_TO_DEVICE);
 489         talitos_write(sc->sc_base_addr +
 490                 chsel*TALITOS_CH_OFFSET + TALITOS_CH_FF, 0);
 491         talitos_write(sc->sc_base_addr +
 492                 chsel*TALITOS_CH_OFFSET + TALITOS_CH_FF_HI, v);
 493         return 0;
 494 }
 495
 496 static int
 497 talitos_process(device_t dev, struct cryptop *crp, int hint)
 498 {
 499         int i, err = 0, ivsize;
 500         struct talitos_softc *sc = device_get_softc(dev);
 501         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 502         caddr_t iv;
 503         struct talitos_session *ses;
 504         struct talitos_desc *td;
 505         unsigned long flags;
 506         /* descriptor mappings */
 507         int hmac_key, hmac_data, cipher_iv, cipher_key,
 508                 in_fifo, out_fifo, cipher_iv_out;
 509         static int chsel = -1;
 510         u_int32_t rand_iv[4];
 511
 512         DPRINTF("%s()\n", __FUNCTION__);
 513
 514         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 515                 return EINVAL;
 516         }
 517         crp->crp_etype = 0;
 518         if (TALITOS_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 519                 return EINVAL;
 520         }
 521
 522         ses = &sc->sc_sessions[TALITOS_SESSION(crp->crp_sid)];
 523
 524         /* enter the channel scheduler */
 525         spin_lock_irqsave(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 526
 527         /* reuse channel that already had/has requests for the required EU */
 528         for (i = 0; i < sc->sc_num_channels; i++) {
 529                 if (sc->sc_chnlastalg[i] == crp->crp_desc->crd_alg)
 530                         break;
 531         }
 532         if (i == sc->sc_num_channels) {
 533                 /*
 534                  * haven't seen this algo the last sc_num_channels or more
 535                  * use round robin in this case
 536                  * nb: sc->sc_num_channels must be power of 2
 537                  */
 538                 chsel = (chsel + 1) & (sc->sc_num_channels - 1);
 539         } else {
 540                 /*
 541                  * matches channel with same target execution unit;
 542                  * use same channel in this case
 543                  */
 544                 chsel = i;
 545         }
 546         sc->sc_chnlastalg[chsel] = crp->crp_desc->crd_alg;
 547
 548         /* release the channel scheduler lock */
 549         spin_unlock_irqrestore(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 550
 551         /* acquire the selected channel fifo lock */
 552         spin_lock_irqsave(&sc->sc_chnfifolock[chsel], flags);
 553
 554         /* find and reserve next available descriptor-cryptop pair */
 555         for (i = 0; i < sc->sc_chfifo_len; i++) {
 556                 if (sc->sc_chnfifo[chsel][i].cf_desc.hdr == 0) {
 557                         /*
 558                          * ensure correct descriptor formation by
 559                          * avoiding inadvertently setting "optional" entries
 560                          * e.g. not using "optional" dptr2 for MD/HMAC descs
 561                          */
 562                         memset(&sc->sc_chnfifo[chsel][i].cf_desc,
 563                                 0, sizeof(*td));
 564                         /* reserve it with done notification request bit */
 565                         sc->sc_chnfifo[chsel][i].cf_desc.hdr |=
 566                                 TALITOS_DONE_NOTIFY;
 567                         break;
 568                 }
 569         }
 570         spin_unlock_irqrestore(&sc->sc_chnfifolock[chsel], flags);
 571
 572         if (i == sc->sc_chfifo_len) {
 573                 /* fifo full */
 574                 err = ERESTART;
 575                 goto errout;
 576         }
 577
 578         td = &sc->sc_chnfifo[chsel][i].cf_desc;
 579         sc->sc_chnfifo[chsel][i].cf_crp = crp;
 580
 581         crd1 = crp->crp_desc;
 582         if (crd1 == NULL) {
 583                 err = EINVAL;
 584                 goto errout;
 585         }
 586         crd2 = crd1->crd_next;
 587         /* prevent compiler warning */
 588         hmac_key = 0;
 589         hmac_data = 0;
 590         if (crd2 == NULL) {
 591                 td->hdr |= TD_TYPE_COMMON_NONSNOOP_NO_AFEU;
 592                 /* assign descriptor dword ptr mappings for this desc. type */
 593                 cipher_iv = 1;
 594                 cipher_key = 2;
 595                 in_fifo = 3;
 596                 cipher_iv_out = 5;
 597                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 598                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 599                     crd1->crd_alg == CRYPTO_SHA1 ||
 600                     crd1->crd_alg == CRYPTO_MD5) {
 601                         out_fifo = 5;
 602                         maccrd = crd1;
 603                         enccrd = NULL;
 604                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 605                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 606                     crd1->crd_alg == CRYPTO_AES_CBC ||
 607                     crd1->crd_alg == CRYPTO_ARC4) {
 608                         out_fifo = 4;
 609                         maccrd = NULL;
 610                         enccrd = crd1;
 611                 } else {
 612                         DPRINTF("UNKNOWN crd1->crd_alg %d\n", crd1->crd_alg);
 613                         err = EINVAL;
 614                         goto errout;
 615                 }
 616         } else {
 617                 if (sc->sc_desc_types & TALITOS_HAS_DT_IPSEC_ESP) {
 618                         td->hdr |= TD_TYPE_IPSEC_ESP;
 619                 } else {
 620                         DPRINTF("unimplemented: multiple descriptor ipsec\n");
 621                         err = EINVAL;
 622                         goto errout;
 623                 }
 624                 /* assign descriptor dword ptr mappings for this desc. type */
 625                 hmac_key = 0;
 626                 hmac_data = 1;
 627                 cipher_iv = 2;
 628                 cipher_key = 3;
 629                 in_fifo = 4;
 630                 out_fifo = 5;
 631                 cipher_iv_out = 6;
 632                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 633                      crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 634                      crd1->crd_alg == CRYPTO_MD5 ||
 635                      crd1->crd_alg == CRYPTO_SHA1) &&
 636                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 637                      crd2->crd_alg == CRYPTO_3DES_CBC ||
 638                      crd2->crd_alg == CRYPTO_AES_CBC ||
 639                      crd2->crd_alg == CRYPTO_ARC4) &&
 640                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 641                         maccrd = crd1;
 642                         enccrd = crd2;
 643                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 644                      crd1->crd_alg == CRYPTO_ARC4 ||
 645                      crd1->crd_alg == CRYPTO_3DES_CBC ||
 646                      crd1->crd_alg == CRYPTO_AES_CBC) &&
 647                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 648                      crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 649                      crd2->crd_alg == CRYPTO_MD5 ||
 650                      crd2->crd_alg == CRYPTO_SHA1) &&
 651                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 652                         enccrd = crd1;
 653                         maccrd = crd2;
 654                 } else {
 655                         /* We cannot order the SEC as requested */
 656                         printk("%s: cannot do the order\n",
 657                                         device_get_nameunit(sc->sc_cdev));
 658                         err = EINVAL;
 659                         goto errout;
 660                 }
 661         }
 662         /* assign in_fifo and out_fifo based on input/output struct type */
 663         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 664                 /* using SKB buffers */
 665                 struct sk_buff *skb = (struct sk_buff *)crp->crp_buf;
 666                 if (skb_shinfo(skb)->nr_frags) {
 667                         printk("%s: skb frags unimplemented\n",
 668                                         device_get_nameunit(sc->sc_cdev));
 669                         err = EINVAL;
 670                         goto errout;
 671                 }
 672                 td->ptr[in_fifo].ptr = dma_map_single(NULL, skb->data,
 673                         skb->len, DMA_TO_DEVICE);
 674                 td->ptr[in_fifo].len = skb->len;
 675                 td->ptr[out_fifo].ptr = dma_map_single(NULL, skb->data,
 676                         skb->len, DMA_TO_DEVICE);
 677                 td->ptr[out_fifo].len = skb->len;
 678                 td->ptr[hmac_data].ptr = dma_map_single(NULL, skb->data,
 679                         skb->len, DMA_TO_DEVICE);
 680         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 681                 /* using IOV buffers */
 682                 struct uio *uiop = (struct uio *)crp->crp_buf;
 683                 if (uiop->uio_iovcnt > 1) {
 684                         printk("%s: iov frags unimplemented\n",
 685                                         device_get_nameunit(sc->sc_cdev));
 686                         err = EINVAL;
 687                         goto errout;
 688                 }
 689                 td->ptr[in_fifo].ptr = dma_map_single(NULL,
 690                         uiop->uio_iov->iov_base, crp->crp_ilen, DMA_TO_DEVICE);
 691                 td->ptr[in_fifo].len = crp->crp_ilen;
 692                 /* crp_olen is never set; always use crp_ilen */
 693                 td->ptr[out_fifo].ptr = dma_map_single(NULL,
 694                         uiop->uio_iov->iov_base,
 695                         crp->crp_ilen, DMA_TO_DEVICE);
 696                 td->ptr[out_fifo].len = crp->crp_ilen;
 697         } else {
 698                 /* using contig buffers */
 699                 td->ptr[in_fifo].ptr = dma_map_single(NULL,
 700                         crp->crp_buf, crp->crp_ilen, DMA_TO_DEVICE);
 701                 td->ptr[in_fifo].len = crp->crp_ilen;
 702                 td->ptr[out_fifo].ptr = dma_map_single(NULL,
 703                         crp->crp_buf, crp->crp_ilen, DMA_TO_DEVICE);
 704                 td->ptr[out_fifo].len = crp->crp_ilen;
 705         }
 706         if (enccrd) {
 707                 switch (enccrd->crd_alg) {
 708                 case CRYPTO_3DES_CBC:
 709                         td->hdr |= TALITOS_MODE0_DEU_3DES;
 710                         /* FALLTHROUGH */
 711                 case CRYPTO_DES_CBC:
 712                         td->hdr |= TALITOS_SEL0_DEU
 713                                 |  TALITOS_MODE0_DEU_CBC;
 714                         if (enccrd->crd_flags & CRD_F_ENCRYPT)
 715                                 td->hdr |= TALITOS_MODE0_DEU_ENC;
 716                         ivsize = 2*sizeof(u_int32_t);
 717                         DPRINTF("%cDES ses %d ch %d len %d\n",
 718                                 (td->hdr & TALITOS_MODE0_DEU_3DES)?'3':'1',
 719                                 (u32)TALITOS_SESSION(crp->crp_sid),
 720                                 chsel, td->ptr[in_fifo].len);
 721                         break;
 722                 case CRYPTO_AES_CBC:
 723                         td->hdr |= TALITOS_SEL0_AESU
 724                                 |  TALITOS_MODE0_AESU_CBC;
 725                         if (enccrd->crd_flags & CRD_F_ENCRYPT)
 726                                 td->hdr |= TALITOS_MODE0_AESU_ENC;
 727                         ivsize = 4*sizeof(u_int32_t);
 728                         DPRINTF("AES  ses %d ch %d len %d\n",
 729                                 (u32)TALITOS_SESSION(crp->crp_sid),
 730                                 chsel, td->ptr[in_fifo].len);
 731                         break;
 732                 default:
 733                         printk("%s: unimplemented enccrd->crd_alg %d\n",
 734                                         device_get_nameunit(sc->sc_cdev), enccrd->crd_alg);
 735                         err = EINVAL;
 736                         goto errout;
 737                 }
 738                 /*
 739                  * Setup encrypt/decrypt state.  When using basic ops
 740                  * we can't use an inline IV because hash/crypt offset
 741                  * must be from the end of the IV to the start of the
 742                  * crypt data and this leaves out the preceding header
 743                  * from the hash calculation.  Instead we place the IV
 744                  * in the state record and set the hash/crypt offset to
 745                  * copy both the header+IV.
 746                  */
 747                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 748                         td->hdr |= TALITOS_DIR_OUTBOUND;
 749                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 750                                 iv = enccrd->crd_iv;
 751                         else
 752                                 read_random((iv = (caddr_t) rand_iv), sizeof(rand_iv));
 753                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 754                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
 755                                     enccrd->crd_inject, ivsize, iv);
 756                         }
 757                 } else {
 758                         td->hdr |= TALITOS_DIR_INBOUND;
 759                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 760                                 iv = enccrd->crd_iv;
 761                         } else {
 762                                 iv = (caddr_t) rand_iv;
 763                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 764                                     enccrd->crd_inject, ivsize, iv);
 765                         }
 766                 }
 767                 td->ptr[cipher_iv].ptr = dma_map_single(NULL, iv, ivsize,
 768                         DMA_TO_DEVICE);
 769                 td->ptr[cipher_iv].len = ivsize;
 770                 /*
 771                  * we don't need the cipher iv out length/pointer
 772                  * field to do ESP IPsec. Therefore we set the len field as 0,
 773                  * which tells the SEC not to do anything with this len/ptr
 774                  * field. Previously, when length/pointer as pointing to iv,
 775                  * it gave us corruption of packets.
 776                  */
 777                 td->ptr[cipher_iv_out].len = 0;
 778         }
 779         if (enccrd && maccrd) {
 780                 /* this is ipsec only for now */
 781                 td->hdr |= TALITOS_SEL1_MDEU
 782                         |  TALITOS_MODE1_MDEU_INIT
 783                         |  TALITOS_MODE1_MDEU_PAD;
 784                 switch (maccrd->crd_alg) {
 785                         case    CRYPTO_MD5:
 786                                 td->hdr |= TALITOS_MODE1_MDEU_MD5;
 787                                 break;
 788                         case    CRYPTO_MD5_HMAC:
 789                                 td->hdr |= TALITOS_MODE1_MDEU_MD5_HMAC;
 790                                 break;
 791                         case    CRYPTO_SHA1:
 792                                 td->hdr |= TALITOS_MODE1_MDEU_SHA1;
 793                                 break;
 794                         case    CRYPTO_SHA1_HMAC:
 795                                 td->hdr |= TALITOS_MODE1_MDEU_SHA1_HMAC;
 796                                 break;
 797                         default:
 798                                 /* We cannot order the SEC as requested */
 799                                 printk("%s: cannot do the order\n",
 800                                                 device_get_nameunit(sc->sc_cdev));
 801                                 err = EINVAL;
 802                                 goto errout;
 803                 }
 804                 if ((maccrd->crd_alg == CRYPTO_MD5_HMAC) ||
 805                    (maccrd->crd_alg == CRYPTO_SHA1_HMAC)) {
 806                         /*
 807                          * The offset from hash data to the start of
 808                          * crypt data is the difference in the skips.
 809                          */
 810                         /* ipsec only for now */
 811                         td->ptr[hmac_key].ptr = dma_map_single(NULL,
 812                                 ses->ses_hmac, ses->ses_hmac_len, DMA_TO_DEVICE);
 813                         td->ptr[hmac_key].len = ses->ses_hmac_len;
 814                         td->ptr[in_fifo].ptr  += enccrd->crd_skip;
 815                         td->ptr[in_fifo].len  =  enccrd->crd_len;
 816                         td->ptr[out_fifo].ptr += enccrd->crd_skip;
 817                         td->ptr[out_fifo].len =  enccrd->crd_len;
 818                         /* bytes of HMAC to postpend to ciphertext */
 819                         td->ptr[out_fifo].extent =  ses->ses_mlen;
 820                         td->ptr[hmac_data].ptr += maccrd->crd_skip;
 821                         td->ptr[hmac_data].len = enccrd->crd_skip - maccrd->crd_skip;
 822                 }
 823                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 824                         printk("%s: CRD_F_KEY_EXPLICIT unimplemented\n",
 825                                         device_get_nameunit(sc->sc_cdev));
 826                 }
 827         }
 828         if (!enccrd && maccrd) {
 829                 /* single MD5 or SHA */
 830                 td->hdr |= TALITOS_SEL0_MDEU
 831                                 |  TALITOS_MODE0_MDEU_INIT
 832                                 |  TALITOS_MODE0_MDEU_PAD;
 833                 switch (maccrd->crd_alg) {
 834                         case    CRYPTO_MD5:
 835                                 td->hdr |= TALITOS_MODE0_MDEU_MD5;
 836                                 DPRINTF("MD5  ses %d ch %d len %d\n",
 837                                         (u32)TALITOS_SESSION(crp->crp_sid),
 838                                         chsel, td->ptr[in_fifo].len);
 839                                 break;
 840                         case    CRYPTO_MD5_HMAC:
 841                                 td->hdr |= TALITOS_MODE0_MDEU_MD5_HMAC;
 842                                 break;
 843                         case    CRYPTO_SHA1:
 844                                 td->hdr |= TALITOS_MODE0_MDEU_SHA1;
 845                                 DPRINTF("SHA1 ses %d ch %d len %d\n",
 846                                         (u32)TALITOS_SESSION(crp->crp_sid),
 847                                         chsel, td->ptr[in_fifo].len);
 848                                 break;
 849                         case    CRYPTO_SHA1_HMAC:
 850                                 td->hdr |= TALITOS_MODE0_MDEU_SHA1_HMAC;
 851                                 break;
 852                         default:
 853                                 /* We cannot order the SEC as requested */
 854                                 DPRINTF("cannot do the order\n");
 855                                 err = EINVAL;
 856                                 goto errout;
 857                 }
 858
 859                 if (crp->crp_flags & CRYPTO_F_IOV)
 860                         td->ptr[out_fifo].ptr += maccrd->crd_inject;
 861
 862                 if ((maccrd->crd_alg == CRYPTO_MD5_HMAC) ||
 863                    (maccrd->crd_alg == CRYPTO_SHA1_HMAC)) {
 864                         td->ptr[hmac_key].ptr = dma_map_single(NULL,
 865                                 ses->ses_hmac, ses->ses_hmac_len,
 866                                 DMA_TO_DEVICE);
 867                         td->ptr[hmac_key].len = ses->ses_hmac_len;
 868                 }
 869         }
 870         else {
 871                 /* using process key (session data has duplicate) */
 872                 td->ptr[cipher_key].ptr = dma_map_single(NULL,
 873                         enccrd->crd_key, (enccrd->crd_klen + 7) / 8,
 874                         DMA_TO_DEVICE);
 875                 td->ptr[cipher_key].len = (enccrd->crd_klen + 7) / 8;
 876         }
 877         /* descriptor complete - GO! */
 878         return talitos_submit(sc, td, chsel);
 879
 880 errout:
 881         if (err != ERESTART) {
 882                 crp->crp_etype = err;
 883                 crypto_done(crp);
 884         }
 885         return err;
 886 }
 887
 888 /* go through all channels descriptors, notifying OCF what has
 889  * _and_hasn't_ successfully completed and reset the device
 890  * (otherwise it's up to decoding desc hdrs!)
 891  */
 892 static void talitos_errorprocessing(struct talitos_softc *sc)
 893 {
 894         unsigned long flags;
 895         int i, j;
 896
 897         /* disable further scheduling until under control */
 898         spin_lock_irqsave(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 899
 900         if (debug) dump_talitos_status(sc);
 901         /* go through descriptors, try and salvage those successfully done,
 902          * and EIO those that weren't
 903          */
 904         for (i = 0; i < sc->sc_num_channels; i++) {
 905                 spin_lock_irqsave(&sc->sc_chnfifolock[i], flags);
 906                 for (j = 0; j < sc->sc_chfifo_len; j++) {
 907                         if (sc->sc_chnfifo[i][j].cf_desc.hdr) {
 908                                 if ((sc->sc_chnfifo[i][j].cf_desc.hdr
 909                                         & TALITOS_HDR_DONE_BITS)
 910                                         != TALITOS_HDR_DONE_BITS) {
 911                                         /* this one didn't finish */
 912                                         /* signify in crp->etype */
 913                                         sc->sc_chnfifo[i][j].cf_crp->crp_etype
 914                                                 = EIO;
 915                                 }
 916                         } else
 917                                 continue; /* free entry */
 918                         /* either way, notify ocf */
 919                         crypto_done(sc->sc_chnfifo[i][j].cf_crp);
 920                         /* and tag it available again
 921                          *
 922                          * memset to ensure correct descriptor formation by
 923                          * avoiding inadvertently setting "optional" entries
 924                          * e.g. not using "optional" dptr2 MD/HMAC processing
 925                          */
 926                         memset(&sc->sc_chnfifo[i][j].cf_desc,
 927                                 0, sizeof(struct talitos_desc));
 928                 }
 929                 spin_unlock_irqrestore(&sc->sc_chnfifolock[i], flags);
 930         }
 931         /* reset and initialize the SEC h/w device */
 932         talitos_reset_device(sc);
 933         talitos_init_device(sc);
 934 #ifdef CONFIG_OCF_RANDOMHARVEST
 935         if (sc->sc_exec_units & TALITOS_HAS_EU_RNG)
 936                 talitos_rng_init(sc);
 937 #endif
 938
 939         /* Okay. Stand by. */
 940         spin_unlock_irqrestore(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 941
 942         return;
 943 }
 944
 945 /* go through all channels descriptors, notifying OCF what's been done */
 946 static void talitos_doneprocessing(struct talitos_softc *sc)
 947 {
 948         unsigned long flags;
 949         int i, j;
 950
 951         /* go through descriptors looking for done bits */
 952         for (i = 0; i < sc->sc_num_channels; i++) {
 953                 spin_lock_irqsave(&sc->sc_chnfifolock[i], flags);
 954                 for (j = 0; j < sc->sc_chfifo_len; j++) {
 955                         /* descriptor has done bits set? */
 956                         if ((sc->sc_chnfifo[i][j].cf_desc.hdr
 957                                 & TALITOS_HDR_DONE_BITS)
 958                                 == TALITOS_HDR_DONE_BITS) {
 959                                 /* notify ocf */
 960                                 crypto_done(sc->sc_chnfifo[i][j].cf_crp);
 961                                 /* and tag it available again
 962                                  *
 963                                  * memset to ensure correct descriptor formation by
 964                                  * avoiding inadvertently setting "optional" entries
 965                                  * e.g. not using "optional" dptr2 MD/HMAC processing
 966                                  */
 967                                 memset(&sc->sc_chnfifo[i][j].cf_desc,
 968                                         0, sizeof(struct talitos_desc));
 969                         }
 970                 }
 971                 spin_unlock_irqrestore(&sc->sc_chnfifolock[i], flags);
 972         }
 973         return;
 974 }
 975
 976 static irqreturn_t
 977 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 978 talitos_intr(int irq, void *arg)
 979 #else
 980 talitos_intr(int irq, void *arg, struct pt_regs *regs)
 981 #endif
 982 {
 983         struct talitos_softc *sc = arg;
 984         u_int32_t v, v_hi;
 985
 986         /* ack */
 987         v = talitos_read(sc->sc_base_addr + TALITOS_ISR);
 988         v_hi = talitos_read(sc->sc_base_addr + TALITOS_ISR_HI);
 989         talitos_write(sc->sc_base_addr + TALITOS_ICR, v);
 990         talitos_write(sc->sc_base_addr + TALITOS_ICR_HI, v_hi);
 991
 992         if (unlikely(v & TALITOS_ISR_ERROR)) {
 993                 /* Okay, Houston, we've had a problem here. */
 994                 printk(KERN_DEBUG "%s: got error interrupt - ISR 0x%08x_%08x\n",
 995                                 device_get_nameunit(sc->sc_cdev), v, v_hi);
 996                 talitos_errorprocessing(sc);
 997         } else
 998         if (likely(v & TALITOS_ISR_DONE)) {
 999                 talitos_doneprocessing(sc);
1000         }
1001         return IRQ_HANDLED;
1002 }
1003
1004 /*
1005  * Initialize registers we need to touch only once.
1006  */
1007 static void
1008 talitos_init_device(struct talitos_softc *sc)
1009 {
1010         u_int32_t v;
1011         int i;
1012
1013         DPRINTF("%s()\n", __FUNCTION__);
1014
1015         /* init all channels */
1016         for (i = 0; i < sc->sc_num_channels; i++) {
1017                 v = talitos_read(sc->sc_base_addr +
1018                         i*TALITOS_CH_OFFSET + TALITOS_CH_CCCR_HI);
1019                 v |= TALITOS_CH_CCCR_HI_CDWE
1020                   |  TALITOS_CH_CCCR_HI_CDIE;  /* invoke interrupt if done */
1021                 talitos_write(sc->sc_base_addr +
1022                         i*TALITOS_CH_OFFSET + TALITOS_CH_CCCR_HI, v);
1023         }
1024         /* enable all interrupts */
1025         v = talitos_read(sc->sc_base_addr + TALITOS_IMR);
1026         v |= TALITOS_IMR_ALL;
1027         talitos_write(sc->sc_base_addr + TALITOS_IMR, v);
1028         v = talitos_read(sc->sc_base_addr + TALITOS_IMR_HI);
1029         v |= TALITOS_IMR_HI_ERRONLY;
1030         talitos_write(sc->sc_base_addr + TALITOS_IMR_HI, v);
1031         return;
1032 }
1033
1034 /*
1035  * set the master reset bit on the device.
1036  */
1037 static void
1038 talitos_reset_device_master(struct talitos_softc *sc)
1039 {
1040         u_int32_t v;
1041
1042         /* Reset the device by writing 1 to MCR:SWR and waiting 'til cleared */
1043         v = talitos_read(sc->sc_base_addr + TALITOS_MCR);
1044         talitos_write(sc->sc_base_addr + TALITOS_MCR, v | TALITOS_MCR_SWR);
1045
1046         while (talitos_read(sc->sc_base_addr + TALITOS_MCR) & TALITOS_MCR_SWR)
1047                 cpu_relax();
1048
1049         return;
1050 }
1051
1052 /*
1053  * Resets the device.  Values in the registers are left as is
1054  * from the reset (i.e. initial values are assigned elsewhere).
1055  */
1056 static void
1057 talitos_reset_device(struct talitos_softc *sc)
1058 {
1059         u_int32_t v;
1060         int i;
1061
1062         DPRINTF("%s()\n", __FUNCTION__);
1063
1064         /*
1065          * Master reset
1066          * errata documentation: warning: certain SEC interrupts
1067          * are not fully cleared by writing the MCR:SWR bit,
1068          * set bit twice to completely reset
1069          */
1070         talitos_reset_device_master(sc);        /* once */
1071         talitos_reset_device_master(sc);        /* and once again */
1072
1073         /* reset all channels */
1074         for (i = 0; i < sc->sc_num_channels; i++) {
1075                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
1076                         TALITOS_CH_CCCR);
1077                 talitos_write(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
1078                         TALITOS_CH_CCCR, v | TALITOS_CH_CCCR_RESET);
1079         }
1080 }
1081
1082 /* Set up the crypto device structure, private data,
1083  * and anything else we need before we start */
1084 #ifdef CONFIG_PPC_MERGE
1085 static int talitos_probe(struct of_device *ofdev, const struct of_device_id *match)
1086 #else
1087 static int talitos_probe(struct platform_device *pdev)
1088 #endif
1089 {
1090         struct talitos_softc *sc = NULL;
1091         struct resource *r;
1092 #ifdef CONFIG_PPC_MERGE
1093         struct device *device = &ofdev->dev;
1094         struct device_node *np = ofdev->node;
1095         const unsigned int *prop;
1096         int err;
1097         struct resource res;
1098 #endif
1099         static int num_chips = 0;
1100         int rc;
1101         int i;
1102
1103         DPRINTF("%s()\n", __FUNCTION__);
1104
1105         sc = (struct talitos_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
1106         if (!sc)
1107                 return -ENOMEM;
1108         memset(sc, 0, sizeof(*sc));
1109
1110         softc_device_init(sc, DRV_NAME, num_chips, talitos_methods);
1111
1112         sc->sc_irq = -1;
1113         sc->sc_cid = -1;
1114 #ifndef CONFIG_PPC_MERGE
1115         sc->sc_dev = pdev;
1116 #endif
1117         sc->sc_num = num_chips++;
1118
1119 #ifdef CONFIG_PPC_MERGE
1120         dev_set_drvdata(device, sc);
1121 #else
1122         platform_set_drvdata(sc->sc_dev, sc);
1123 #endif
1124
1125         /* get the irq line */
1126 #ifdef CONFIG_PPC_MERGE
1127         err = of_address_to_resource(np, 0, &res);
1128         if (err)
1129                 return -EINVAL;
1130         r = &res;
1131
1132         sc->sc_irq = irq_of_parse_and_map(np, 0);
1133 #else
1134         /* get a pointer to the register memory */
1135         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1136
1137         sc->sc_irq = platform_get_irq(pdev, 0);
1138 #endif
1139         rc = request_irq(sc->sc_irq, talitos_intr, 0,
1140                         device_get_nameunit(sc->sc_cdev), sc);
1141         if (rc) {
1142                 printk(KERN_ERR "%s: failed to hook irq %d\n",
1143                                 device_get_nameunit(sc->sc_cdev), sc->sc_irq);
1144                 sc->sc_irq = -1;
1145                 goto out;
1146         }
1147
1148         sc->sc_base_addr = (ocf_iomem_t) ioremap(r->start, (r->end - r->start));
1149         if (!sc->sc_base_addr) {
1150                 printk(KERN_ERR "%s: failed to ioremap\n",
1151                                 device_get_nameunit(sc->sc_cdev));
1152                 goto out;
1153         }
1154
1155         /* figure out our SEC's properties and capabilities */
1156         sc->sc_chiprev = (u64)talitos_read(sc->sc_base_addr + TALITOS_ID) << 32
1157                  | talitos_read(sc->sc_base_addr + TALITOS_ID_HI);
1158         DPRINTF("sec id 0x%llx\n", sc->sc_chiprev);
1159
1160 #ifdef CONFIG_PPC_MERGE
1161         /* get SEC properties from device tree, defaulting to SEC 2.0 */
1162
1163         prop = of_get_property(np, "num-channels", NULL);
1164         sc->sc_num_channels = prop ? *prop : TALITOS_NCHANNELS_SEC_2_0;
1165
1166         prop = of_get_property(np, "channel-fifo-len", NULL);
1167         sc->sc_chfifo_len = prop ? *prop : TALITOS_CHFIFOLEN_SEC_2_0;
1168
1169         prop = of_get_property(np, "exec-units-mask", NULL);
1170         sc->sc_exec_units = prop ? *prop : TALITOS_HAS_EUS_SEC_2_0;
1171
1172         prop = of_get_property(np, "descriptor-types-mask", NULL);
1173         sc->sc_desc_types = prop ? *prop : TALITOS_HAS_DESCTYPES_SEC_2_0;
1174 #else
1175         /* bulk should go away with openfirmware flat device tree support */
1176         if (sc->sc_chiprev & TALITOS_ID_SEC_2_0) {
1177                 sc->sc_num_channels = TALITOS_NCHANNELS_SEC_2_0;
1178                 sc->sc_chfifo_len = TALITOS_CHFIFOLEN_SEC_2_0;
1179                 sc->sc_exec_units = TALITOS_HAS_EUS_SEC_2_0;
1180                 sc->sc_desc_types = TALITOS_HAS_DESCTYPES_SEC_2_0;
1181         } else {
1182                 printk(KERN_ERR "%s: failed to id device\n",
1183                                 device_get_nameunit(sc->sc_cdev));
1184                 goto out;
1185         }
1186 #endif
1187
1188         /* + 1 is for the meta-channel lock used by the channel scheduler */
1189         sc->sc_chnfifolock = (spinlock_t *) kmalloc(
1190                 (sc->sc_num_channels + 1) * sizeof(spinlock_t), GFP_KERNEL);
1191         if (!sc->sc_chnfifolock)
1192                 goto out;
1193         for (i = 0; i < sc->sc_num_channels + 1; i++) {
1194                 spin_lock_init(&sc->sc_chnfifolock[i]);
1195         }
1196
1197         sc->sc_chnlastalg = (int *) kmalloc(
1198                 sc->sc_num_channels * sizeof(int), GFP_KERNEL);
1199         if (!sc->sc_chnlastalg)
1200                 goto out;
1201         memset(sc->sc_chnlastalg, 0, sc->sc_num_channels * sizeof(int));
1202
1203         sc->sc_chnfifo = (struct desc_cryptop_pair **) kmalloc(
1204                 sc->sc_num_channels * sizeof(struct desc_cryptop_pair *),
1205                 GFP_KERNEL);
1206         if (!sc->sc_chnfifo)
1207                 goto out;
1208         for (i = 0; i < sc->sc_num_channels; i++) {
1209                 sc->sc_chnfifo[i] = (struct desc_cryptop_pair *) kmalloc(
1210                         sc->sc_chfifo_len * sizeof(struct desc_cryptop_pair),
1211                         GFP_KERNEL);
1212                 if (!sc->sc_chnfifo[i])
1213                         goto out;
1214                 memset(sc->sc_chnfifo[i], 0,
1215                         sc->sc_chfifo_len * sizeof(struct desc_cryptop_pair));
1216         }
1217
1218         /* reset and initialize the SEC h/w device */
1219         talitos_reset_device(sc);
1220         talitos_init_device(sc);
1221
1222         sc->sc_cid = crypto_get_driverid(softc_get_device(sc),CRYPTOCAP_F_HARDWARE);
1223         if (sc->sc_cid < 0) {
1224                 printk(KERN_ERR "%s: could not get crypto driver id\n",
1225                                 device_get_nameunit(sc->sc_cdev));
1226                 goto out;
1227         }
1228
1229         /* register algorithms with the framework */
1230         printk("%s:", device_get_nameunit(sc->sc_cdev));
1231
1232         if (sc->sc_exec_units & TALITOS_HAS_EU_RNG)  {
1233                 printk(" rng");
1234 #ifdef CONFIG_OCF_RANDOMHARVEST
1235                 talitos_rng_init(sc);
1236                 crypto_rregister(sc->sc_cid, talitos_read_random, sc);
1237 #endif
1238         }
1239         if (sc->sc_exec_units & TALITOS_HAS_EU_DEU) {
1240                 printk(" des/3des");
1241                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
1242                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
1243         }
1244         if (sc->sc_exec_units & TALITOS_HAS_EU_AESU) {
1245                 printk(" aes");
1246                 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
1247         }
1248         if (sc->sc_exec_units & TALITOS_HAS_EU_MDEU) {
1249                 printk(" md5");
1250                 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
1251                 /* HMAC support only with IPsec for now */
1252                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
1253                 printk(" sha1");
1254                 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
1255                 /* HMAC support only with IPsec for now */
1256                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
1257         }
1258         printk("\n");
1259         return 0;
1260
1261 out:
1262 #ifndef CONFIG_PPC_MERGE
1263         talitos_remove(pdev);
1264 #endif
1265         return -ENOMEM;
1266 }
1267
1268 #ifdef CONFIG_PPC_MERGE
1269 static int talitos_remove(struct of_device *ofdev)
1270 #else
1271 static int talitos_remove(struct platform_device *pdev)
1272 #endif
1273 {
1274 #ifdef CONFIG_PPC_MERGE
1275         struct talitos_softc *sc = dev_get_drvdata(&ofdev->dev);
1276 #else
1277         struct talitos_softc *sc = platform_get_drvdata(pdev);
1278 #endif
1279         int i;
1280
1281         DPRINTF("%s()\n", __FUNCTION__);
1282         if (sc->sc_cid >= 0)
1283                 crypto_unregister_all(sc->sc_cid);
1284         if (sc->sc_chnfifo) {
1285                 for (i = 0; i < sc->sc_num_channels; i++)
1286                         if (sc->sc_chnfifo[i])
1287                                 kfree(sc->sc_chnfifo[i]);
1288                 kfree(sc->sc_chnfifo);
1289         }
1290         if (sc->sc_chnlastalg)
1291                 kfree(sc->sc_chnlastalg);
1292         if (sc->sc_chnfifolock)
1293                 kfree(sc->sc_chnfifolock);
1294         if (sc->sc_irq != -1)
1295                 free_irq(sc->sc_irq, sc);
1296         if (sc->sc_base_addr)
1297                 iounmap((void *) sc->sc_base_addr);
1298         kfree(sc);
1299         return 0;
1300 }
1301
1302 #ifdef CONFIG_PPC_MERGE
1303 static struct of_device_id talitos_match[] = {
1304         {
1305                 .type = "crypto",
1306                 .compatible = "talitos",
1307         },
1308         {},
1309 };
1310
1311 MODULE_DEVICE_TABLE(of, talitos_match);
1312
1313 static struct of_platform_driver talitos_driver = {
1314         .name           = DRV_NAME,
1315         .match_table    = talitos_match,
1316         .probe          = talitos_probe,
1317         .remove         = talitos_remove,
1318 };
1319
1320 static int __init talitos_init(void)
1321 {
1322         return of_register_platform_driver(&talitos_driver);
1323 }
1324
1325 static void __exit talitos_exit(void)
1326 {
1327         of_unregister_platform_driver(&talitos_driver);
1328 }
1329 #else
1330 /* Structure for a platform device driver */
1331 static struct platform_driver talitos_driver = {
1332         .probe = talitos_probe,
1333         .remove = talitos_remove,
1334         .driver = {
1335                 .name = "fsl-sec2",
1336         }
1337 };
1338
1339 static int __init talitos_init(void)
1340 {
1341         return platform_driver_register(&talitos_driver);
1342 }
1343
1344 static void __exit talitos_exit(void)
1345 {
1346         platform_driver_unregister(&talitos_driver);
1347 }
1348 #endif
1349
1350 module_init(talitos_init);
1351 module_exit(talitos_exit);
1352
1353 MODULE_LICENSE("Dual BSD/GPL");
1354 MODULE_AUTHOR("kim.phillips@freescale.com");
1355 MODULE_DESCRIPTION("OCF driver for Freescale SEC (talitos)");