target/linux/generic-2.6/files/crypto/ocf/talitos/talitos.c

   1 /*
   2  * crypto/ocf/talitos/talitos.c
   3  *
   4  * An OCF-Linux module that uses Freescale's SEC to do the crypto.
   5  * Based on crypto/ocf/hifn and crypto/ocf/safe OCF drivers
   6  *
   7  * Copyright (c) 2006 Freescale Semiconductor, Inc.
   8  *
   9  * This code written by Kim A. B. Phillips <kim.phillips@freescale.com>
  10  * some code copied from files with the following:
  11  * Copyright (C) 2004-2007 David McCullough <david_mccullough@mcafee.com>
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. The name of the author may not be used to endorse or promote products
  23  *    derived from this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  28  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  34  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * ---------------------------------------------------------------------------
  37  *
  38  * NOTES:
  39  *
  40  * The Freescale SEC (also known as 'talitos') resides on the
  41  * internal bus, and runs asynchronous to the processor core.  It has
  42  * a wide gamut of cryptographic acceleration features, including single-
  43  * pass IPsec (also known as algorithm chaining).  To properly utilize
  44  * all of the SEC's performance enhancing features, further reworking
  45  * of higher level code (framework, applications) will be necessary.
  46  *
  47  * The following table shows which SEC version is present in which devices:
  48  *
  49  * Devices       SEC version
  50  *
  51  * 8272, 8248    SEC 1.0
  52  * 885, 875      SEC 1.2
  53  * 8555E, 8541E  SEC 2.0
  54  * 8349E         SEC 2.01
  55  * 8548E         SEC 2.1
  56  *
  57  * The following table shows the features offered by each SEC version:
  58  *
  59  *                             Max.   chan-
  60  * version  Bus I/F       Clock  nels  DEU AESU AFEU MDEU PKEU RNG KEU
  61  *
  62  * SEC 1.0  internal 64b  100MHz   4     1    1    1    1    1   1   0
  63  * SEC 1.2  internal 32b   66MHz   1     1    1    0    1    0   0   0
  64  * SEC 2.0  internal 64b  166MHz   4     1    1    1    1    1   1   0
  65  * SEC 2.01 internal 64b  166MHz   4     1    1    1    1    1   1   0
  66  * SEC 2.1  internal 64b  333MHz   4     1    1    1    1    1   1   1
  67  *
  68  * Each execution unit in the SEC has two modes of execution; channel and
  69  * slave/debug.  This driver employs the channel infrastructure in the
  70  * device for convenience.  Only the RNG is directly accessed due to the
  71  * convenience of its random fifo pool.  The relationship between the
  72  * channels and execution units is depicted in the following diagram:
  73  *
  74  *    -------   ------------
  75  * ---| ch0 |---|          |
  76  *    -------   |          |
  77  *              |          |------+-------+-------+-------+------------
  78  *    -------   |          |      |       |       |       |           |
  79  * ---| ch1 |---|          |      |       |       |       |           |
  80  *    -------   |          |   ------  ------  ------  ------      ------
  81  *              |controller|   |DEU |  |AESU|  |MDEU|  |PKEU| ...  |RNG |
  82  *    -------   |          |   ------  ------  ------  ------      ------
  83  * ---| ch2 |---|          |      |       |       |       |           |
  84  *    -------   |          |      |       |       |       |           |
  85  *              |          |------+-------+-------+-------+------------
  86  *    -------   |          |
  87  * ---| ch3 |---|          |
  88  *    -------   ------------
  89  *
  90  * Channel ch0 may drive an aes operation to the aes unit (AESU),
  91  * and, at the same time, ch1 may drive a message digest operation
  92  * to the mdeu. Each channel has an input descriptor FIFO, and the
  93  * FIFO can contain, e.g. on the 8541E, up to 24 entries, before a
  94  * a buffer overrun error is triggered. The controller is responsible
  95  * for fetching the data from descriptor pointers, and passing the
  96  * data to the appropriate EUs. The controller also writes the
  97  * cryptographic operation's result to memory. The SEC notifies
  98  * completion by triggering an interrupt and/or setting the 1st byte
  99  * of the hdr field to 0xff.
 100  *
 101  * TODO:
 102  * o support more algorithms
 103  * o support more versions of the SEC
 104  * o add support for linux 2.4
 105  * o scatter-gather (sg) support
 106  * o add support for public key ops (PKEU)
 107  * o add statistics
 108  */
 109
 110 #ifndef AUTOCONF_INCLUDED
 111 #include <linux/config.h>
 112 #endif
 113 #include <linux/module.h>
 114 #include <linux/init.h>
 115 #include <linux/interrupt.h>
 116 #include <linux/spinlock.h>
 117 #include <linux/random.h>
 118 #include <linux/skbuff.h>
 119 #include <asm/scatterlist.h>
 120 #include <linux/dma-mapping.h>  /* dma_map_single() */
 121 #include <linux/moduleparam.h>
 122
 123 #include <linux/version.h>
 124 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15)
 125 #include <linux/platform_device.h>
 126 #endif
 127
 128 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 129 #include <linux/of_platform.h>
 130 #endif
 131
 132 #include <cryptodev.h>
 133 #include <uio.h>
 134
 135 #define DRV_NAME "talitos"
 136
 137 #include "talitos_dev.h"
 138 #include "talitos_soft.h"
 139
 140 #define read_random(p,l) get_random_bytes(p,l)
 141
 142 const char talitos_driver_name[] = "Talitos OCF";
 143 const char talitos_driver_version[] = "0.2";
 144
 145 static int talitos_newsession(device_t dev, u_int32_t *sidp,
 146                                                                 struct cryptoini *cri);
 147 static int talitos_freesession(device_t dev, u_int64_t tid);
 148 static int talitos_process(device_t dev, struct cryptop *crp, int hint);
 149 static void dump_talitos_status(struct talitos_softc *sc);
 150 static int talitos_submit(struct talitos_softc *sc, struct talitos_desc *td,
 151                                                                 int chsel);
 152 static void talitos_doneprocessing(struct talitos_softc *sc);
 153 static void talitos_init_device(struct talitos_softc *sc);
 154 static void talitos_reset_device_master(struct talitos_softc *sc);
 155 static void talitos_reset_device(struct talitos_softc *sc);
 156 static void talitos_errorprocessing(struct talitos_softc *sc);
 157 #ifdef CONFIG_PPC_MERGE
 158 static int talitos_probe(struct of_device *ofdev, const struct of_device_id *match);
 159 static int talitos_remove(struct of_device *ofdev);
 160 #else
 161 static int talitos_probe(struct platform_device *pdev);
 162 static int talitos_remove(struct platform_device *pdev);
 163 #endif
 164 #ifdef CONFIG_OCF_RANDOMHARVEST
 165 static int talitos_read_random(void *arg, u_int32_t *buf, int maxwords);
 166 static void talitos_rng_init(struct talitos_softc *sc);
 167 #endif
 168
 169 static device_method_t talitos_methods = {
 170         /* crypto device methods */
 171         DEVMETHOD(cryptodev_newsession, talitos_newsession),
 172         DEVMETHOD(cryptodev_freesession,talitos_freesession),
 173         DEVMETHOD(cryptodev_process,    talitos_process),
 174 };
 175
 176 #define debug talitos_debug
 177 int talitos_debug = 0;
 178 module_param(talitos_debug, int, 0644);
 179 MODULE_PARM_DESC(talitos_debug, "Enable debug");
 180
 181 static inline void talitos_write(volatile unsigned *addr, u32 val)
 182 {
 183         out_be32(addr, val);
 184 }
 185
 186 static inline u32 talitos_read(volatile unsigned *addr)
 187 {
 188         u32 val;
 189         val = in_be32(addr);
 190         return val;
 191 }
 192
 193 static void dump_talitos_status(struct talitos_softc *sc)
 194 {
 195         unsigned int v, v_hi, i, *ptr;
 196         v = talitos_read(sc->sc_base_addr + TALITOS_MCR);
 197         v_hi = talitos_read(sc->sc_base_addr + TALITOS_MCR_HI);
 198         printk(KERN_INFO "%s: MCR          0x%08x_%08x\n",
 199                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 200         v = talitos_read(sc->sc_base_addr + TALITOS_IMR);
 201         v_hi = talitos_read(sc->sc_base_addr + TALITOS_IMR_HI);
 202         printk(KERN_INFO "%s: IMR          0x%08x_%08x\n",
 203                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 204         v = talitos_read(sc->sc_base_addr + TALITOS_ISR);
 205         v_hi = talitos_read(sc->sc_base_addr + TALITOS_ISR_HI);
 206         printk(KERN_INFO "%s: ISR          0x%08x_%08x\n",
 207                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 208         for (i = 0; i < sc->sc_num_channels; i++) {
 209                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 210                         TALITOS_CH_CDPR);
 211                 v_hi = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 212                         TALITOS_CH_CDPR_HI);
 213                 printk(KERN_INFO "%s: CDPR     ch%d 0x%08x_%08x\n",
 214                                 device_get_nameunit(sc->sc_cdev), i, v, v_hi);
 215         }
 216         for (i = 0; i < sc->sc_num_channels; i++) {
 217                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 218                         TALITOS_CH_CCPSR);
 219                 v_hi = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 220                         TALITOS_CH_CCPSR_HI);
 221                 printk(KERN_INFO "%s: CCPSR    ch%d 0x%08x_%08x\n",
 222                                 device_get_nameunit(sc->sc_cdev), i, v, v_hi);
 223         }
 224         ptr = sc->sc_base_addr + TALITOS_CH_DESCBUF;
 225         for (i = 0; i < 16; i++) {
 226                 v = talitos_read(ptr++); v_hi = talitos_read(ptr++);
 227                 printk(KERN_INFO "%s: DESCBUF  ch0 0x%08x_%08x (tdp%02d)\n",
 228                                 device_get_nameunit(sc->sc_cdev), v, v_hi, i);
 229         }
 230         return;
 231 }
 232
 233
 234 #ifdef CONFIG_OCF_RANDOMHARVEST
 235 /*
 236  * pull random numbers off the RNG FIFO, not exceeding amount available
 237  */
 238 static int
 239 talitos_read_random(void *arg, u_int32_t *buf, int maxwords)
 240 {
 241         struct talitos_softc *sc = (struct talitos_softc *) arg;
 242         int rc;
 243         u_int32_t v;
 244
 245         DPRINTF("%s()\n", __FUNCTION__);
 246
 247         /* check for things like FIFO underflow */
 248         v = talitos_read(sc->sc_base_addr + TALITOS_RNGISR_HI);
 249         if (unlikely(v)) {
 250                 printk(KERN_ERR "%s: RNGISR_HI error %08x\n",
 251                                 device_get_nameunit(sc->sc_cdev), v);
 252                 return 0;
 253         }
 254         /*
 255          * OFL is number of available 64-bit words,
 256          * shift and convert to a 32-bit word count
 257          */
 258         v = talitos_read(sc->sc_base_addr + TALITOS_RNGSR_HI);
 259         v = (v & TALITOS_RNGSR_HI_OFL) >> (16 - 1);
 260         if (maxwords > v)
 261                 maxwords = v;
 262         for (rc = 0; rc < maxwords; rc++) {
 263                 buf[rc] = talitos_read(sc->sc_base_addr +
 264                         TALITOS_RNG_FIFO + rc*sizeof(u_int32_t));
 265         }
 266         if (maxwords & 1) {
 267                 /*
 268                  * RNG will complain with an AE in the RNGISR
 269                  * if we don't complete the pairs of 32-bit reads
 270                  * to its 64-bit register based FIFO
 271                  */
 272                 v = talitos_read(sc->sc_base_addr +
 273                         TALITOS_RNG_FIFO + rc*sizeof(u_int32_t));
 274         }
 275
 276         return rc;
 277 }
 278
 279 static void
 280 talitos_rng_init(struct talitos_softc *sc)
 281 {
 282         u_int32_t v;
 283
 284         DPRINTF("%s()\n", __FUNCTION__);
 285         /* reset RNG EU */
 286         v = talitos_read(sc->sc_base_addr + TALITOS_RNGRCR_HI);
 287         v |= TALITOS_RNGRCR_HI_SR;
 288         talitos_write(sc->sc_base_addr + TALITOS_RNGRCR_HI, v);
 289         while ((talitos_read(sc->sc_base_addr + TALITOS_RNGSR_HI)
 290                 & TALITOS_RNGSR_HI_RD) == 0)
 291                         cpu_relax();
 292         /*
 293          * we tell the RNG to start filling the RNG FIFO
 294          * by writing the RNGDSR
 295          */
 296         v = talitos_read(sc->sc_base_addr + TALITOS_RNGDSR_HI);
 297         talitos_write(sc->sc_base_addr + TALITOS_RNGDSR_HI, v);
 298         /*
 299          * 64 bits of data will be pushed onto the FIFO every
 300          * 256 SEC cycles until the FIFO is full.  The RNG then
 301          * attempts to keep the FIFO full.
 302          */
 303         v = talitos_read(sc->sc_base_addr + TALITOS_RNGISR_HI);
 304         if (v) {
 305                 printk(KERN_ERR "%s: RNGISR_HI error %08x\n",
 306                         device_get_nameunit(sc->sc_cdev), v);
 307                 return;
 308         }
 309         /*
 310          * n.b. we need to add a FIPS test here - if the RNG is going
 311          * to fail, it's going to fail at reset time
 312          */
 313         return;
 314 }
 315 #endif /* CONFIG_OCF_RANDOMHARVEST */
 316
 317 /*
 318  * Generate a new software session.
 319  */
 320 static int
 321 talitos_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 322 {
 323         struct cryptoini *c, *encini = NULL, *macini = NULL;
 324         struct talitos_softc *sc = device_get_softc(dev);
 325         struct talitos_session *ses = NULL;
 326         int sesn;
 327
 328         DPRINTF("%s()\n", __FUNCTION__);
 329         if (sidp == NULL || cri == NULL || sc == NULL) {
 330                 DPRINTF("%s,%d - EINVAL\n", __FILE__, __LINE__);
 331                 return EINVAL;
 332         }
 333         for (c = cri; c != NULL; c = c->cri_next) {
 334                 if (c->cri_alg == CRYPTO_MD5 ||
 335                     c->cri_alg == CRYPTO_MD5_HMAC ||
 336                     c->cri_alg == CRYPTO_SHA1 ||
 337                     c->cri_alg == CRYPTO_SHA1_HMAC ||
 338                     c->cri_alg == CRYPTO_NULL_HMAC) {
 339                         if (macini)
 340                                 return EINVAL;
 341                         macini = c;
 342                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 343                     c->cri_alg == CRYPTO_3DES_CBC ||
 344                     c->cri_alg == CRYPTO_AES_CBC ||
 345                     c->cri_alg == CRYPTO_NULL_CBC) {
 346                         if (encini)
 347                                 return EINVAL;
 348                         encini = c;
 349                 } else {
 350                         DPRINTF("UNKNOWN c->cri_alg %d\n", encini->cri_alg);
 351                         return EINVAL;
 352                 }
 353         }
 354         if (encini == NULL && macini == NULL)
 355                 return EINVAL;
 356         if (encini) {
 357                 /* validate key length */
 358                 switch (encini->cri_alg) {
 359                 case CRYPTO_DES_CBC:
 360                         if (encini->cri_klen != 64)
 361                                 return EINVAL;
 362                         break;
 363                 case CRYPTO_3DES_CBC:
 364                         if (encini->cri_klen != 192) {
 365                                 return EINVAL;
 366                         }
 367                         break;
 368                 case CRYPTO_AES_CBC:
 369                         if (encini->cri_klen != 128 &&
 370                             encini->cri_klen != 192 &&
 371                             encini->cri_klen != 256)
 372                                 return EINVAL;
 373                         break;
 374                 default:
 375                         DPRINTF("UNKNOWN encini->cri_alg %d\n",
 376                                 encini->cri_alg);
 377                         return EINVAL;
 378                 }
 379         }
 380
 381         if (sc->sc_sessions == NULL) {
 382                 ses = sc->sc_sessions = (struct talitos_session *)
 383                         kmalloc(sizeof(struct talitos_session), SLAB_ATOMIC);
 384                 if (ses == NULL)
 385                         return ENOMEM;
 386                 memset(ses, 0, sizeof(struct talitos_session));
 387                 sesn = 0;
 388                 sc->sc_nsessions = 1;
 389         } else {
 390                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 391                         if (sc->sc_sessions[sesn].ses_used == 0) {
 392                                 ses = &sc->sc_sessions[sesn];
 393                                 break;
 394                         }
 395                 }
 396
 397                 if (ses == NULL) {
 398                         /* allocating session */
 399                         sesn = sc->sc_nsessions;
 400                         ses = (struct talitos_session *) kmalloc(
 401                                 (sesn + 1) * sizeof(struct talitos_session),
 402                                 SLAB_ATOMIC);
 403                         if (ses == NULL)
 404                                 return ENOMEM;
 405                         memset(ses, 0,
 406                                 (sesn + 1) * sizeof(struct talitos_session));
 407                         memcpy(ses, sc->sc_sessions,
 408                                 sesn * sizeof(struct talitos_session));
 409                         memset(sc->sc_sessions, 0,
 410                                 sesn * sizeof(struct talitos_session));
 411                         kfree(sc->sc_sessions);
 412                         sc->sc_sessions = ses;
 413                         ses = &sc->sc_sessions[sesn];
 414                         sc->sc_nsessions++;
 415                 }
 416         }
 417
 418         ses->ses_used = 1;
 419
 420         if (encini) {
 421                 /* get an IV */
 422                 /* XXX may read fewer than requested */
 423                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 424
 425                 ses->ses_klen = (encini->cri_klen + 7) / 8;
 426                 memcpy(ses->ses_key, encini->cri_key, ses->ses_klen);
 427                 if (macini) {
 428                         /* doing hash on top of cipher */
 429                         ses->ses_hmac_len = (macini->cri_klen + 7) / 8;
 430                         memcpy(ses->ses_hmac, macini->cri_key,
 431                                 ses->ses_hmac_len);
 432                 }
 433         } else if (macini) {
 434                 /* doing hash */
 435                 ses->ses_klen = (macini->cri_klen + 7) / 8;
 436                 memcpy(ses->ses_key, macini->cri_key, ses->ses_klen);
 437         }
 438
 439         /* back compat way of determining MSC result len */
 440         if (macini) {
 441                 ses->ses_mlen = macini->cri_mlen;
 442                 if (ses->ses_mlen == 0) {
 443                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 444                                 ses->ses_mlen = MD5_HASH_LEN;
 445                         else
 446                                 ses->ses_mlen = SHA1_HASH_LEN;
 447                 }
 448         }
 449
 450         /* really should make up a template td here,
 451          * and only fill things like i/o and direction in process() */
 452
 453         /* assign session ID */
 454         *sidp = TALITOS_SID(sc->sc_num, sesn);
 455         return 0;
 456 }
 457
 458 /*
 459  * Deallocate a session.
 460  */
 461 static int
 462 talitos_freesession(device_t dev, u_int64_t tid)
 463 {
 464         struct talitos_softc *sc = device_get_softc(dev);
 465         int session, ret;
 466         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 467
 468         if (sc == NULL)
 469                 return EINVAL;
 470         session = TALITOS_SESSION(sid);
 471         if (session < sc->sc_nsessions) {
 472                 memset(&sc->sc_sessions[session], 0,
 473                         sizeof(sc->sc_sessions[session]));
 474                 ret = 0;
 475         } else
 476                 ret = EINVAL;
 477         return ret;
 478 }
 479
 480 /*
 481  * launch device processing - it will come back with done notification
 482  * in the form of an interrupt and/or HDR_DONE_BITS in header
 483  */
 484 static int
 485 talitos_submit(
 486         struct talitos_softc *sc,
 487         struct talitos_desc *td,
 488         int chsel)
 489 {
 490         u_int32_t v;
 491
 492         v = dma_map_single(NULL, td, sizeof(*td), DMA_TO_DEVICE);
 493         talitos_write(sc->sc_base_addr +
 494                 chsel*TALITOS_CH_OFFSET + TALITOS_CH_FF, 0);
 495         talitos_write(sc->sc_base_addr +
 496                 chsel*TALITOS_CH_OFFSET + TALITOS_CH_FF_HI, v);
 497         return 0;
 498 }
 499
 500 static int
 501 talitos_process(device_t dev, struct cryptop *crp, int hint)
 502 {
 503         int i, err = 0, ivsize;
 504         struct talitos_softc *sc = device_get_softc(dev);
 505         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 506         caddr_t iv;
 507         struct talitos_session *ses;
 508         struct talitos_desc *td;
 509         unsigned long flags;
 510         /* descriptor mappings */
 511         int hmac_key, hmac_data, cipher_iv, cipher_key,
 512                 in_fifo, out_fifo, cipher_iv_out;
 513         static int chsel = -1;
 514
 515         DPRINTF("%s()\n", __FUNCTION__);
 516
 517         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 518                 return EINVAL;
 519         }
 520         crp->crp_etype = 0;
 521         if (TALITOS_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 522                 return EINVAL;
 523         }
 524
 525         ses = &sc->sc_sessions[TALITOS_SESSION(crp->crp_sid)];
 526
 527         /* enter the channel scheduler */
 528         spin_lock_irqsave(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 529
 530         /* reuse channel that already had/has requests for the required EU */
 531         for (i = 0; i < sc->sc_num_channels; i++) {
 532                 if (sc->sc_chnlastalg[i] == crp->crp_desc->crd_alg)
 533                         break;
 534         }
 535         if (i == sc->sc_num_channels) {
 536                 /*
 537                  * haven't seen this algo the last sc_num_channels or more
 538                  * use round robin in this case
 539                  * nb: sc->sc_num_channels must be power of 2
 540                  */
 541                 chsel = (chsel + 1) & (sc->sc_num_channels - 1);
 542         } else {
 543                 /*
 544                  * matches channel with same target execution unit;
 545                  * use same channel in this case
 546                  */
 547                 chsel = i;
 548         }
 549         sc->sc_chnlastalg[chsel] = crp->crp_desc->crd_alg;
 550
 551         /* release the channel scheduler lock */
 552         spin_unlock_irqrestore(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 553
 554         /* acquire the selected channel fifo lock */
 555         spin_lock_irqsave(&sc->sc_chnfifolock[chsel], flags);
 556
 557         /* find and reserve next available descriptor-cryptop pair */
 558         for (i = 0; i < sc->sc_chfifo_len; i++) {
 559                 if (sc->sc_chnfifo[chsel][i].cf_desc.hdr == 0) {
 560                         /*
 561                          * ensure correct descriptor formation by
 562                          * avoiding inadvertently setting "optional" entries
 563                          * e.g. not using "optional" dptr2 for MD/HMAC descs
 564                          */
 565                         memset(&sc->sc_chnfifo[chsel][i].cf_desc,
 566                                 0, sizeof(*td));
 567                         /* reserve it with done notification request bit */
 568                         sc->sc_chnfifo[chsel][i].cf_desc.hdr |=
 569                                 TALITOS_DONE_NOTIFY;
 570                         break;
 571                 }
 572         }
 573         spin_unlock_irqrestore(&sc->sc_chnfifolock[chsel], flags);
 574
 575         if (i == sc->sc_chfifo_len) {
 576                 /* fifo full */
 577                 err = ERESTART;
 578                 goto errout;
 579         }
 580
 581         td = &sc->sc_chnfifo[chsel][i].cf_desc;
 582         sc->sc_chnfifo[chsel][i].cf_crp = crp;
 583
 584         crd1 = crp->crp_desc;
 585         if (crd1 == NULL) {
 586                 err = EINVAL;
 587                 goto errout;
 588         }
 589         crd2 = crd1->crd_next;
 590         /* prevent compiler warning */
 591         hmac_key = 0;
 592         hmac_data = 0;
 593         if (crd2 == NULL) {
 594                 td->hdr |= TD_TYPE_COMMON_NONSNOOP_NO_AFEU;
 595                 /* assign descriptor dword ptr mappings for this desc. type */
 596                 cipher_iv = 1;
 597                 cipher_key = 2;
 598                 in_fifo = 3;
 599                 cipher_iv_out = 5;
 600                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 601                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 602                     crd1->crd_alg == CRYPTO_SHA1 ||
 603                     crd1->crd_alg == CRYPTO_MD5) {
 604                         out_fifo = 5;
 605                         maccrd = crd1;
 606                         enccrd = NULL;
 607                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 608                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 609                     crd1->crd_alg == CRYPTO_AES_CBC ||
 610                     crd1->crd_alg == CRYPTO_ARC4) {
 611                         out_fifo = 4;
 612                         maccrd = NULL;
 613                         enccrd = crd1;
 614                 } else {
 615                         DPRINTF("UNKNOWN crd1->crd_alg %d\n", crd1->crd_alg);
 616                         err = EINVAL;
 617                         goto errout;
 618                 }
 619         } else {
 620                 if (sc->sc_desc_types & TALITOS_HAS_DT_IPSEC_ESP) {
 621                         td->hdr |= TD_TYPE_IPSEC_ESP;
 622                 } else {
 623                         DPRINTF("unimplemented: multiple descriptor ipsec\n");
 624                         err = EINVAL;
 625                         goto errout;
 626                 }
 627                 /* assign descriptor dword ptr mappings for this desc. type */
 628                 hmac_key = 0;
 629                 hmac_data = 1;
 630                 cipher_iv = 2;
 631                 cipher_key = 3;
 632                 in_fifo = 4;
 633                 out_fifo = 5;
 634                 cipher_iv_out = 6;
 635                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 636                      crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 637                      crd1->crd_alg == CRYPTO_MD5 ||
 638                      crd1->crd_alg == CRYPTO_SHA1) &&
 639                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 640                      crd2->crd_alg == CRYPTO_3DES_CBC ||
 641                      crd2->crd_alg == CRYPTO_AES_CBC ||
 642                      crd2->crd_alg == CRYPTO_ARC4) &&
 643                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 644                         maccrd = crd1;
 645                         enccrd = crd2;
 646                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 647                      crd1->crd_alg == CRYPTO_ARC4 ||
 648                      crd1->crd_alg == CRYPTO_3DES_CBC ||
 649                      crd1->crd_alg == CRYPTO_AES_CBC) &&
 650                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 651                      crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 652                      crd2->crd_alg == CRYPTO_MD5 ||
 653                      crd2->crd_alg == CRYPTO_SHA1) &&
 654                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 655                         enccrd = crd1;
 656                         maccrd = crd2;
 657                 } else {
 658                         /* We cannot order the SEC as requested */
 659                         printk("%s: cannot do the order\n",
 660                                         device_get_nameunit(sc->sc_cdev));
 661                         err = EINVAL;
 662                         goto errout;
 663                 }
 664         }
 665         /* assign in_fifo and out_fifo based on input/output struct type */
 666         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 667                 /* using SKB buffers */
 668                 struct sk_buff *skb = (struct sk_buff *)crp->crp_buf;
 669                 if (skb_shinfo(skb)->nr_frags) {
 670                         printk("%s: skb frags unimplemented\n",
 671                                         device_get_nameunit(sc->sc_cdev));
 672                         err = EINVAL;
 673                         goto errout;
 674                 }
 675                 td->ptr[in_fifo].ptr = dma_map_single(NULL, skb->data,
 676                         skb->len, DMA_TO_DEVICE);
 677                 td->ptr[in_fifo].len = skb->len;
 678                 td->ptr[out_fifo].ptr = dma_map_single(NULL, skb->data,
 679                         skb->len, DMA_TO_DEVICE);
 680                 td->ptr[out_fifo].len = skb->len;
 681                 td->ptr[hmac_data].ptr = dma_map_single(NULL, skb->data,
 682                         skb->len, DMA_TO_DEVICE);
 683         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 684                 /* using IOV buffers */
 685                 struct uio *uiop = (struct uio *)crp->crp_buf;
 686                 if (uiop->uio_iovcnt > 1) {
 687                         printk("%s: iov frags unimplemented\n",
 688                                         device_get_nameunit(sc->sc_cdev));
 689                         err = EINVAL;
 690                         goto errout;
 691                 }
 692                 td->ptr[in_fifo].ptr = dma_map_single(NULL,
 693                         uiop->uio_iov->iov_base, crp->crp_ilen, DMA_TO_DEVICE);
 694                 td->ptr[in_fifo].len = crp->crp_ilen;
 695                 /* crp_olen is never set; always use crp_ilen */
 696                 td->ptr[out_fifo].ptr = dma_map_single(NULL,
 697                         uiop->uio_iov->iov_base,
 698                         crp->crp_ilen, DMA_TO_DEVICE);
 699                 td->ptr[out_fifo].len = crp->crp_ilen;
 700         } else {
 701                 /* using contig buffers */
 702                 td->ptr[in_fifo].ptr = dma_map_single(NULL,
 703                         crp->crp_buf, crp->crp_ilen, DMA_TO_DEVICE);
 704                 td->ptr[in_fifo].len = crp->crp_ilen;
 705                 td->ptr[out_fifo].ptr = dma_map_single(NULL,
 706                         crp->crp_buf, crp->crp_ilen, DMA_TO_DEVICE);
 707                 td->ptr[out_fifo].len = crp->crp_ilen;
 708         }
 709         if (enccrd) {
 710                 switch (enccrd->crd_alg) {
 711                 case CRYPTO_3DES_CBC:
 712                         td->hdr |= TALITOS_MODE0_DEU_3DES;
 713                         /* FALLTHROUGH */
 714                 case CRYPTO_DES_CBC:
 715                         td->hdr |= TALITOS_SEL0_DEU
 716                                 |  TALITOS_MODE0_DEU_CBC;
 717                         if (enccrd->crd_flags & CRD_F_ENCRYPT)
 718                                 td->hdr |= TALITOS_MODE0_DEU_ENC;
 719                         ivsize = 2*sizeof(u_int32_t);
 720                         DPRINTF("%cDES ses %d ch %d len %d\n",
 721                                 (td->hdr & TALITOS_MODE0_DEU_3DES)?'3':'1',
 722                                 (u32)TALITOS_SESSION(crp->crp_sid),
 723                                 chsel, td->ptr[in_fifo].len);
 724                         break;
 725                 case CRYPTO_AES_CBC:
 726                         td->hdr |= TALITOS_SEL0_AESU
 727                                 |  TALITOS_MODE0_AESU_CBC;
 728                         if (enccrd->crd_flags & CRD_F_ENCRYPT)
 729                                 td->hdr |= TALITOS_MODE0_AESU_ENC;
 730                         ivsize = 4*sizeof(u_int32_t);
 731                         DPRINTF("AES  ses %d ch %d len %d\n",
 732                                 (u32)TALITOS_SESSION(crp->crp_sid),
 733                                 chsel, td->ptr[in_fifo].len);
 734                         break;
 735                 default:
 736                         printk("%s: unimplemented enccrd->crd_alg %d\n",
 737                                         device_get_nameunit(sc->sc_cdev), enccrd->crd_alg);
 738                         err = EINVAL;
 739                         goto errout;
 740                 }
 741                 /*
 742                  * Setup encrypt/decrypt state.  When using basic ops
 743                  * we can't use an inline IV because hash/crypt offset
 744                  * must be from the end of the IV to the start of the
 745                  * crypt data and this leaves out the preceding header
 746                  * from the hash calculation.  Instead we place the IV
 747                  * in the state record and set the hash/crypt offset to
 748                  * copy both the header+IV.
 749                  */
 750                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 751                         td->hdr |= TALITOS_DIR_OUTBOUND;
 752                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 753                                 iv = enccrd->crd_iv;
 754                         else
 755                                 iv = (caddr_t) ses->ses_iv;
 756                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 757                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
 758                                     enccrd->crd_inject, ivsize, iv);
 759                         }
 760                 } else {
 761                         td->hdr |= TALITOS_DIR_INBOUND;
 762                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 763                                 iv = enccrd->crd_iv;
 764                                 bcopy(enccrd->crd_iv, iv, ivsize);
 765                         } else {
 766                                 iv = (caddr_t) ses->ses_iv;
 767                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 768                                     enccrd->crd_inject, ivsize, iv);
 769                         }
 770                 }
 771                 td->ptr[cipher_iv].ptr = dma_map_single(NULL, iv, ivsize,
 772                         DMA_TO_DEVICE);
 773                 td->ptr[cipher_iv].len = ivsize;
 774                 /*
 775                  * we don't need the cipher iv out length/pointer
 776                  * field to do ESP IPsec. Therefore we set the len field as 0,
 777                  * which tells the SEC not to do anything with this len/ptr
 778                  * field. Previously, when length/pointer as pointing to iv,
 779                  * it gave us corruption of packets.
 780                  */
 781                 td->ptr[cipher_iv_out].len = 0;
 782         }
 783         if (enccrd && maccrd) {
 784                 /* this is ipsec only for now */
 785                 td->hdr |= TALITOS_SEL1_MDEU
 786                         |  TALITOS_MODE1_MDEU_INIT
 787                         |  TALITOS_MODE1_MDEU_PAD;
 788                 switch (maccrd->crd_alg) {
 789                         case    CRYPTO_MD5:
 790                                 td->hdr |= TALITOS_MODE1_MDEU_MD5;
 791                                 break;
 792                         case    CRYPTO_MD5_HMAC:
 793                                 td->hdr |= TALITOS_MODE1_MDEU_MD5_HMAC;
 794                                 break;
 795                         case    CRYPTO_SHA1:
 796                                 td->hdr |= TALITOS_MODE1_MDEU_SHA1;
 797                                 break;
 798                         case    CRYPTO_SHA1_HMAC:
 799                                 td->hdr |= TALITOS_MODE1_MDEU_SHA1_HMAC;
 800                                 break;
 801                         default:
 802                                 /* We cannot order the SEC as requested */
 803                                 printk("%s: cannot do the order\n",
 804                                                 device_get_nameunit(sc->sc_cdev));
 805                                 err = EINVAL;
 806                                 goto errout;
 807                 }
 808                 if ((maccrd->crd_alg == CRYPTO_MD5_HMAC) ||
 809                    (maccrd->crd_alg == CRYPTO_SHA1_HMAC)) {
 810                         /*
 811                          * The offset from hash data to the start of
 812                          * crypt data is the difference in the skips.
 813                          */
 814                         /* ipsec only for now */
 815                         td->ptr[hmac_key].ptr = dma_map_single(NULL,
 816                                 ses->ses_hmac, ses->ses_hmac_len, DMA_TO_DEVICE);
 817                         td->ptr[hmac_key].len = ses->ses_hmac_len;
 818                         td->ptr[in_fifo].ptr  += enccrd->crd_skip;
 819                         td->ptr[in_fifo].len  =  enccrd->crd_len;
 820                         td->ptr[out_fifo].ptr += enccrd->crd_skip;
 821                         td->ptr[out_fifo].len =  enccrd->crd_len;
 822                         /* bytes of HMAC to postpend to ciphertext */
 823                         td->ptr[out_fifo].extent =  ses->ses_mlen;
 824                         td->ptr[hmac_data].ptr += maccrd->crd_skip;
 825                         td->ptr[hmac_data].len = enccrd->crd_skip - maccrd->crd_skip;
 826                 }
 827                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 828                         printk("%s: CRD_F_KEY_EXPLICIT unimplemented\n",
 829                                         device_get_nameunit(sc->sc_cdev));
 830                 }
 831         }
 832         if (!enccrd && maccrd) {
 833                 /* single MD5 or SHA */
 834                 td->hdr |= TALITOS_SEL0_MDEU
 835                                 |  TALITOS_MODE0_MDEU_INIT
 836                                 |  TALITOS_MODE0_MDEU_PAD;
 837                 switch (maccrd->crd_alg) {
 838                         case    CRYPTO_MD5:
 839                                 td->hdr |= TALITOS_MODE0_MDEU_MD5;
 840                                 DPRINTF("MD5  ses %d ch %d len %d\n",
 841                                         (u32)TALITOS_SESSION(crp->crp_sid),
 842                                         chsel, td->ptr[in_fifo].len);
 843                                 break;
 844                         case    CRYPTO_MD5_HMAC:
 845                                 td->hdr |= TALITOS_MODE0_MDEU_MD5_HMAC;
 846                                 break;
 847                         case    CRYPTO_SHA1:
 848                                 td->hdr |= TALITOS_MODE0_MDEU_SHA1;
 849                                 DPRINTF("SHA1 ses %d ch %d len %d\n",
 850                                         (u32)TALITOS_SESSION(crp->crp_sid),
 851                                         chsel, td->ptr[in_fifo].len);
 852                                 break;
 853                         case    CRYPTO_SHA1_HMAC:
 854                                 td->hdr |= TALITOS_MODE0_MDEU_SHA1_HMAC;
 855                                 break;
 856                         default:
 857                                 /* We cannot order the SEC as requested */
 858                                 DPRINTF("cannot do the order\n");
 859                                 err = EINVAL;
 860                                 goto errout;
 861                 }
 862
 863                 if (crp->crp_flags & CRYPTO_F_IOV)
 864                         td->ptr[out_fifo].ptr += maccrd->crd_inject;
 865
 866                 if ((maccrd->crd_alg == CRYPTO_MD5_HMAC) ||
 867                    (maccrd->crd_alg == CRYPTO_SHA1_HMAC)) {
 868                         td->ptr[hmac_key].ptr = dma_map_single(NULL,
 869                                 ses->ses_hmac, ses->ses_hmac_len,
 870                                 DMA_TO_DEVICE);
 871                         td->ptr[hmac_key].len = ses->ses_hmac_len;
 872                 }
 873         }
 874         else {
 875                 /* using process key (session data has duplicate) */
 876                 td->ptr[cipher_key].ptr = dma_map_single(NULL,
 877                         enccrd->crd_key, (enccrd->crd_klen + 7) / 8,
 878                         DMA_TO_DEVICE);
 879                 td->ptr[cipher_key].len = (enccrd->crd_klen + 7) / 8;
 880         }
 881         /* descriptor complete - GO! */
 882         return talitos_submit(sc, td, chsel);
 883
 884 errout:
 885         if (err != ERESTART) {
 886                 crp->crp_etype = err;
 887                 crypto_done(crp);
 888         }
 889         return err;
 890 }
 891
 892 /* go through all channels descriptors, notifying OCF what has
 893  * _and_hasn't_ successfully completed and reset the device
 894  * (otherwise it's up to decoding desc hdrs!)
 895  */
 896 static void talitos_errorprocessing(struct talitos_softc *sc)
 897 {
 898         unsigned long flags;
 899         int i, j;
 900
 901         /* disable further scheduling until under control */
 902         spin_lock_irqsave(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 903
 904         if (debug) dump_talitos_status(sc);
 905         /* go through descriptors, try and salvage those successfully done,
 906          * and EIO those that weren't
 907          */
 908         for (i = 0; i < sc->sc_num_channels; i++) {
 909                 spin_lock_irqsave(&sc->sc_chnfifolock[i], flags);
 910                 for (j = 0; j < sc->sc_chfifo_len; j++) {
 911                         if (sc->sc_chnfifo[i][j].cf_desc.hdr) {
 912                                 if ((sc->sc_chnfifo[i][j].cf_desc.hdr
 913                                         & TALITOS_HDR_DONE_BITS)
 914                                         != TALITOS_HDR_DONE_BITS) {
 915                                         /* this one didn't finish */
 916                                         /* signify in crp->etype */
 917                                         sc->sc_chnfifo[i][j].cf_crp->crp_etype
 918                                                 = EIO;
 919                                 }
 920                         } else
 921                                 continue; /* free entry */
 922                         /* either way, notify ocf */
 923                         crypto_done(sc->sc_chnfifo[i][j].cf_crp);
 924                         /* and tag it available again
 925                          *
 926                          * memset to ensure correct descriptor formation by
 927                          * avoiding inadvertently setting "optional" entries
 928                          * e.g. not using "optional" dptr2 MD/HMAC processing
 929                          */
 930                         memset(&sc->sc_chnfifo[i][j].cf_desc,
 931                                 0, sizeof(struct talitos_desc));
 932                 }
 933                 spin_unlock_irqrestore(&sc->sc_chnfifolock[i], flags);
 934         }
 935         /* reset and initialize the SEC h/w device */
 936         talitos_reset_device(sc);
 937         talitos_init_device(sc);
 938 #ifdef CONFIG_OCF_RANDOMHARVEST
 939         if (sc->sc_exec_units & TALITOS_HAS_EU_RNG)
 940                 talitos_rng_init(sc);
 941 #endif
 942
 943         /* Okay. Stand by. */
 944         spin_unlock_irqrestore(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 945
 946         return;
 947 }
 948
 949 /* go through all channels descriptors, notifying OCF what's been done */
 950 static void talitos_doneprocessing(struct talitos_softc *sc)
 951 {
 952         unsigned long flags;
 953         int i, j;
 954
 955         /* go through descriptors looking for done bits */
 956         for (i = 0; i < sc->sc_num_channels; i++) {
 957                 spin_lock_irqsave(&sc->sc_chnfifolock[i], flags);
 958                 for (j = 0; j < sc->sc_chfifo_len; j++) {
 959                         /* descriptor has done bits set? */
 960                         if ((sc->sc_chnfifo[i][j].cf_desc.hdr
 961                                 & TALITOS_HDR_DONE_BITS)
 962                                 == TALITOS_HDR_DONE_BITS) {
 963                                 /* notify ocf */
 964                                 crypto_done(sc->sc_chnfifo[i][j].cf_crp);
 965                                 /* and tag it available again
 966                                  *
 967                                  * memset to ensure correct descriptor formation by
 968                                  * avoiding inadvertently setting "optional" entries
 969                                  * e.g. not using "optional" dptr2 MD/HMAC processing
 970                                  */
 971                                 memset(&sc->sc_chnfifo[i][j].cf_desc,
 972                                         0, sizeof(struct talitos_desc));
 973                         }
 974                 }
 975                 spin_unlock_irqrestore(&sc->sc_chnfifolock[i], flags);
 976         }
 977         return;
 978 }
 979
 980 static irqreturn_t
 981 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 982 talitos_intr(int irq, void *arg)
 983 #else
 984 talitos_intr(int irq, void *arg, struct pt_regs *regs)
 985 #endif
 986 {
 987         struct talitos_softc *sc = arg;
 988         u_int32_t v, v_hi;
 989
 990         /* ack */
 991         v = talitos_read(sc->sc_base_addr + TALITOS_ISR);
 992         v_hi = talitos_read(sc->sc_base_addr + TALITOS_ISR_HI);
 993         talitos_write(sc->sc_base_addr + TALITOS_ICR, v);
 994         talitos_write(sc->sc_base_addr + TALITOS_ICR_HI, v_hi);
 995
 996         if (unlikely(v & TALITOS_ISR_ERROR)) {
 997                 /* Okay, Houston, we've had a problem here. */
 998                 printk(KERN_DEBUG "%s: got error interrupt - ISR 0x%08x_%08x\n",
 999                                 device_get_nameunit(sc->sc_cdev), v, v_hi);
1000                 talitos_errorprocessing(sc);
1001         } else
1002         if (likely(v & TALITOS_ISR_DONE)) {
1003                 talitos_doneprocessing(sc);
1004         }
1005         return IRQ_HANDLED;
1006 }
1007
1008 /*
1009  * Initialize registers we need to touch only once.
1010  */
1011 static void
1012 talitos_init_device(struct talitos_softc *sc)
1013 {
1014         u_int32_t v;
1015         int i;
1016
1017         DPRINTF("%s()\n", __FUNCTION__);
1018
1019         /* init all channels */
1020         for (i = 0; i < sc->sc_num_channels; i++) {
1021                 v = talitos_read(sc->sc_base_addr +
1022                         i*TALITOS_CH_OFFSET + TALITOS_CH_CCCR_HI);
1023                 v |= TALITOS_CH_CCCR_HI_CDWE
1024                   |  TALITOS_CH_CCCR_HI_CDIE;  /* invoke interrupt if done */
1025                 talitos_write(sc->sc_base_addr +
1026                         i*TALITOS_CH_OFFSET + TALITOS_CH_CCCR_HI, v);
1027         }
1028         /* enable all interrupts */
1029         v = talitos_read(sc->sc_base_addr + TALITOS_IMR);
1030         v |= TALITOS_IMR_ALL;
1031         talitos_write(sc->sc_base_addr + TALITOS_IMR, v);
1032         v = talitos_read(sc->sc_base_addr + TALITOS_IMR_HI);
1033         v |= TALITOS_IMR_HI_ERRONLY;
1034         talitos_write(sc->sc_base_addr + TALITOS_IMR_HI, v);
1035         return;
1036 }
1037
1038 /*
1039  * set the master reset bit on the device.
1040  */
1041 static void
1042 talitos_reset_device_master(struct talitos_softc *sc)
1043 {
1044         u_int32_t v;
1045
1046         /* Reset the device by writing 1 to MCR:SWR and waiting 'til cleared */
1047         v = talitos_read(sc->sc_base_addr + TALITOS_MCR);
1048         talitos_write(sc->sc_base_addr + TALITOS_MCR, v | TALITOS_MCR_SWR);
1049
1050         while (talitos_read(sc->sc_base_addr + TALITOS_MCR) & TALITOS_MCR_SWR)
1051                 cpu_relax();
1052
1053         return;
1054 }
1055
1056 /*
1057  * Resets the device.  Values in the registers are left as is
1058  * from the reset (i.e. initial values are assigned elsewhere).
1059  */
1060 static void
1061 talitos_reset_device(struct talitos_softc *sc)
1062 {
1063         u_int32_t v;
1064         int i;
1065
1066         DPRINTF("%s()\n", __FUNCTION__);
1067
1068         /*
1069          * Master reset
1070          * errata documentation: warning: certain SEC interrupts
1071          * are not fully cleared by writing the MCR:SWR bit,
1072          * set bit twice to completely reset
1073          */
1074         talitos_reset_device_master(sc);        /* once */
1075         talitos_reset_device_master(sc);        /* and once again */
1076
1077         /* reset all channels */
1078         for (i = 0; i < sc->sc_num_channels; i++) {
1079                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
1080                         TALITOS_CH_CCCR);
1081                 talitos_write(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
1082                         TALITOS_CH_CCCR, v | TALITOS_CH_CCCR_RESET);
1083         }
1084 }
1085
1086 /* Set up the crypto device structure, private data,
1087  * and anything else we need before we start */
1088 #ifdef CONFIG_PPC_MERGE
1089 static int talitos_probe(struct of_device *ofdev, const struct of_device_id *match)
1090 #else
1091 static int talitos_probe(struct platform_device *pdev)
1092 #endif
1093 {
1094         struct talitos_softc *sc = NULL;
1095         struct resource *r;
1096 #ifdef CONFIG_PPC_MERGE
1097         struct device *device = &ofdev->dev;
1098         struct device_node *np = ofdev->node;
1099         const unsigned int *prop;
1100         int err;
1101         struct resource res;
1102 #endif
1103         static int num_chips = 0;
1104         int rc;
1105         int i;
1106
1107         DPRINTF("%s()\n", __FUNCTION__);
1108
1109         sc = (struct talitos_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
1110         if (!sc)
1111                 return -ENOMEM;
1112         memset(sc, 0, sizeof(*sc));
1113
1114         softc_device_init(sc, DRV_NAME, num_chips, talitos_methods);
1115
1116         sc->sc_irq = -1;
1117         sc->sc_cid = -1;
1118 #ifndef CONFIG_PPC_MERGE
1119         sc->sc_dev = pdev;
1120 #endif
1121         sc->sc_num = num_chips++;
1122
1123 #ifdef CONFIG_PPC_MERGE
1124         dev_set_drvdata(device, sc);
1125 #else
1126         platform_set_drvdata(sc->sc_dev, sc);
1127 #endif
1128
1129         /* get the irq line */
1130 #ifdef CONFIG_PPC_MERGE
1131         err = of_address_to_resource(np, 0, &res);
1132         if (err)
1133                 return -EINVAL;
1134         r = &res;
1135
1136         sc->sc_irq = irq_of_parse_and_map(np, 0);
1137 #else
1138         /* get a pointer to the register memory */
1139         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1140
1141         sc->sc_irq = platform_get_irq(pdev, 0);
1142 #endif
1143         rc = request_irq(sc->sc_irq, talitos_intr, 0,
1144                         device_get_nameunit(sc->sc_cdev), sc);
1145         if (rc) {
1146                 printk(KERN_ERR "%s: failed to hook irq %d\n",
1147                                 device_get_nameunit(sc->sc_cdev), sc->sc_irq);
1148                 sc->sc_irq = -1;
1149                 goto out;
1150         }
1151
1152         sc->sc_base_addr = (ocf_iomem_t) ioremap(r->start, (r->end - r->start));
1153         if (!sc->sc_base_addr) {
1154                 printk(KERN_ERR "%s: failed to ioremap\n",
1155                                 device_get_nameunit(sc->sc_cdev));
1156                 goto out;
1157         }
1158
1159         /* figure out our SEC's properties and capabilities */
1160         sc->sc_chiprev = (u64)talitos_read(sc->sc_base_addr + TALITOS_ID) << 32
1161                  | talitos_read(sc->sc_base_addr + TALITOS_ID_HI);
1162         DPRINTF("sec id 0x%llx\n", sc->sc_chiprev);
1163
1164 #ifdef CONFIG_PPC_MERGE
1165         /* get SEC properties from device tree, defaulting to SEC 2.0 */
1166
1167         prop = of_get_property(np, "num-channels", NULL);
1168         sc->sc_num_channels = prop ? *prop : TALITOS_NCHANNELS_SEC_2_0;
1169
1170         prop = of_get_property(np, "channel-fifo-len", NULL);
1171         sc->sc_chfifo_len = prop ? *prop : TALITOS_CHFIFOLEN_SEC_2_0;
1172
1173         prop = of_get_property(np, "exec-units-mask", NULL);
1174         sc->sc_exec_units = prop ? *prop : TALITOS_HAS_EUS_SEC_2_0;
1175
1176         prop = of_get_property(np, "descriptor-types-mask", NULL);
1177         sc->sc_desc_types = prop ? *prop : TALITOS_HAS_DESCTYPES_SEC_2_0;
1178 #else
1179         /* bulk should go away with openfirmware flat device tree support */
1180         if (sc->sc_chiprev & TALITOS_ID_SEC_2_0) {
1181                 sc->sc_num_channels = TALITOS_NCHANNELS_SEC_2_0;
1182                 sc->sc_chfifo_len = TALITOS_CHFIFOLEN_SEC_2_0;
1183                 sc->sc_exec_units = TALITOS_HAS_EUS_SEC_2_0;
1184                 sc->sc_desc_types = TALITOS_HAS_DESCTYPES_SEC_2_0;
1185         } else {
1186                 printk(KERN_ERR "%s: failed to id device\n",
1187                                 device_get_nameunit(sc->sc_cdev));
1188                 goto out;
1189         }
1190 #endif
1191
1192         /* + 1 is for the meta-channel lock used by the channel scheduler */
1193         sc->sc_chnfifolock = (spinlock_t *) kmalloc(
1194                 (sc->sc_num_channels + 1) * sizeof(spinlock_t), GFP_KERNEL);
1195         if (!sc->sc_chnfifolock)
1196                 goto out;
1197         for (i = 0; i < sc->sc_num_channels + 1; i++) {
1198                 spin_lock_init(&sc->sc_chnfifolock[i]);
1199         }
1200
1201         sc->sc_chnlastalg = (int *) kmalloc(
1202                 sc->sc_num_channels * sizeof(int), GFP_KERNEL);
1203         if (!sc->sc_chnlastalg)
1204                 goto out;
1205         memset(sc->sc_chnlastalg, 0, sc->sc_num_channels * sizeof(int));
1206
1207         sc->sc_chnfifo = (struct desc_cryptop_pair **) kmalloc(
1208                 sc->sc_num_channels * sizeof(struct desc_cryptop_pair *),
1209                 GFP_KERNEL);
1210         if (!sc->sc_chnfifo)
1211                 goto out;
1212         for (i = 0; i < sc->sc_num_channels; i++) {
1213                 sc->sc_chnfifo[i] = (struct desc_cryptop_pair *) kmalloc(
1214                         sc->sc_chfifo_len * sizeof(struct desc_cryptop_pair),
1215                         GFP_KERNEL);
1216                 if (!sc->sc_chnfifo[i])
1217                         goto out;
1218                 memset(sc->sc_chnfifo[i], 0,
1219                         sc->sc_chfifo_len * sizeof(struct desc_cryptop_pair));
1220         }
1221
1222         /* reset and initialize the SEC h/w device */
1223         talitos_reset_device(sc);
1224         talitos_init_device(sc);
1225
1226         sc->sc_cid = crypto_get_driverid(softc_get_device(sc),CRYPTOCAP_F_HARDWARE);
1227         if (sc->sc_cid < 0) {
1228                 printk(KERN_ERR "%s: could not get crypto driver id\n",
1229                                 device_get_nameunit(sc->sc_cdev));
1230                 goto out;
1231         }
1232
1233         /* register algorithms with the framework */
1234         printk("%s:", device_get_nameunit(sc->sc_cdev));
1235
1236         if (sc->sc_exec_units & TALITOS_HAS_EU_RNG)  {
1237                 printk(" rng");
1238 #ifdef CONFIG_OCF_RANDOMHARVEST
1239                 talitos_rng_init(sc);
1240                 crypto_rregister(sc->sc_cid, talitos_read_random, sc);
1241 #endif
1242         }
1243         if (sc->sc_exec_units & TALITOS_HAS_EU_DEU) {
1244                 printk(" des/3des");
1245                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
1246                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
1247         }
1248         if (sc->sc_exec_units & TALITOS_HAS_EU_AESU) {
1249                 printk(" aes");
1250                 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
1251         }
1252         if (sc->sc_exec_units & TALITOS_HAS_EU_MDEU) {
1253                 printk(" md5");
1254                 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
1255                 /* HMAC support only with IPsec for now */
1256                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
1257                 printk(" sha1");
1258                 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
1259                 /* HMAC support only with IPsec for now */
1260                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
1261         }
1262         printk("\n");
1263         return 0;
1264
1265 out:
1266 #ifndef CONFIG_PPC_MERGE
1267         talitos_remove(pdev);
1268 #endif
1269         return -ENOMEM;
1270 }
1271
1272 #ifdef CONFIG_PPC_MERGE
1273 static int talitos_remove(struct of_device *ofdev)
1274 #else
1275 static int talitos_remove(struct platform_device *pdev)
1276 #endif
1277 {
1278 #ifdef CONFIG_PPC_MERGE
1279         struct talitos_softc *sc = dev_get_drvdata(&ofdev->dev);
1280 #else
1281         struct talitos_softc *sc = platform_get_drvdata(pdev);
1282 #endif
1283         int i;
1284
1285         DPRINTF("%s()\n", __FUNCTION__);
1286         if (sc->sc_cid >= 0)
1287                 crypto_unregister_all(sc->sc_cid);
1288         if (sc->sc_chnfifo) {
1289                 for (i = 0; i < sc->sc_num_channels; i++)
1290                         if (sc->sc_chnfifo[i])
1291                                 kfree(sc->sc_chnfifo[i]);
1292                 kfree(sc->sc_chnfifo);
1293         }
1294         if (sc->sc_chnlastalg)
1295                 kfree(sc->sc_chnlastalg);
1296         if (sc->sc_chnfifolock)
1297                 kfree(sc->sc_chnfifolock);
1298         if (sc->sc_irq != -1)
1299                 free_irq(sc->sc_irq, sc);
1300         if (sc->sc_base_addr)
1301                 iounmap((void *) sc->sc_base_addr);
1302         kfree(sc);
1303         return 0;
1304 }
1305
1306 #ifdef CONFIG_PPC_MERGE
1307 static struct of_device_id talitos_match[] = {
1308         {
1309                 .type = "crypto",
1310                 .compatible = "talitos",
1311         },
1312         {},
1313 };
1314
1315 MODULE_DEVICE_TABLE(of, talitos_match);
1316
1317 static struct of_platform_driver talitos_driver = {
1318         .name           = DRV_NAME,
1319         .match_table    = talitos_match,
1320         .probe          = talitos_probe,
1321         .remove         = talitos_remove,
1322 };
1323
1324 static int __init talitos_init(void)
1325 {
1326         return of_register_platform_driver(&talitos_driver);
1327 }
1328
1329 static void __exit talitos_exit(void)
1330 {
1331         of_unregister_platform_driver(&talitos_driver);
1332 }
1333 #else
1334 /* Structure for a platform device driver */
1335 static struct platform_driver talitos_driver = {
1336         .probe = talitos_probe,
1337         .remove = talitos_remove,
1338         .driver = {
1339                 .name = "fsl-sec2",
1340         }
1341 };
1342
1343 static int __init talitos_init(void)
1344 {
1345         return platform_driver_register(&talitos_driver);
1346 }
1347
1348 static void __exit talitos_exit(void)
1349 {
1350         platform_driver_unregister(&talitos_driver);
1351 }
1352 #endif
1353
1354 module_init(talitos_init);
1355 module_exit(talitos_exit);
1356
1357 MODULE_LICENSE("Dual BSD/GPL");
1358 MODULE_AUTHOR("kim.phillips@freescale.com");
1359 MODULE_DESCRIPTION("OCF driver for Freescale SEC (talitos)");