target/linux/generic/files/crypto/ocf/talitos/talitos.c

   1 /*
   2  * crypto/ocf/talitos/talitos.c
   3  *
   4  * An OCF-Linux module that uses Freescale's SEC to do the crypto.
   5  * Based on crypto/ocf/hifn and crypto/ocf/safe OCF drivers
   6  *
   7  * Copyright (c) 2006 Freescale Semiconductor, Inc.
   8  *
   9  * This code written by Kim A. B. Phillips <kim.phillips@freescale.com>
  10  * some code copied from files with the following:
  11  * Copyright (C) 2004-2007 David McCullough <david_mccullough@mcafee.com>
  12  *
  13  * Redistribution and use in source and binary forms, with or without
  14  * modification, are permitted provided that the following conditions
  15  * are met:
  16  *
  17  * 1. Redistributions of source code must retain the above copyright
  18  *    notice, this list of conditions and the following disclaimer.
  19  * 2. Redistributions in binary form must reproduce the above copyright
  20  *    notice, this list of conditions and the following disclaimer in the
  21  *    documentation and/or other materials provided with the distribution.
  22  * 3. The name of the author may not be used to endorse or promote products
  23  *    derived from this software without specific prior written permission.
  24  *
  25  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  26  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  27  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  28  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  29  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  34  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  * ---------------------------------------------------------------------------
  37  *
  38  * NOTES:
  39  *
  40  * The Freescale SEC (also known as 'talitos') resides on the
  41  * internal bus, and runs asynchronous to the processor core.  It has
  42  * a wide gamut of cryptographic acceleration features, including single-
  43  * pass IPsec (also known as algorithm chaining).  To properly utilize
  44  * all of the SEC's performance enhancing features, further reworking
  45  * of higher level code (framework, applications) will be necessary.
  46  *
  47  * The following table shows which SEC version is present in which devices:
  48  *
  49  * Devices       SEC version
  50  *
  51  * 8272, 8248    SEC 1.0
  52  * 885, 875      SEC 1.2
  53  * 8555E, 8541E  SEC 2.0
  54  * 8349E         SEC 2.01
  55  * 8548E         SEC 2.1
  56  *
  57  * The following table shows the features offered by each SEC version:
  58  *
  59  *                             Max.   chan-
  60  * version  Bus I/F       Clock  nels  DEU AESU AFEU MDEU PKEU RNG KEU
  61  *
  62  * SEC 1.0  internal 64b  100MHz   4     1    1    1    1    1   1   0
  63  * SEC 1.2  internal 32b   66MHz   1     1    1    0    1    0   0   0
  64  * SEC 2.0  internal 64b  166MHz   4     1    1    1    1    1   1   0
  65  * SEC 2.01 internal 64b  166MHz   4     1    1    1    1    1   1   0
  66  * SEC 2.1  internal 64b  333MHz   4     1    1    1    1    1   1   1
  67  *
  68  * Each execution unit in the SEC has two modes of execution; channel and
  69  * slave/debug.  This driver employs the channel infrastructure in the
  70  * device for convenience.  Only the RNG is directly accessed due to the
  71  * convenience of its random fifo pool.  The relationship between the
  72  * channels and execution units is depicted in the following diagram:
  73  *
  74  *    -------   ------------
  75  * ---| ch0 |---|          |
  76  *    -------   |          |
  77  *              |          |------+-------+-------+-------+------------
  78  *    -------   |          |      |       |       |       |           |
  79  * ---| ch1 |---|          |      |       |       |       |           |
  80  *    -------   |          |   ------  ------  ------  ------      ------
  81  *              |controller|   |DEU |  |AESU|  |MDEU|  |PKEU| ...  |RNG |
  82  *    -------   |          |   ------  ------  ------  ------      ------
  83  * ---| ch2 |---|          |      |       |       |       |           |
  84  *    -------   |          |      |       |       |       |           |
  85  *              |          |------+-------+-------+-------+------------
  86  *    -------   |          |
  87  * ---| ch3 |---|          |
  88  *    -------   ------------
  89  *
  90  * Channel ch0 may drive an aes operation to the aes unit (AESU),
  91  * and, at the same time, ch1 may drive a message digest operation
  92  * to the mdeu. Each channel has an input descriptor FIFO, and the
  93  * FIFO can contain, e.g. on the 8541E, up to 24 entries, before a
  94  * a buffer overrun error is triggered. The controller is responsible
  95  * for fetching the data from descriptor pointers, and passing the
  96  * data to the appropriate EUs. The controller also writes the
  97  * cryptographic operation's result to memory. The SEC notifies
  98  * completion by triggering an interrupt and/or setting the 1st byte
  99  * of the hdr field to 0xff.
 100  *
 101  * TODO:
 102  * o support more algorithms
 103  * o support more versions of the SEC
 104  * o add support for linux 2.4
 105  * o scatter-gather (sg) support
 106  * o add support for public key ops (PKEU)
 107  * o add statistics
 108  */
 109
 110 #include <linux/version.h>
 111 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,33))
 112 #include <generated/autoconf.h>
 113 #else
 114 #include <linux/autoconf.h>
 115 #endif
 116 #include <linux/module.h>
 117 #include <linux/init.h>
 118 #include <linux/interrupt.h>
 119 #include <linux/spinlock.h>
 120 #include <linux/random.h>
 121 #include <linux/skbuff.h>
 122 #include <asm/scatterlist.h>
 123 #include <linux/dma-mapping.h>  /* dma_map_single() */
 124 #include <linux/moduleparam.h>
 125
 126 #include <linux/version.h>
 127 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,15)
 128 #include <linux/platform_device.h>
 129 #endif
 130
 131 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 132 #include <linux/of_platform.h>
 133 #endif
 134
 135 #include <cryptodev.h>
 136 #include <uio.h>
 137
 138 #define DRV_NAME "talitos"
 139
 140 #include "talitos_dev.h"
 141 #include "talitos_soft.h"
 142
 143 #define read_random(p,l) get_random_bytes(p,l)
 144
 145 const char talitos_driver_name[] = "Talitos OCF";
 146 const char talitos_driver_version[] = "0.2";
 147
 148 static int talitos_newsession(device_t dev, u_int32_t *sidp,
 149                                                                 struct cryptoini *cri);
 150 static int talitos_freesession(device_t dev, u_int64_t tid);
 151 static int talitos_process(device_t dev, struct cryptop *crp, int hint);
 152 static void dump_talitos_status(struct talitos_softc *sc);
 153 static int talitos_submit(struct talitos_softc *sc, struct talitos_desc *td,
 154                                                                 int chsel);
 155 static void talitos_doneprocessing(struct talitos_softc *sc);
 156 static void talitos_init_device(struct talitos_softc *sc);
 157 static void talitos_reset_device_master(struct talitos_softc *sc);
 158 static void talitos_reset_device(struct talitos_softc *sc);
 159 static void talitos_errorprocessing(struct talitos_softc *sc);
 160 #ifdef CONFIG_PPC_MERGE
 161 static int talitos_probe(struct of_device *ofdev, const struct of_device_id *match);
 162 static int talitos_remove(struct of_device *ofdev);
 163 #else
 164 static int talitos_probe(struct platform_device *pdev);
 165 static int talitos_remove(struct platform_device *pdev);
 166 #endif
 167 #ifdef CONFIG_OCF_RANDOMHARVEST
 168 static int talitos_read_random(void *arg, u_int32_t *buf, int maxwords);
 169 static void talitos_rng_init(struct talitos_softc *sc);
 170 #endif
 171
 172 static device_method_t talitos_methods = {
 173         /* crypto device methods */
 174         DEVMETHOD(cryptodev_newsession, talitos_newsession),
 175         DEVMETHOD(cryptodev_freesession,talitos_freesession),
 176         DEVMETHOD(cryptodev_process,    talitos_process),
 177 };
 178
 179 #define debug talitos_debug
 180 int talitos_debug = 0;
 181 module_param(talitos_debug, int, 0644);
 182 MODULE_PARM_DESC(talitos_debug, "Enable debug");
 183
 184 static inline void talitos_write(volatile unsigned *addr, u32 val)
 185 {
 186         out_be32(addr, val);
 187 }
 188
 189 static inline u32 talitos_read(volatile unsigned *addr)
 190 {
 191         u32 val;
 192         val = in_be32(addr);
 193         return val;
 194 }
 195
 196 static void dump_talitos_status(struct talitos_softc *sc)
 197 {
 198         unsigned int v, v_hi, i, *ptr;
 199         v = talitos_read(sc->sc_base_addr + TALITOS_MCR);
 200         v_hi = talitos_read(sc->sc_base_addr + TALITOS_MCR_HI);
 201         printk(KERN_INFO "%s: MCR          0x%08x_%08x\n",
 202                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 203         v = talitos_read(sc->sc_base_addr + TALITOS_IMR);
 204         v_hi = talitos_read(sc->sc_base_addr + TALITOS_IMR_HI);
 205         printk(KERN_INFO "%s: IMR          0x%08x_%08x\n",
 206                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 207         v = talitos_read(sc->sc_base_addr + TALITOS_ISR);
 208         v_hi = talitos_read(sc->sc_base_addr + TALITOS_ISR_HI);
 209         printk(KERN_INFO "%s: ISR          0x%08x_%08x\n",
 210                         device_get_nameunit(sc->sc_cdev), v, v_hi);
 211         for (i = 0; i < sc->sc_num_channels; i++) {
 212                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 213                         TALITOS_CH_CDPR);
 214                 v_hi = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 215                         TALITOS_CH_CDPR_HI);
 216                 printk(KERN_INFO "%s: CDPR     ch%d 0x%08x_%08x\n",
 217                                 device_get_nameunit(sc->sc_cdev), i, v, v_hi);
 218         }
 219         for (i = 0; i < sc->sc_num_channels; i++) {
 220                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 221                         TALITOS_CH_CCPSR);
 222                 v_hi = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
 223                         TALITOS_CH_CCPSR_HI);
 224                 printk(KERN_INFO "%s: CCPSR    ch%d 0x%08x_%08x\n",
 225                                 device_get_nameunit(sc->sc_cdev), i, v, v_hi);
 226         }
 227         ptr = sc->sc_base_addr + TALITOS_CH_DESCBUF;
 228         for (i = 0; i < 16; i++) {
 229                 v = talitos_read(ptr++); v_hi = talitos_read(ptr++);
 230                 printk(KERN_INFO "%s: DESCBUF  ch0 0x%08x_%08x (tdp%02d)\n",
 231                                 device_get_nameunit(sc->sc_cdev), v, v_hi, i);
 232         }
 233         return;
 234 }
 235
 236
 237 #ifdef CONFIG_OCF_RANDOMHARVEST
 238 /*
 239  * pull random numbers off the RNG FIFO, not exceeding amount available
 240  */
 241 static int
 242 talitos_read_random(void *arg, u_int32_t *buf, int maxwords)
 243 {
 244         struct talitos_softc *sc = (struct talitos_softc *) arg;
 245         int rc;
 246         u_int32_t v;
 247
 248         DPRINTF("%s()\n", __FUNCTION__);
 249
 250         /* check for things like FIFO underflow */
 251         v = talitos_read(sc->sc_base_addr + TALITOS_RNGISR_HI);
 252         if (unlikely(v)) {
 253                 printk(KERN_ERR "%s: RNGISR_HI error %08x\n",
 254                                 device_get_nameunit(sc->sc_cdev), v);
 255                 return 0;
 256         }
 257         /*
 258          * OFL is number of available 64-bit words,
 259          * shift and convert to a 32-bit word count
 260          */
 261         v = talitos_read(sc->sc_base_addr + TALITOS_RNGSR_HI);
 262         v = (v & TALITOS_RNGSR_HI_OFL) >> (16 - 1);
 263         if (maxwords > v)
 264                 maxwords = v;
 265         for (rc = 0; rc < maxwords; rc++) {
 266                 buf[rc] = talitos_read(sc->sc_base_addr +
 267                         TALITOS_RNG_FIFO + rc*sizeof(u_int32_t));
 268         }
 269         if (maxwords & 1) {
 270                 /*
 271                  * RNG will complain with an AE in the RNGISR
 272                  * if we don't complete the pairs of 32-bit reads
 273                  * to its 64-bit register based FIFO
 274                  */
 275                 v = talitos_read(sc->sc_base_addr +
 276                         TALITOS_RNG_FIFO + rc*sizeof(u_int32_t));
 277         }
 278
 279         return rc;
 280 }
 281
 282 static void
 283 talitos_rng_init(struct talitos_softc *sc)
 284 {
 285         u_int32_t v;
 286
 287         DPRINTF("%s()\n", __FUNCTION__);
 288         /* reset RNG EU */
 289         v = talitos_read(sc->sc_base_addr + TALITOS_RNGRCR_HI);
 290         v |= TALITOS_RNGRCR_HI_SR;
 291         talitos_write(sc->sc_base_addr + TALITOS_RNGRCR_HI, v);
 292         while ((talitos_read(sc->sc_base_addr + TALITOS_RNGSR_HI)
 293                 & TALITOS_RNGSR_HI_RD) == 0)
 294                         cpu_relax();
 295         /*
 296          * we tell the RNG to start filling the RNG FIFO
 297          * by writing the RNGDSR
 298          */
 299         v = talitos_read(sc->sc_base_addr + TALITOS_RNGDSR_HI);
 300         talitos_write(sc->sc_base_addr + TALITOS_RNGDSR_HI, v);
 301         /*
 302          * 64 bits of data will be pushed onto the FIFO every
 303          * 256 SEC cycles until the FIFO is full.  The RNG then
 304          * attempts to keep the FIFO full.
 305          */
 306         v = talitos_read(sc->sc_base_addr + TALITOS_RNGISR_HI);
 307         if (v) {
 308                 printk(KERN_ERR "%s: RNGISR_HI error %08x\n",
 309                         device_get_nameunit(sc->sc_cdev), v);
 310                 return;
 311         }
 312         /*
 313          * n.b. we need to add a FIPS test here - if the RNG is going
 314          * to fail, it's going to fail at reset time
 315          */
 316         return;
 317 }
 318 #endif /* CONFIG_OCF_RANDOMHARVEST */
 319
 320 /*
 321  * Generate a new software session.
 322  */
 323 static int
 324 talitos_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 325 {
 326         struct cryptoini *c, *encini = NULL, *macini = NULL;
 327         struct talitos_softc *sc = device_get_softc(dev);
 328         struct talitos_session *ses = NULL;
 329         int sesn;
 330
 331         DPRINTF("%s()\n", __FUNCTION__);
 332         if (sidp == NULL || cri == NULL || sc == NULL) {
 333                 DPRINTF("%s,%d - EINVAL\n", __FILE__, __LINE__);
 334                 return EINVAL;
 335         }
 336         for (c = cri; c != NULL; c = c->cri_next) {
 337                 if (c->cri_alg == CRYPTO_MD5 ||
 338                     c->cri_alg == CRYPTO_MD5_HMAC ||
 339                     c->cri_alg == CRYPTO_SHA1 ||
 340                     c->cri_alg == CRYPTO_SHA1_HMAC ||
 341                     c->cri_alg == CRYPTO_NULL_HMAC) {
 342                         if (macini)
 343                                 return EINVAL;
 344                         macini = c;
 345                 } else if (c->cri_alg == CRYPTO_DES_CBC ||
 346                     c->cri_alg == CRYPTO_3DES_CBC ||
 347                     c->cri_alg == CRYPTO_AES_CBC ||
 348                     c->cri_alg == CRYPTO_NULL_CBC) {
 349                         if (encini)
 350                                 return EINVAL;
 351                         encini = c;
 352                 } else {
 353                         DPRINTF("UNKNOWN c->cri_alg %d\n", encini->cri_alg);
 354                         return EINVAL;
 355                 }
 356         }
 357         if (encini == NULL && macini == NULL)
 358                 return EINVAL;
 359         if (encini) {
 360                 /* validate key length */
 361                 switch (encini->cri_alg) {
 362                 case CRYPTO_DES_CBC:
 363                         if (encini->cri_klen != 64)
 364                                 return EINVAL;
 365                         break;
 366                 case CRYPTO_3DES_CBC:
 367                         if (encini->cri_klen != 192) {
 368                                 return EINVAL;
 369                         }
 370                         break;
 371                 case CRYPTO_AES_CBC:
 372                         if (encini->cri_klen != 128 &&
 373                             encini->cri_klen != 192 &&
 374                             encini->cri_klen != 256)
 375                                 return EINVAL;
 376                         break;
 377                 default:
 378                         DPRINTF("UNKNOWN encini->cri_alg %d\n",
 379                                 encini->cri_alg);
 380                         return EINVAL;
 381                 }
 382         }
 383
 384         if (sc->sc_sessions == NULL) {
 385                 ses = sc->sc_sessions = (struct talitos_session *)
 386                         kmalloc(sizeof(struct talitos_session), SLAB_ATOMIC);
 387                 if (ses == NULL)
 388                         return ENOMEM;
 389                 memset(ses, 0, sizeof(struct talitos_session));
 390                 sesn = 0;
 391                 sc->sc_nsessions = 1;
 392         } else {
 393                 for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 394                         if (sc->sc_sessions[sesn].ses_used == 0) {
 395                                 ses = &sc->sc_sessions[sesn];
 396                                 break;
 397                         }
 398                 }
 399
 400                 if (ses == NULL) {
 401                         /* allocating session */
 402                         sesn = sc->sc_nsessions;
 403                         ses = (struct talitos_session *) kmalloc(
 404                                 (sesn + 1) * sizeof(struct talitos_session),
 405                                 SLAB_ATOMIC);
 406                         if (ses == NULL)
 407                                 return ENOMEM;
 408                         memset(ses, 0,
 409                                 (sesn + 1) * sizeof(struct talitos_session));
 410                         memcpy(ses, sc->sc_sessions,
 411                                 sesn * sizeof(struct talitos_session));
 412                         memset(sc->sc_sessions, 0,
 413                                 sesn * sizeof(struct talitos_session));
 414                         kfree(sc->sc_sessions);
 415                         sc->sc_sessions = ses;
 416                         ses = &sc->sc_sessions[sesn];
 417                         sc->sc_nsessions++;
 418                 }
 419         }
 420
 421         ses->ses_used = 1;
 422
 423         if (encini) {
 424                 /* get an IV */
 425                 /* XXX may read fewer than requested */
 426                 read_random(ses->ses_iv, sizeof(ses->ses_iv));
 427
 428                 ses->ses_klen = (encini->cri_klen + 7) / 8;
 429                 memcpy(ses->ses_key, encini->cri_key, ses->ses_klen);
 430                 if (macini) {
 431                         /* doing hash on top of cipher */
 432                         ses->ses_hmac_len = (macini->cri_klen + 7) / 8;
 433                         memcpy(ses->ses_hmac, macini->cri_key,
 434                                 ses->ses_hmac_len);
 435                 }
 436         } else if (macini) {
 437                 /* doing hash */
 438                 ses->ses_klen = (macini->cri_klen + 7) / 8;
 439                 memcpy(ses->ses_key, macini->cri_key, ses->ses_klen);
 440         }
 441
 442         /* back compat way of determining MSC result len */
 443         if (macini) {
 444                 ses->ses_mlen = macini->cri_mlen;
 445                 if (ses->ses_mlen == 0) {
 446                         if (macini->cri_alg == CRYPTO_MD5_HMAC)
 447                                 ses->ses_mlen = MD5_HASH_LEN;
 448                         else
 449                                 ses->ses_mlen = SHA1_HASH_LEN;
 450                 }
 451         }
 452
 453         /* really should make up a template td here,
 454          * and only fill things like i/o and direction in process() */
 455
 456         /* assign session ID */
 457         *sidp = TALITOS_SID(sc->sc_num, sesn);
 458         return 0;
 459 }
 460
 461 /*
 462  * Deallocate a session.
 463  */
 464 static int
 465 talitos_freesession(device_t dev, u_int64_t tid)
 466 {
 467         struct talitos_softc *sc = device_get_softc(dev);
 468         int session, ret;
 469         u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 470
 471         if (sc == NULL)
 472                 return EINVAL;
 473         session = TALITOS_SESSION(sid);
 474         if (session < sc->sc_nsessions) {
 475                 memset(&sc->sc_sessions[session], 0,
 476                         sizeof(sc->sc_sessions[session]));
 477                 ret = 0;
 478         } else
 479                 ret = EINVAL;
 480         return ret;
 481 }
 482
 483 /*
 484  * launch device processing - it will come back with done notification
 485  * in the form of an interrupt and/or HDR_DONE_BITS in header
 486  */
 487 static int
 488 talitos_submit(
 489         struct talitos_softc *sc,
 490         struct talitos_desc *td,
 491         int chsel)
 492 {
 493         u_int32_t v;
 494
 495         v = dma_map_single(NULL, td, sizeof(*td), DMA_TO_DEVICE);
 496         talitos_write(sc->sc_base_addr +
 497                 chsel*TALITOS_CH_OFFSET + TALITOS_CH_FF, 0);
 498         talitos_write(sc->sc_base_addr +
 499                 chsel*TALITOS_CH_OFFSET + TALITOS_CH_FF_HI, v);
 500         return 0;
 501 }
 502
 503 static int
 504 talitos_process(device_t dev, struct cryptop *crp, int hint)
 505 {
 506         int i, err = 0, ivsize;
 507         struct talitos_softc *sc = device_get_softc(dev);
 508         struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 509         caddr_t iv;
 510         struct talitos_session *ses;
 511         struct talitos_desc *td;
 512         unsigned long flags;
 513         /* descriptor mappings */
 514         int hmac_key, hmac_data, cipher_iv, cipher_key,
 515                 in_fifo, out_fifo, cipher_iv_out;
 516         static int chsel = -1;
 517
 518         DPRINTF("%s()\n", __FUNCTION__);
 519
 520         if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 521                 return EINVAL;
 522         }
 523         crp->crp_etype = 0;
 524         if (TALITOS_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 525                 return EINVAL;
 526         }
 527
 528         ses = &sc->sc_sessions[TALITOS_SESSION(crp->crp_sid)];
 529
 530         /* enter the channel scheduler */
 531         spin_lock_irqsave(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 532
 533         /* reuse channel that already had/has requests for the required EU */
 534         for (i = 0; i < sc->sc_num_channels; i++) {
 535                 if (sc->sc_chnlastalg[i] == crp->crp_desc->crd_alg)
 536                         break;
 537         }
 538         if (i == sc->sc_num_channels) {
 539                 /*
 540                  * haven't seen this algo the last sc_num_channels or more
 541                  * use round robin in this case
 542                  * nb: sc->sc_num_channels must be power of 2
 543                  */
 544                 chsel = (chsel + 1) & (sc->sc_num_channels - 1);
 545         } else {
 546                 /*
 547                  * matches channel with same target execution unit;
 548                  * use same channel in this case
 549                  */
 550                 chsel = i;
 551         }
 552         sc->sc_chnlastalg[chsel] = crp->crp_desc->crd_alg;
 553
 554         /* release the channel scheduler lock */
 555         spin_unlock_irqrestore(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 556
 557         /* acquire the selected channel fifo lock */
 558         spin_lock_irqsave(&sc->sc_chnfifolock[chsel], flags);
 559
 560         /* find and reserve next available descriptor-cryptop pair */
 561         for (i = 0; i < sc->sc_chfifo_len; i++) {
 562                 if (sc->sc_chnfifo[chsel][i].cf_desc.hdr == 0) {
 563                         /*
 564                          * ensure correct descriptor formation by
 565                          * avoiding inadvertently setting "optional" entries
 566                          * e.g. not using "optional" dptr2 for MD/HMAC descs
 567                          */
 568                         memset(&sc->sc_chnfifo[chsel][i].cf_desc,
 569                                 0, sizeof(*td));
 570                         /* reserve it with done notification request bit */
 571                         sc->sc_chnfifo[chsel][i].cf_desc.hdr |=
 572                                 TALITOS_DONE_NOTIFY;
 573                         break;
 574                 }
 575         }
 576         spin_unlock_irqrestore(&sc->sc_chnfifolock[chsel], flags);
 577
 578         if (i == sc->sc_chfifo_len) {
 579                 /* fifo full */
 580                 err = ERESTART;
 581                 goto errout;
 582         }
 583
 584         td = &sc->sc_chnfifo[chsel][i].cf_desc;
 585         sc->sc_chnfifo[chsel][i].cf_crp = crp;
 586
 587         crd1 = crp->crp_desc;
 588         if (crd1 == NULL) {
 589                 err = EINVAL;
 590                 goto errout;
 591         }
 592         crd2 = crd1->crd_next;
 593         /* prevent compiler warning */
 594         hmac_key = 0;
 595         hmac_data = 0;
 596         if (crd2 == NULL) {
 597                 td->hdr |= TD_TYPE_COMMON_NONSNOOP_NO_AFEU;
 598                 /* assign descriptor dword ptr mappings for this desc. type */
 599                 cipher_iv = 1;
 600                 cipher_key = 2;
 601                 in_fifo = 3;
 602                 cipher_iv_out = 5;
 603                 if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 604                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 605                     crd1->crd_alg == CRYPTO_SHA1 ||
 606                     crd1->crd_alg == CRYPTO_MD5) {
 607                         out_fifo = 5;
 608                         maccrd = crd1;
 609                         enccrd = NULL;
 610                 } else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 611                     crd1->crd_alg == CRYPTO_3DES_CBC ||
 612                     crd1->crd_alg == CRYPTO_AES_CBC ||
 613                     crd1->crd_alg == CRYPTO_ARC4) {
 614                         out_fifo = 4;
 615                         maccrd = NULL;
 616                         enccrd = crd1;
 617                 } else {
 618                         DPRINTF("UNKNOWN crd1->crd_alg %d\n", crd1->crd_alg);
 619                         err = EINVAL;
 620                         goto errout;
 621                 }
 622         } else {
 623                 if (sc->sc_desc_types & TALITOS_HAS_DT_IPSEC_ESP) {
 624                         td->hdr |= TD_TYPE_IPSEC_ESP;
 625                 } else {
 626                         DPRINTF("unimplemented: multiple descriptor ipsec\n");
 627                         err = EINVAL;
 628                         goto errout;
 629                 }
 630                 /* assign descriptor dword ptr mappings for this desc. type */
 631                 hmac_key = 0;
 632                 hmac_data = 1;
 633                 cipher_iv = 2;
 634                 cipher_key = 3;
 635                 in_fifo = 4;
 636                 out_fifo = 5;
 637                 cipher_iv_out = 6;
 638                 if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 639                      crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 640                      crd1->crd_alg == CRYPTO_MD5 ||
 641                      crd1->crd_alg == CRYPTO_SHA1) &&
 642                     (crd2->crd_alg == CRYPTO_DES_CBC ||
 643                      crd2->crd_alg == CRYPTO_3DES_CBC ||
 644                      crd2->crd_alg == CRYPTO_AES_CBC ||
 645                      crd2->crd_alg == CRYPTO_ARC4) &&
 646                     ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 647                         maccrd = crd1;
 648                         enccrd = crd2;
 649                 } else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 650                      crd1->crd_alg == CRYPTO_ARC4 ||
 651                      crd1->crd_alg == CRYPTO_3DES_CBC ||
 652                      crd1->crd_alg == CRYPTO_AES_CBC) &&
 653                     (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 654                      crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 655                      crd2->crd_alg == CRYPTO_MD5 ||
 656                      crd2->crd_alg == CRYPTO_SHA1) &&
 657                     (crd1->crd_flags & CRD_F_ENCRYPT)) {
 658                         enccrd = crd1;
 659                         maccrd = crd2;
 660                 } else {
 661                         /* We cannot order the SEC as requested */
 662                         printk("%s: cannot do the order\n",
 663                                         device_get_nameunit(sc->sc_cdev));
 664                         err = EINVAL;
 665                         goto errout;
 666                 }
 667         }
 668         /* assign in_fifo and out_fifo based on input/output struct type */
 669         if (crp->crp_flags & CRYPTO_F_SKBUF) {
 670                 /* using SKB buffers */
 671                 struct sk_buff *skb = (struct sk_buff *)crp->crp_buf;
 672                 if (skb_shinfo(skb)->nr_frags) {
 673                         printk("%s: skb frags unimplemented\n",
 674                                         device_get_nameunit(sc->sc_cdev));
 675                         err = EINVAL;
 676                         goto errout;
 677                 }
 678                 td->ptr[in_fifo].ptr = dma_map_single(NULL, skb->data,
 679                         skb->len, DMA_TO_DEVICE);
 680                 td->ptr[in_fifo].len = skb->len;
 681                 td->ptr[out_fifo].ptr = dma_map_single(NULL, skb->data,
 682                         skb->len, DMA_TO_DEVICE);
 683                 td->ptr[out_fifo].len = skb->len;
 684                 td->ptr[hmac_data].ptr = dma_map_single(NULL, skb->data,
 685                         skb->len, DMA_TO_DEVICE);
 686         } else if (crp->crp_flags & CRYPTO_F_IOV) {
 687                 /* using IOV buffers */
 688                 struct uio *uiop = (struct uio *)crp->crp_buf;
 689                 if (uiop->uio_iovcnt > 1) {
 690                         printk("%s: iov frags unimplemented\n",
 691                                         device_get_nameunit(sc->sc_cdev));
 692                         err = EINVAL;
 693                         goto errout;
 694                 }
 695                 td->ptr[in_fifo].ptr = dma_map_single(NULL,
 696                         uiop->uio_iov->iov_base, crp->crp_ilen, DMA_TO_DEVICE);
 697                 td->ptr[in_fifo].len = crp->crp_ilen;
 698                 /* crp_olen is never set; always use crp_ilen */
 699                 td->ptr[out_fifo].ptr = dma_map_single(NULL,
 700                         uiop->uio_iov->iov_base,
 701                         crp->crp_ilen, DMA_TO_DEVICE);
 702                 td->ptr[out_fifo].len = crp->crp_ilen;
 703         } else {
 704                 /* using contig buffers */
 705                 td->ptr[in_fifo].ptr = dma_map_single(NULL,
 706                         crp->crp_buf, crp->crp_ilen, DMA_TO_DEVICE);
 707                 td->ptr[in_fifo].len = crp->crp_ilen;
 708                 td->ptr[out_fifo].ptr = dma_map_single(NULL,
 709                         crp->crp_buf, crp->crp_ilen, DMA_TO_DEVICE);
 710                 td->ptr[out_fifo].len = crp->crp_ilen;
 711         }
 712         if (enccrd) {
 713                 switch (enccrd->crd_alg) {
 714                 case CRYPTO_3DES_CBC:
 715                         td->hdr |= TALITOS_MODE0_DEU_3DES;
 716                         /* FALLTHROUGH */
 717                 case CRYPTO_DES_CBC:
 718                         td->hdr |= TALITOS_SEL0_DEU
 719                                 |  TALITOS_MODE0_DEU_CBC;
 720                         if (enccrd->crd_flags & CRD_F_ENCRYPT)
 721                                 td->hdr |= TALITOS_MODE0_DEU_ENC;
 722                         ivsize = 2*sizeof(u_int32_t);
 723                         DPRINTF("%cDES ses %d ch %d len %d\n",
 724                                 (td->hdr & TALITOS_MODE0_DEU_3DES)?'3':'1',
 725                                 (u32)TALITOS_SESSION(crp->crp_sid),
 726                                 chsel, td->ptr[in_fifo].len);
 727                         break;
 728                 case CRYPTO_AES_CBC:
 729                         td->hdr |= TALITOS_SEL0_AESU
 730                                 |  TALITOS_MODE0_AESU_CBC;
 731                         if (enccrd->crd_flags & CRD_F_ENCRYPT)
 732                                 td->hdr |= TALITOS_MODE0_AESU_ENC;
 733                         ivsize = 4*sizeof(u_int32_t);
 734                         DPRINTF("AES  ses %d ch %d len %d\n",
 735                                 (u32)TALITOS_SESSION(crp->crp_sid),
 736                                 chsel, td->ptr[in_fifo].len);
 737                         break;
 738                 default:
 739                         printk("%s: unimplemented enccrd->crd_alg %d\n",
 740                                         device_get_nameunit(sc->sc_cdev), enccrd->crd_alg);
 741                         err = EINVAL;
 742                         goto errout;
 743                 }
 744                 /*
 745                  * Setup encrypt/decrypt state.  When using basic ops
 746                  * we can't use an inline IV because hash/crypt offset
 747                  * must be from the end of the IV to the start of the
 748                  * crypt data and this leaves out the preceding header
 749                  * from the hash calculation.  Instead we place the IV
 750                  * in the state record and set the hash/crypt offset to
 751                  * copy both the header+IV.
 752                  */
 753                 if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 754                         td->hdr |= TALITOS_DIR_OUTBOUND;
 755                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 756                                 iv = enccrd->crd_iv;
 757                         else
 758                                 iv = (caddr_t) ses->ses_iv;
 759                         if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 760                                 crypto_copyback(crp->crp_flags, crp->crp_buf,
 761                                     enccrd->crd_inject, ivsize, iv);
 762                         }
 763                 } else {
 764                         td->hdr |= TALITOS_DIR_INBOUND;
 765                         if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 766                                 iv = enccrd->crd_iv;
 767                                 bcopy(enccrd->crd_iv, iv, ivsize);
 768                         } else {
 769                                 iv = (caddr_t) ses->ses_iv;
 770                                 crypto_copydata(crp->crp_flags, crp->crp_buf,
 771                                     enccrd->crd_inject, ivsize, iv);
 772                         }
 773                 }
 774                 td->ptr[cipher_iv].ptr = dma_map_single(NULL, iv, ivsize,
 775                         DMA_TO_DEVICE);
 776                 td->ptr[cipher_iv].len = ivsize;
 777                 /*
 778                  * we don't need the cipher iv out length/pointer
 779                  * field to do ESP IPsec. Therefore we set the len field as 0,
 780                  * which tells the SEC not to do anything with this len/ptr
 781                  * field. Previously, when length/pointer as pointing to iv,
 782                  * it gave us corruption of packets.
 783                  */
 784                 td->ptr[cipher_iv_out].len = 0;
 785         }
 786         if (enccrd && maccrd) {
 787                 /* this is ipsec only for now */
 788                 td->hdr |= TALITOS_SEL1_MDEU
 789                         |  TALITOS_MODE1_MDEU_INIT
 790                         |  TALITOS_MODE1_MDEU_PAD;
 791                 switch (maccrd->crd_alg) {
 792                         case    CRYPTO_MD5:
 793                                 td->hdr |= TALITOS_MODE1_MDEU_MD5;
 794                                 break;
 795                         case    CRYPTO_MD5_HMAC:
 796                                 td->hdr |= TALITOS_MODE1_MDEU_MD5_HMAC;
 797                                 break;
 798                         case    CRYPTO_SHA1:
 799                                 td->hdr |= TALITOS_MODE1_MDEU_SHA1;
 800                                 break;
 801                         case    CRYPTO_SHA1_HMAC:
 802                                 td->hdr |= TALITOS_MODE1_MDEU_SHA1_HMAC;
 803                                 break;
 804                         default:
 805                                 /* We cannot order the SEC as requested */
 806                                 printk("%s: cannot do the order\n",
 807                                                 device_get_nameunit(sc->sc_cdev));
 808                                 err = EINVAL;
 809                                 goto errout;
 810                 }
 811                 if ((maccrd->crd_alg == CRYPTO_MD5_HMAC) ||
 812                    (maccrd->crd_alg == CRYPTO_SHA1_HMAC)) {
 813                         /*
 814                          * The offset from hash data to the start of
 815                          * crypt data is the difference in the skips.
 816                          */
 817                         /* ipsec only for now */
 818                         td->ptr[hmac_key].ptr = dma_map_single(NULL,
 819                                 ses->ses_hmac, ses->ses_hmac_len, DMA_TO_DEVICE);
 820                         td->ptr[hmac_key].len = ses->ses_hmac_len;
 821                         td->ptr[in_fifo].ptr  += enccrd->crd_skip;
 822                         td->ptr[in_fifo].len  =  enccrd->crd_len;
 823                         td->ptr[out_fifo].ptr += enccrd->crd_skip;
 824                         td->ptr[out_fifo].len =  enccrd->crd_len;
 825                         /* bytes of HMAC to postpend to ciphertext */
 826                         td->ptr[out_fifo].extent =  ses->ses_mlen;
 827                         td->ptr[hmac_data].ptr += maccrd->crd_skip;
 828                         td->ptr[hmac_data].len = enccrd->crd_skip - maccrd->crd_skip;
 829                 }
 830                 if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 831                         printk("%s: CRD_F_KEY_EXPLICIT unimplemented\n",
 832                                         device_get_nameunit(sc->sc_cdev));
 833                 }
 834         }
 835         if (!enccrd && maccrd) {
 836                 /* single MD5 or SHA */
 837                 td->hdr |= TALITOS_SEL0_MDEU
 838                                 |  TALITOS_MODE0_MDEU_INIT
 839                                 |  TALITOS_MODE0_MDEU_PAD;
 840                 switch (maccrd->crd_alg) {
 841                         case    CRYPTO_MD5:
 842                                 td->hdr |= TALITOS_MODE0_MDEU_MD5;
 843                                 DPRINTF("MD5  ses %d ch %d len %d\n",
 844                                         (u32)TALITOS_SESSION(crp->crp_sid),
 845                                         chsel, td->ptr[in_fifo].len);
 846                                 break;
 847                         case    CRYPTO_MD5_HMAC:
 848                                 td->hdr |= TALITOS_MODE0_MDEU_MD5_HMAC;
 849                                 break;
 850                         case    CRYPTO_SHA1:
 851                                 td->hdr |= TALITOS_MODE0_MDEU_SHA1;
 852                                 DPRINTF("SHA1 ses %d ch %d len %d\n",
 853                                         (u32)TALITOS_SESSION(crp->crp_sid),
 854                                         chsel, td->ptr[in_fifo].len);
 855                                 break;
 856                         case    CRYPTO_SHA1_HMAC:
 857                                 td->hdr |= TALITOS_MODE0_MDEU_SHA1_HMAC;
 858                                 break;
 859                         default:
 860                                 /* We cannot order the SEC as requested */
 861                                 DPRINTF("cannot do the order\n");
 862                                 err = EINVAL;
 863                                 goto errout;
 864                 }
 865
 866                 if (crp->crp_flags & CRYPTO_F_IOV)
 867                         td->ptr[out_fifo].ptr += maccrd->crd_inject;
 868
 869                 if ((maccrd->crd_alg == CRYPTO_MD5_HMAC) ||
 870                    (maccrd->crd_alg == CRYPTO_SHA1_HMAC)) {
 871                         td->ptr[hmac_key].ptr = dma_map_single(NULL,
 872                                 ses->ses_hmac, ses->ses_hmac_len,
 873                                 DMA_TO_DEVICE);
 874                         td->ptr[hmac_key].len = ses->ses_hmac_len;
 875                 }
 876         }
 877         else {
 878                 /* using process key (session data has duplicate) */
 879                 td->ptr[cipher_key].ptr = dma_map_single(NULL,
 880                         enccrd->crd_key, (enccrd->crd_klen + 7) / 8,
 881                         DMA_TO_DEVICE);
 882                 td->ptr[cipher_key].len = (enccrd->crd_klen + 7) / 8;
 883         }
 884         /* descriptor complete - GO! */
 885         return talitos_submit(sc, td, chsel);
 886
 887 errout:
 888         if (err != ERESTART) {
 889                 crp->crp_etype = err;
 890                 crypto_done(crp);
 891         }
 892         return err;
 893 }
 894
 895 /* go through all channels descriptors, notifying OCF what has
 896  * _and_hasn't_ successfully completed and reset the device
 897  * (otherwise it's up to decoding desc hdrs!)
 898  */
 899 static void talitos_errorprocessing(struct talitos_softc *sc)
 900 {
 901         unsigned long flags;
 902         int i, j;
 903
 904         /* disable further scheduling until under control */
 905         spin_lock_irqsave(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 906
 907         if (debug) dump_talitos_status(sc);
 908         /* go through descriptors, try and salvage those successfully done,
 909          * and EIO those that weren't
 910          */
 911         for (i = 0; i < sc->sc_num_channels; i++) {
 912                 spin_lock_irqsave(&sc->sc_chnfifolock[i], flags);
 913                 for (j = 0; j < sc->sc_chfifo_len; j++) {
 914                         if (sc->sc_chnfifo[i][j].cf_desc.hdr) {
 915                                 if ((sc->sc_chnfifo[i][j].cf_desc.hdr
 916                                         & TALITOS_HDR_DONE_BITS)
 917                                         != TALITOS_HDR_DONE_BITS) {
 918                                         /* this one didn't finish */
 919                                         /* signify in crp->etype */
 920                                         sc->sc_chnfifo[i][j].cf_crp->crp_etype
 921                                                 = EIO;
 922                                 }
 923                         } else
 924                                 continue; /* free entry */
 925                         /* either way, notify ocf */
 926                         crypto_done(sc->sc_chnfifo[i][j].cf_crp);
 927                         /* and tag it available again
 928                          *
 929                          * memset to ensure correct descriptor formation by
 930                          * avoiding inadvertently setting "optional" entries
 931                          * e.g. not using "optional" dptr2 MD/HMAC processing
 932                          */
 933                         memset(&sc->sc_chnfifo[i][j].cf_desc,
 934                                 0, sizeof(struct talitos_desc));
 935                 }
 936                 spin_unlock_irqrestore(&sc->sc_chnfifolock[i], flags);
 937         }
 938         /* reset and initialize the SEC h/w device */
 939         talitos_reset_device(sc);
 940         talitos_init_device(sc);
 941 #ifdef CONFIG_OCF_RANDOMHARVEST
 942         if (sc->sc_exec_units & TALITOS_HAS_EU_RNG)
 943                 talitos_rng_init(sc);
 944 #endif
 945
 946         /* Okay. Stand by. */
 947         spin_unlock_irqrestore(&sc->sc_chnfifolock[sc->sc_num_channels], flags);
 948
 949         return;
 950 }
 951
 952 /* go through all channels descriptors, notifying OCF what's been done */
 953 static void talitos_doneprocessing(struct talitos_softc *sc)
 954 {
 955         unsigned long flags;
 956         int i, j;
 957
 958         /* go through descriptors looking for done bits */
 959         for (i = 0; i < sc->sc_num_channels; i++) {
 960                 spin_lock_irqsave(&sc->sc_chnfifolock[i], flags);
 961                 for (j = 0; j < sc->sc_chfifo_len; j++) {
 962                         /* descriptor has done bits set? */
 963                         if ((sc->sc_chnfifo[i][j].cf_desc.hdr
 964                                 & TALITOS_HDR_DONE_BITS)
 965                                 == TALITOS_HDR_DONE_BITS) {
 966                                 /* notify ocf */
 967                                 crypto_done(sc->sc_chnfifo[i][j].cf_crp);
 968                                 /* and tag it available again
 969                                  *
 970                                  * memset to ensure correct descriptor formation by
 971                                  * avoiding inadvertently setting "optional" entries
 972                                  * e.g. not using "optional" dptr2 MD/HMAC processing
 973                                  */
 974                                 memset(&sc->sc_chnfifo[i][j].cf_desc,
 975                                         0, sizeof(struct talitos_desc));
 976                         }
 977                 }
 978                 spin_unlock_irqrestore(&sc->sc_chnfifolock[i], flags);
 979         }
 980         return;
 981 }
 982
 983 static irqreturn_t
 984 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,19)
 985 talitos_intr(int irq, void *arg)
 986 #else
 987 talitos_intr(int irq, void *arg, struct pt_regs *regs)
 988 #endif
 989 {
 990         struct talitos_softc *sc = arg;
 991         u_int32_t v, v_hi;
 992
 993         /* ack */
 994         v = talitos_read(sc->sc_base_addr + TALITOS_ISR);
 995         v_hi = talitos_read(sc->sc_base_addr + TALITOS_ISR_HI);
 996         talitos_write(sc->sc_base_addr + TALITOS_ICR, v);
 997         talitos_write(sc->sc_base_addr + TALITOS_ICR_HI, v_hi);
 998
 999         if (unlikely(v & TALITOS_ISR_ERROR)) {
1000                 /* Okay, Houston, we've had a problem here. */
1001                 printk(KERN_DEBUG "%s: got error interrupt - ISR 0x%08x_%08x\n",
1002                                 device_get_nameunit(sc->sc_cdev), v, v_hi);
1003                 talitos_errorprocessing(sc);
1004         } else
1005         if (likely(v & TALITOS_ISR_DONE)) {
1006                 talitos_doneprocessing(sc);
1007         }
1008         return IRQ_HANDLED;
1009 }
1010
1011 /*
1012  * Initialize registers we need to touch only once.
1013  */
1014 static void
1015 talitos_init_device(struct talitos_softc *sc)
1016 {
1017         u_int32_t v;
1018         int i;
1019
1020         DPRINTF("%s()\n", __FUNCTION__);
1021
1022         /* init all channels */
1023         for (i = 0; i < sc->sc_num_channels; i++) {
1024                 v = talitos_read(sc->sc_base_addr +
1025                         i*TALITOS_CH_OFFSET + TALITOS_CH_CCCR_HI);
1026                 v |= TALITOS_CH_CCCR_HI_CDWE
1027                   |  TALITOS_CH_CCCR_HI_CDIE;  /* invoke interrupt if done */
1028                 talitos_write(sc->sc_base_addr +
1029                         i*TALITOS_CH_OFFSET + TALITOS_CH_CCCR_HI, v);
1030         }
1031         /* enable all interrupts */
1032         v = talitos_read(sc->sc_base_addr + TALITOS_IMR);
1033         v |= TALITOS_IMR_ALL;
1034         talitos_write(sc->sc_base_addr + TALITOS_IMR, v);
1035         v = talitos_read(sc->sc_base_addr + TALITOS_IMR_HI);
1036         v |= TALITOS_IMR_HI_ERRONLY;
1037         talitos_write(sc->sc_base_addr + TALITOS_IMR_HI, v);
1038         return;
1039 }
1040
1041 /*
1042  * set the master reset bit on the device.
1043  */
1044 static void
1045 talitos_reset_device_master(struct talitos_softc *sc)
1046 {
1047         u_int32_t v;
1048
1049         /* Reset the device by writing 1 to MCR:SWR and waiting 'til cleared */
1050         v = talitos_read(sc->sc_base_addr + TALITOS_MCR);
1051         talitos_write(sc->sc_base_addr + TALITOS_MCR, v | TALITOS_MCR_SWR);
1052
1053         while (talitos_read(sc->sc_base_addr + TALITOS_MCR) & TALITOS_MCR_SWR)
1054                 cpu_relax();
1055
1056         return;
1057 }
1058
1059 /*
1060  * Resets the device.  Values in the registers are left as is
1061  * from the reset (i.e. initial values are assigned elsewhere).
1062  */
1063 static void
1064 talitos_reset_device(struct talitos_softc *sc)
1065 {
1066         u_int32_t v;
1067         int i;
1068
1069         DPRINTF("%s()\n", __FUNCTION__);
1070
1071         /*
1072          * Master reset
1073          * errata documentation: warning: certain SEC interrupts
1074          * are not fully cleared by writing the MCR:SWR bit,
1075          * set bit twice to completely reset
1076          */
1077         talitos_reset_device_master(sc);        /* once */
1078         talitos_reset_device_master(sc);        /* and once again */
1079
1080         /* reset all channels */
1081         for (i = 0; i < sc->sc_num_channels; i++) {
1082                 v = talitos_read(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
1083                         TALITOS_CH_CCCR);
1084                 talitos_write(sc->sc_base_addr + i*TALITOS_CH_OFFSET +
1085                         TALITOS_CH_CCCR, v | TALITOS_CH_CCCR_RESET);
1086         }
1087 }
1088
1089 /* Set up the crypto device structure, private data,
1090  * and anything else we need before we start */
1091 #ifdef CONFIG_PPC_MERGE
1092 static int talitos_probe(struct of_device *ofdev, const struct of_device_id *match)
1093 #else
1094 static int talitos_probe(struct platform_device *pdev)
1095 #endif
1096 {
1097         struct talitos_softc *sc = NULL;
1098         struct resource *r;
1099 #ifdef CONFIG_PPC_MERGE
1100         struct device *device = &ofdev->dev;
1101         struct device_node *np = ofdev->node;
1102         const unsigned int *prop;
1103         int err;
1104         struct resource res;
1105 #endif
1106         static int num_chips = 0;
1107         int rc;
1108         int i;
1109
1110         DPRINTF("%s()\n", __FUNCTION__);
1111
1112         sc = (struct talitos_softc *) kmalloc(sizeof(*sc), GFP_KERNEL);
1113         if (!sc)
1114                 return -ENOMEM;
1115         memset(sc, 0, sizeof(*sc));
1116
1117         softc_device_init(sc, DRV_NAME, num_chips, talitos_methods);
1118
1119         sc->sc_irq = -1;
1120         sc->sc_cid = -1;
1121 #ifndef CONFIG_PPC_MERGE
1122         sc->sc_dev = pdev;
1123 #endif
1124         sc->sc_num = num_chips++;
1125
1126 #ifdef CONFIG_PPC_MERGE
1127         dev_set_drvdata(device, sc);
1128 #else
1129         platform_set_drvdata(sc->sc_dev, sc);
1130 #endif
1131
1132         /* get the irq line */
1133 #ifdef CONFIG_PPC_MERGE
1134         err = of_address_to_resource(np, 0, &res);
1135         if (err)
1136                 return -EINVAL;
1137         r = &res;
1138
1139         sc->sc_irq = irq_of_parse_and_map(np, 0);
1140 #else
1141         /* get a pointer to the register memory */
1142         r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1143
1144         sc->sc_irq = platform_get_irq(pdev, 0);
1145 #endif
1146         rc = request_irq(sc->sc_irq, talitos_intr, 0,
1147                         device_get_nameunit(sc->sc_cdev), sc);
1148         if (rc) {
1149                 printk(KERN_ERR "%s: failed to hook irq %d\n",
1150                                 device_get_nameunit(sc->sc_cdev), sc->sc_irq);
1151                 sc->sc_irq = -1;
1152                 goto out;
1153         }
1154
1155         sc->sc_base_addr = (ocf_iomem_t) ioremap(r->start, (r->end - r->start));
1156         if (!sc->sc_base_addr) {
1157                 printk(KERN_ERR "%s: failed to ioremap\n",
1158                                 device_get_nameunit(sc->sc_cdev));
1159                 goto out;
1160         }
1161
1162         /* figure out our SEC's properties and capabilities */
1163         sc->sc_chiprev = (u64)talitos_read(sc->sc_base_addr + TALITOS_ID) << 32
1164                  | talitos_read(sc->sc_base_addr + TALITOS_ID_HI);
1165         DPRINTF("sec id 0x%llx\n", sc->sc_chiprev);
1166
1167 #ifdef CONFIG_PPC_MERGE
1168         /* get SEC properties from device tree, defaulting to SEC 2.0 */
1169
1170         prop = of_get_property(np, "num-channels", NULL);
1171         sc->sc_num_channels = prop ? *prop : TALITOS_NCHANNELS_SEC_2_0;
1172
1173         prop = of_get_property(np, "channel-fifo-len", NULL);
1174         sc->sc_chfifo_len = prop ? *prop : TALITOS_CHFIFOLEN_SEC_2_0;
1175
1176         prop = of_get_property(np, "exec-units-mask", NULL);
1177         sc->sc_exec_units = prop ? *prop : TALITOS_HAS_EUS_SEC_2_0;
1178
1179         prop = of_get_property(np, "descriptor-types-mask", NULL);
1180         sc->sc_desc_types = prop ? *prop : TALITOS_HAS_DESCTYPES_SEC_2_0;
1181 #else
1182         /* bulk should go away with openfirmware flat device tree support */
1183         if (sc->sc_chiprev & TALITOS_ID_SEC_2_0) {
1184                 sc->sc_num_channels = TALITOS_NCHANNELS_SEC_2_0;
1185                 sc->sc_chfifo_len = TALITOS_CHFIFOLEN_SEC_2_0;
1186                 sc->sc_exec_units = TALITOS_HAS_EUS_SEC_2_0;
1187                 sc->sc_desc_types = TALITOS_HAS_DESCTYPES_SEC_2_0;
1188         } else {
1189                 printk(KERN_ERR "%s: failed to id device\n",
1190                                 device_get_nameunit(sc->sc_cdev));
1191                 goto out;
1192         }
1193 #endif
1194
1195         /* + 1 is for the meta-channel lock used by the channel scheduler */
1196         sc->sc_chnfifolock = (spinlock_t *) kmalloc(
1197                 (sc->sc_num_channels + 1) * sizeof(spinlock_t), GFP_KERNEL);
1198         if (!sc->sc_chnfifolock)
1199                 goto out;
1200         for (i = 0; i < sc->sc_num_channels + 1; i++) {
1201                 spin_lock_init(&sc->sc_chnfifolock[i]);
1202         }
1203
1204         sc->sc_chnlastalg = (int *) kmalloc(
1205                 sc->sc_num_channels * sizeof(int), GFP_KERNEL);
1206         if (!sc->sc_chnlastalg)
1207                 goto out;
1208         memset(sc->sc_chnlastalg, 0, sc->sc_num_channels * sizeof(int));
1209
1210         sc->sc_chnfifo = (struct desc_cryptop_pair **) kmalloc(
1211                 sc->sc_num_channels * sizeof(struct desc_cryptop_pair *),
1212                 GFP_KERNEL);
1213         if (!sc->sc_chnfifo)
1214                 goto out;
1215         for (i = 0; i < sc->sc_num_channels; i++) {
1216                 sc->sc_chnfifo[i] = (struct desc_cryptop_pair *) kmalloc(
1217                         sc->sc_chfifo_len * sizeof(struct desc_cryptop_pair),
1218                         GFP_KERNEL);
1219                 if (!sc->sc_chnfifo[i])
1220                         goto out;
1221                 memset(sc->sc_chnfifo[i], 0,
1222                         sc->sc_chfifo_len * sizeof(struct desc_cryptop_pair));
1223         }
1224
1225         /* reset and initialize the SEC h/w device */
1226         talitos_reset_device(sc);
1227         talitos_init_device(sc);
1228
1229         sc->sc_cid = crypto_get_driverid(softc_get_device(sc),CRYPTOCAP_F_HARDWARE);
1230         if (sc->sc_cid < 0) {
1231                 printk(KERN_ERR "%s: could not get crypto driver id\n",
1232                                 device_get_nameunit(sc->sc_cdev));
1233                 goto out;
1234         }
1235
1236         /* register algorithms with the framework */
1237         printk("%s:", device_get_nameunit(sc->sc_cdev));
1238
1239         if (sc->sc_exec_units & TALITOS_HAS_EU_RNG)  {
1240                 printk(" rng");
1241 #ifdef CONFIG_OCF_RANDOMHARVEST
1242                 talitos_rng_init(sc);
1243                 crypto_rregister(sc->sc_cid, talitos_read_random, sc);
1244 #endif
1245         }
1246         if (sc->sc_exec_units & TALITOS_HAS_EU_DEU) {
1247                 printk(" des/3des");
1248                 crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
1249                 crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
1250         }
1251         if (sc->sc_exec_units & TALITOS_HAS_EU_AESU) {
1252                 printk(" aes");
1253                 crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
1254         }
1255         if (sc->sc_exec_units & TALITOS_HAS_EU_MDEU) {
1256                 printk(" md5");
1257                 crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
1258                 /* HMAC support only with IPsec for now */
1259                 crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
1260                 printk(" sha1");
1261                 crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
1262                 /* HMAC support only with IPsec for now */
1263                 crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
1264         }
1265         printk("\n");
1266         return 0;
1267
1268 out:
1269 #ifndef CONFIG_PPC_MERGE
1270         talitos_remove(pdev);
1271 #endif
1272         return -ENOMEM;
1273 }
1274
1275 #ifdef CONFIG_PPC_MERGE
1276 static int talitos_remove(struct of_device *ofdev)
1277 #else
1278 static int talitos_remove(struct platform_device *pdev)
1279 #endif
1280 {
1281 #ifdef CONFIG_PPC_MERGE
1282         struct talitos_softc *sc = dev_get_drvdata(&ofdev->dev);
1283 #else
1284         struct talitos_softc *sc = platform_get_drvdata(pdev);
1285 #endif
1286         int i;
1287
1288         DPRINTF("%s()\n", __FUNCTION__);
1289         if (sc->sc_cid >= 0)
1290                 crypto_unregister_all(sc->sc_cid);
1291         if (sc->sc_chnfifo) {
1292                 for (i = 0; i < sc->sc_num_channels; i++)
1293                         if (sc->sc_chnfifo[i])
1294                                 kfree(sc->sc_chnfifo[i]);
1295                 kfree(sc->sc_chnfifo);
1296         }
1297         if (sc->sc_chnlastalg)
1298                 kfree(sc->sc_chnlastalg);
1299         if (sc->sc_chnfifolock)
1300                 kfree(sc->sc_chnfifolock);
1301         if (sc->sc_irq != -1)
1302                 free_irq(sc->sc_irq, sc);
1303         if (sc->sc_base_addr)
1304                 iounmap((void *) sc->sc_base_addr);
1305         kfree(sc);
1306         return 0;
1307 }
1308
1309 #ifdef CONFIG_PPC_MERGE
1310 static struct of_device_id talitos_match[] = {
1311         {
1312                 .type = "crypto",
1313                 .compatible = "talitos",
1314         },
1315         {},
1316 };
1317
1318 MODULE_DEVICE_TABLE(of, talitos_match);
1319
1320 static struct of_platform_driver talitos_driver = {
1321         .name           = DRV_NAME,
1322         .match_table    = talitos_match,
1323         .probe          = talitos_probe,
1324         .remove         = talitos_remove,
1325 };
1326
1327 static int __init talitos_init(void)
1328 {
1329         return of_register_platform_driver(&talitos_driver);
1330 }
1331
1332 static void __exit talitos_exit(void)
1333 {
1334         of_unregister_platform_driver(&talitos_driver);
1335 }
1336 #else
1337 /* Structure for a platform device driver */
1338 static struct platform_driver talitos_driver = {
1339         .probe = talitos_probe,
1340         .remove = talitos_remove,
1341         .driver = {
1342                 .name = "fsl-sec2",
1343         }
1344 };
1345
1346 static int __init talitos_init(void)
1347 {
1348         return platform_driver_register(&talitos_driver);
1349 }
1350
1351 static void __exit talitos_exit(void)
1352 {
1353         platform_driver_unregister(&talitos_driver);
1354 }
1355 #endif
1356
1357 module_init(talitos_init);
1358 module_exit(talitos_exit);
1359
1360 MODULE_LICENSE("Dual BSD/GPL");
1361 MODULE_AUTHOR("kim.phillips@freescale.com");
1362 MODULE_DESCRIPTION("OCF driver for Freescale SEC (talitos)");