drivers/iommu/fsl_pamu_domain.c

   1 /*
   2  * This program is free software; you can redistribute it and/or modify
   3  * it under the terms of the GNU General Public License, version 2, as
   4  * published by the Free Software Foundation.
   5  *
   6  * This program is distributed in the hope that it will be useful,
   7  * but WITHOUT ANY WARRANTY; without even the implied warranty of
   8  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   9  * GNU General Public License for more details.
  10  *
  11  * You should have received a copy of the GNU General Public License
  12  * along with this program; if not, write to the Free Software
  13  * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  14  *
  15  * Copyright (C) 2013 Freescale Semiconductor, Inc.
  16  * Author: Varun Sethi <varun.sethi@freescale.com>
  17  *
  18  */
  19
  20 #define pr_fmt(fmt)    "fsl-pamu-domain: %s: " fmt, __func__
  21
  22 #include <linux/init.h>
  23 #include <linux/iommu.h>
  24 #include <linux/notifier.h>
  25 #include <linux/slab.h>
  26 #include <linux/module.h>
  27 #include <linux/types.h>
  28 #include <linux/mm.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/device.h>
  31 #include <linux/of_platform.h>
  32 #include <linux/bootmem.h>
  33 #include <linux/err.h>
  34 #include <asm/io.h>
  35 #include <asm/bitops.h>
  36
  37 #include <asm/pci-bridge.h>
  38 #include <sysdev/fsl_pci.h>
  39
  40 #include "fsl_pamu_domain.h"
  41 #include "pci.h"
  42
  43 /*
  44  * Global spinlock that needs to be held while
  45  * configuring PAMU.
  46  */
  47 static DEFINE_SPINLOCK(iommu_lock);
  48
  49 static struct kmem_cache *fsl_pamu_domain_cache;
  50 static struct kmem_cache *iommu_devinfo_cache;
  51 static DEFINE_SPINLOCK(device_domain_lock);
  52
  53 static int __init iommu_init_mempool(void)
  54 {
  55
  56         fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain",
  57                                          sizeof(struct fsl_dma_domain),
  58                                          0,
  59                                          SLAB_HWCACHE_ALIGN,
  60
  61                                          NULL);
  62         if (!fsl_pamu_domain_cache) {
  63                 pr_debug("Couldn't create fsl iommu_domain cache\n");
  64                 return -ENOMEM;
  65         }
  66
  67         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
  68                                          sizeof(struct device_domain_info),
  69                                          0,
  70                                          SLAB_HWCACHE_ALIGN,
  71                                          NULL);
  72         if (!iommu_devinfo_cache) {
  73                 pr_debug("Couldn't create devinfo cache\n");
  74                 kmem_cache_destroy(fsl_pamu_domain_cache);
  75                 return -ENOMEM;
  76         }
  77
  78         return 0;
  79 }
  80
  81 static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova)
  82 {
  83         u32 win_cnt = dma_domain->win_cnt;
  84         struct dma_window *win_ptr =
  85                                 &dma_domain->win_arr[0];
  86         struct iommu_domain_geometry *geom;
  87
  88         geom = &dma_domain->iommu_domain->geometry;
  89
  90         if (!win_cnt || !dma_domain->geom_size) {
  91                 pr_debug("Number of windows/geometry not configured for the domain\n");
  92                 return 0;
  93         }
  94
  95         if (win_cnt > 1) {
  96                 u64 subwin_size;
  97                 dma_addr_t subwin_iova;
  98                 u32 wnd;
  99
 100                 subwin_size = dma_domain->geom_size >> ilog2(win_cnt);
 101                 subwin_iova = iova & ~(subwin_size - 1);
 102                 wnd = (subwin_iova - geom->aperture_start) >> ilog2(subwin_size);
 103                 win_ptr = &dma_domain->win_arr[wnd];
 104         }
 105
 106         if (win_ptr->valid)
 107                 return (win_ptr->paddr + (iova & (win_ptr->size - 1)));
 108
 109         return 0;
 110 }
 111
 112 static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain)
 113 {
 114         struct dma_window *sub_win_ptr =
 115                                 &dma_domain->win_arr[0];
 116         int i, ret;
 117         unsigned long rpn, flags;
 118
 119         for (i = 0; i < dma_domain->win_cnt; i++) {
 120                 if (sub_win_ptr[i].valid) {
 121                         rpn = sub_win_ptr[i].paddr >>
 122                                  PAMU_PAGE_SHIFT;
 123                         spin_lock_irqsave(&iommu_lock, flags);
 124                         ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i,
 125                                                  sub_win_ptr[i].size,
 126                                                  ~(u32)0,
 127                                                  rpn,
 128                                                  dma_domain->snoop_id,
 129                                                  dma_domain->stash_id,
 130                                                  (i > 0) ? 1 : 0,
 131                                                  sub_win_ptr[i].prot);
 132                         spin_unlock_irqrestore(&iommu_lock, flags);
 133                         if (ret) {
 134                                 pr_debug("PAMU SPAACE configuration failed for liodn %d\n",
 135                                          liodn);
 136                                 return ret;
 137                         }
 138                 }
 139         }
 140
 141         return ret;
 142 }
 143
 144 static int map_win(int liodn, struct fsl_dma_domain *dma_domain)
 145 {
 146         int ret;
 147         struct dma_window *wnd = &dma_domain->win_arr[0];
 148         phys_addr_t wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
 149         unsigned long flags;
 150
 151         spin_lock_irqsave(&iommu_lock, flags);
 152         ret = pamu_config_ppaace(liodn, wnd_addr,
 153                                  wnd->size,
 154                                  ~(u32)0,
 155                                  wnd->paddr >> PAMU_PAGE_SHIFT,
 156                                  dma_domain->snoop_id, dma_domain->stash_id,
 157                                  0, wnd->prot);
 158         spin_unlock_irqrestore(&iommu_lock, flags);
 159         if (ret)
 160                 pr_debug("PAMU PAACE configuration failed for liodn %d\n",
 161                         liodn);
 162
 163         return ret;
 164 }
 165
 166 /* Map the DMA window corresponding to the LIODN */
 167 static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain)
 168 {
 169         if (dma_domain->win_cnt > 1)
 170                 return map_subwins(liodn, dma_domain);
 171         else
 172                 return map_win(liodn, dma_domain);
 173
 174 }
 175
 176 /* Update window/subwindow mapping for the LIODN */
 177 static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 178 {
 179         int ret;
 180         struct dma_window *wnd = &dma_domain->win_arr[wnd_nr];
 181         unsigned long flags;
 182
 183         spin_lock_irqsave(&iommu_lock, flags);
 184         if (dma_domain->win_cnt > 1) {
 185                 ret = pamu_config_spaace(liodn, dma_domain->win_cnt, wnd_nr,
 186                                          wnd->size,
 187                                          ~(u32)0,
 188                                          wnd->paddr >> PAMU_PAGE_SHIFT,
 189                                          dma_domain->snoop_id,
 190                                          dma_domain->stash_id,
 191                                          (wnd_nr > 0) ? 1 : 0,
 192                                          wnd->prot);
 193                 if (ret)
 194                         pr_debug("Subwindow reconfiguration failed for liodn %d\n", liodn);
 195         } else {
 196                 phys_addr_t wnd_addr;
 197
 198                 wnd_addr = dma_domain->iommu_domain->geometry.aperture_start;
 199
 200                 ret = pamu_config_ppaace(liodn, wnd_addr,
 201                                          wnd->size,
 202                                          ~(u32)0,
 203                                          wnd->paddr >> PAMU_PAGE_SHIFT,
 204                                         dma_domain->snoop_id, dma_domain->stash_id,
 205                                         0, wnd->prot);
 206                 if (ret)
 207                         pr_debug("Window reconfiguration failed for liodn %d\n", liodn);
 208         }
 209
 210         spin_unlock_irqrestore(&iommu_lock, flags);
 211
 212         return ret;
 213 }
 214
 215 static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain,
 216                                  u32 val)
 217 {
 218         int ret = 0, i;
 219         unsigned long flags;
 220
 221         spin_lock_irqsave(&iommu_lock, flags);
 222         if (!dma_domain->win_arr) {
 223                 pr_debug("Windows not configured, stash destination update failed for liodn %d\n", liodn);
 224                 spin_unlock_irqrestore(&iommu_lock, flags);
 225                 return -EINVAL;
 226         }
 227
 228         for (i = 0; i < dma_domain->win_cnt; i++) {
 229                 ret = pamu_update_paace_stash(liodn, i, val);
 230                 if (ret) {
 231                         pr_debug("Failed to update SPAACE %d field for liodn %d\n ", i, liodn);
 232                         spin_unlock_irqrestore(&iommu_lock, flags);
 233                         return ret;
 234                 }
 235         }
 236
 237         spin_unlock_irqrestore(&iommu_lock, flags);
 238
 239         return ret;
 240 }
 241
 242 /* Set the geometry parameters for a LIODN */
 243 static int pamu_set_liodn(int liodn, struct device *dev,
 244                            struct fsl_dma_domain *dma_domain,
 245                            struct iommu_domain_geometry *geom_attr,
 246                            u32 win_cnt)
 247 {
 248         phys_addr_t window_addr, window_size;
 249         phys_addr_t subwin_size;
 250         int ret = 0, i;
 251         u32 omi_index = ~(u32)0;
 252         unsigned long flags;
 253
 254         /*
 255          * Configure the omi_index at the geometry setup time.
 256          * This is a static value which depends on the type of
 257          * device and would not change thereafter.
 258          */
 259         get_ome_index(&omi_index, dev);
 260
 261         window_addr = geom_attr->aperture_start;
 262         window_size = dma_domain->geom_size;
 263
 264         spin_lock_irqsave(&iommu_lock, flags);
 265         ret = pamu_disable_liodn(liodn);
 266         if (!ret)
 267                 ret = pamu_config_ppaace(liodn, window_addr, window_size, omi_index,
 268                                          0, dma_domain->snoop_id,
 269                                          dma_domain->stash_id, win_cnt, 0);
 270         spin_unlock_irqrestore(&iommu_lock, flags);
 271         if (ret) {
 272                 pr_debug("PAMU PAACE configuration failed for liodn %d, win_cnt =%d\n", liodn, win_cnt);
 273                 return ret;
 274         }
 275
 276         if (win_cnt > 1) {
 277                 subwin_size = window_size >> ilog2(win_cnt);
 278                 for (i = 0; i < win_cnt; i++) {
 279                         spin_lock_irqsave(&iommu_lock, flags);
 280                         ret = pamu_disable_spaace(liodn, i);
 281                         if (!ret)
 282                                 ret = pamu_config_spaace(liodn, win_cnt, i,
 283                                                          subwin_size, omi_index,
 284                                                          0, dma_domain->snoop_id,
 285                                                          dma_domain->stash_id,
 286                                                          0, 0);
 287                         spin_unlock_irqrestore(&iommu_lock, flags);
 288                         if (ret) {
 289                                 pr_debug("PAMU SPAACE configuration failed for liodn %d\n", liodn);
 290                                 return ret;
 291                         }
 292                 }
 293         }
 294
 295         return ret;
 296 }
 297
 298 static int check_size(u64 size, dma_addr_t iova)
 299 {
 300         /*
 301          * Size must be a power of two and at least be equal
 302          * to PAMU page size.
 303          */
 304         if (!is_power_of_2(size) || size < PAMU_PAGE_SIZE) {
 305                 pr_debug("%s: size too small or not a power of two\n", __func__);
 306                 return -EINVAL;
 307         }
 308
 309         /* iova must be page size aligned*/
 310         if (iova & (size - 1)) {
 311                 pr_debug("%s: address is not aligned with window size\n", __func__);
 312                 return -EINVAL;
 313         }
 314
 315         return 0;
 316 }
 317
 318 static struct fsl_dma_domain *iommu_alloc_dma_domain(void)
 319 {
 320         struct fsl_dma_domain *domain;
 321
 322         domain = kmem_cache_zalloc(fsl_pamu_domain_cache, GFP_KERNEL);
 323         if (!domain)
 324                 return NULL;
 325
 326         domain->stash_id = ~(u32)0;
 327         domain->snoop_id = ~(u32)0;
 328         domain->win_cnt = pamu_get_max_subwin_cnt();
 329         domain->geom_size = 0;
 330
 331         INIT_LIST_HEAD(&domain->devices);
 332
 333         spin_lock_init(&domain->domain_lock);
 334
 335         return domain;
 336 }
 337
 338 static inline struct device_domain_info *find_domain(struct device *dev)
 339 {
 340         return dev->archdata.iommu_domain;
 341 }
 342
 343 static void remove_device_ref(struct device_domain_info *info, u32 win_cnt)
 344 {
 345         unsigned long flags;
 346
 347         list_del(&info->link);
 348         spin_lock_irqsave(&iommu_lock, flags);
 349         if (win_cnt > 1)
 350                 pamu_free_subwins(info->liodn);
 351         pamu_disable_liodn(info->liodn);
 352         spin_unlock_irqrestore(&iommu_lock, flags);
 353         spin_lock_irqsave(&device_domain_lock, flags);
 354         info->dev->archdata.iommu_domain = NULL;
 355         kmem_cache_free(iommu_devinfo_cache, info);
 356         spin_unlock_irqrestore(&device_domain_lock, flags);
 357 }
 358
 359 static void detach_device(struct device *dev, struct fsl_dma_domain *dma_domain)
 360 {
 361         struct device_domain_info *info, *tmp;
 362         unsigned long flags;
 363
 364         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 365         /* Remove the device from the domain device list */
 366         list_for_each_entry_safe(info, tmp, &dma_domain->devices, link) {
 367                 if (!dev || (info->dev == dev))
 368                         remove_device_ref(info, dma_domain->win_cnt);
 369         }
 370         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 371 }
 372
 373 static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct device *dev)
 374 {
 375         struct device_domain_info *info, *old_domain_info;
 376         unsigned long flags;
 377
 378         spin_lock_irqsave(&device_domain_lock, flags);
 379         /*
 380          * Check here if the device is already attached to domain or not.
 381          * If the device is already attached to a domain detach it.
 382          */
 383         old_domain_info = find_domain(dev);
 384         if (old_domain_info && old_domain_info->domain != dma_domain) {
 385                 spin_unlock_irqrestore(&device_domain_lock, flags);
 386                 detach_device(dev, old_domain_info->domain);
 387                 spin_lock_irqsave(&device_domain_lock, flags);
 388         }
 389
 390         info = kmem_cache_zalloc(iommu_devinfo_cache, GFP_ATOMIC);
 391
 392         info->dev = dev;
 393         info->liodn = liodn;
 394         info->domain = dma_domain;
 395
 396         list_add(&info->link, &dma_domain->devices);
 397         /*
 398          * In case of devices with multiple LIODNs just store
 399          * the info for the first LIODN as all
 400          * LIODNs share the same domain
 401          */
 402         if (!old_domain_info)
 403                 dev->archdata.iommu_domain = info;
 404         spin_unlock_irqrestore(&device_domain_lock, flags);
 405
 406 }
 407
 408 static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain,
 409                                             dma_addr_t iova)
 410 {
 411         struct fsl_dma_domain *dma_domain = domain->priv;
 412
 413         if ((iova < domain->geometry.aperture_start) ||
 414                 iova > (domain->geometry.aperture_end))
 415                 return 0;
 416
 417         return get_phys_addr(dma_domain, iova);
 418 }
 419
 420 static int fsl_pamu_domain_has_cap(struct iommu_domain *domain,
 421                                       unsigned long cap)
 422 {
 423         return cap == IOMMU_CAP_CACHE_COHERENCY;
 424 }
 425
 426 static void fsl_pamu_domain_destroy(struct iommu_domain *domain)
 427 {
 428         struct fsl_dma_domain *dma_domain = domain->priv;
 429
 430         domain->priv = NULL;
 431
 432         /* remove all the devices from the device list */
 433         detach_device(NULL, dma_domain);
 434
 435         dma_domain->enabled = 0;
 436         dma_domain->mapped = 0;
 437
 438         kmem_cache_free(fsl_pamu_domain_cache, dma_domain);
 439 }
 440
 441 static int fsl_pamu_domain_init(struct iommu_domain *domain)
 442 {
 443         struct fsl_dma_domain *dma_domain;
 444
 445         dma_domain = iommu_alloc_dma_domain();
 446         if (!dma_domain) {
 447                 pr_debug("dma_domain allocation failed\n");
 448                 return -ENOMEM;
 449         }
 450         domain->priv = dma_domain;
 451         dma_domain->iommu_domain = domain;
 452         /* defaul geometry 64 GB i.e. maximum system address */
 453         domain->geometry.aperture_start = 0;
 454         domain->geometry.aperture_end = (1ULL << 36) - 1;
 455         domain->geometry.force_aperture = true;
 456
 457         return 0;
 458 }
 459
 460 /* Configure geometry settings for all LIODNs associated with domain */
 461 static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain,
 462                                     struct iommu_domain_geometry *geom_attr,
 463                                     u32 win_cnt)
 464 {
 465         struct device_domain_info *info;
 466         int ret = 0;
 467
 468         list_for_each_entry(info, &dma_domain->devices, link) {
 469                 ret = pamu_set_liodn(info->liodn, info->dev, dma_domain,
 470                                       geom_attr, win_cnt);
 471                 if (ret)
 472                         break;
 473         }
 474
 475         return ret;
 476 }
 477
 478 /* Update stash destination for all LIODNs associated with the domain */
 479 static int update_domain_stash(struct fsl_dma_domain *dma_domain, u32 val)
 480 {
 481         struct device_domain_info *info;
 482         int ret = 0;
 483
 484         list_for_each_entry(info, &dma_domain->devices, link) {
 485                 ret = update_liodn_stash(info->liodn, dma_domain, val);
 486                 if (ret)
 487                         break;
 488         }
 489
 490         return ret;
 491 }
 492
 493 /* Update domain mappings for all LIODNs associated with the domain */
 494 static int update_domain_mapping(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 495 {
 496         struct device_domain_info *info;
 497         int ret = 0;
 498
 499         list_for_each_entry(info, &dma_domain->devices, link) {
 500                 ret = update_liodn(info->liodn, dma_domain, wnd_nr);
 501                 if (ret)
 502                         break;
 503         }
 504         return ret;
 505 }
 506
 507 static int disable_domain_win(struct fsl_dma_domain *dma_domain, u32 wnd_nr)
 508 {
 509         struct device_domain_info *info;
 510         int ret = 0;
 511
 512         list_for_each_entry(info, &dma_domain->devices, link) {
 513                 if (dma_domain->win_cnt == 1 && dma_domain->enabled) {
 514                         ret = pamu_disable_liodn(info->liodn);
 515                         if (!ret)
 516                                 dma_domain->enabled = 0;
 517                 } else {
 518                         ret = pamu_disable_spaace(info->liodn, wnd_nr);
 519                 }
 520         }
 521
 522         return ret;
 523 }
 524
 525 static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr)
 526 {
 527         struct fsl_dma_domain *dma_domain = domain->priv;
 528         unsigned long flags;
 529         int ret;
 530
 531         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 532         if (!dma_domain->win_arr) {
 533                 pr_debug("Number of windows not configured\n");
 534                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 535                 return;
 536         }
 537
 538         if (wnd_nr >= dma_domain->win_cnt) {
 539                 pr_debug("Invalid window index\n");
 540                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 541                 return;
 542         }
 543
 544         if (dma_domain->win_arr[wnd_nr].valid) {
 545                 ret = disable_domain_win(dma_domain, wnd_nr);
 546                 if (!ret) {
 547                         dma_domain->win_arr[wnd_nr].valid = 0;
 548                         dma_domain->mapped--;
 549                 }
 550         }
 551
 552         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 553
 554 }
 555
 556 static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr,
 557                                   phys_addr_t paddr, u64 size, int prot)
 558 {
 559         struct fsl_dma_domain *dma_domain = domain->priv;
 560         struct dma_window *wnd;
 561         int pamu_prot = 0;
 562         int ret;
 563         unsigned long flags;
 564         u64 win_size;
 565
 566         if (prot & IOMMU_READ)
 567                 pamu_prot |= PAACE_AP_PERMS_QUERY;
 568         if (prot & IOMMU_WRITE)
 569                 pamu_prot |= PAACE_AP_PERMS_UPDATE;
 570
 571         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 572         if (!dma_domain->win_arr) {
 573                 pr_debug("Number of windows not configured\n");
 574                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 575                 return -ENODEV;
 576         }
 577
 578         if (wnd_nr >= dma_domain->win_cnt) {
 579                 pr_debug("Invalid window index\n");
 580                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 581                 return -EINVAL;
 582         }
 583
 584         win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt);
 585         if (size > win_size) {
 586                 pr_debug("Invalid window size \n");
 587                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 588                 return -EINVAL;
 589         }
 590
 591         if (dma_domain->win_cnt == 1) {
 592                 if (dma_domain->enabled) {
 593                         pr_debug("Disable the window before updating the mapping\n");
 594                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 595                         return -EBUSY;
 596                 }
 597
 598                 ret = check_size(size, domain->geometry.aperture_start);
 599                 if (ret) {
 600                         pr_debug("Aperture start not aligned to the size\n");
 601                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 602                         return -EINVAL;
 603                 }
 604         }
 605
 606         wnd = &dma_domain->win_arr[wnd_nr];
 607         if (!wnd->valid) {
 608                 wnd->paddr = paddr;
 609                 wnd->size = size;
 610                 wnd->prot = pamu_prot;
 611
 612                 ret = update_domain_mapping(dma_domain, wnd_nr);
 613                 if (!ret) {
 614                         wnd->valid = 1;
 615                         dma_domain->mapped++;
 616                 }
 617         } else {
 618                 pr_debug("Disable the window before updating the mapping\n");
 619                 ret = -EBUSY;
 620         }
 621
 622         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 623
 624         return ret;
 625 }
 626
 627 /*
 628  * Attach the LIODN to the DMA domain and configure the geometry
 629  * and window mappings.
 630  */
 631 static int handle_attach_device(struct fsl_dma_domain *dma_domain,
 632                                  struct device *dev, const u32 *liodn,
 633                                  int num)
 634 {
 635         unsigned long flags;
 636         struct iommu_domain *domain = dma_domain->iommu_domain;
 637         int ret = 0;
 638         int i;
 639
 640         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 641         for (i = 0; i < num; i++) {
 642
 643                 /* Ensure that LIODN value is valid */
 644                 if (liodn[i] >= PAACE_NUMBER_ENTRIES) {
 645                         pr_debug("Invalid liodn %d, attach device failed for %s\n",
 646                                 liodn[i], dev->of_node->full_name);
 647                         ret = -EINVAL;
 648                         break;
 649                 }
 650
 651                 attach_device(dma_domain, liodn[i], dev);
 652                 /*
 653                  * Check if geometry has already been configured
 654                  * for the domain. If yes, set the geometry for
 655                  * the LIODN.
 656                  */
 657                 if (dma_domain->win_arr) {
 658                         u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0;
 659                         ret = pamu_set_liodn(liodn[i], dev, dma_domain,
 660                                               &domain->geometry,
 661                                               win_cnt);
 662                         if (ret)
 663                                 break;
 664                         if (dma_domain->mapped) {
 665                                 /*
 666                                  * Create window/subwindow mapping for
 667                                  * the LIODN.
 668                                  */
 669                                 ret = map_liodn(liodn[i], dma_domain);
 670                                 if (ret)
 671                                         break;
 672                         }
 673                 }
 674         }
 675         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 676
 677         return ret;
 678 }
 679
 680 static int fsl_pamu_attach_device(struct iommu_domain *domain,
 681                                   struct device *dev)
 682 {
 683         struct fsl_dma_domain *dma_domain = domain->priv;
 684         const u32 *liodn;
 685         u32 liodn_cnt;
 686         int len, ret = 0;
 687         struct pci_dev *pdev = NULL;
 688         struct pci_controller *pci_ctl;
 689
 690         /*
 691          * Use LIODN of the PCI controller while attaching a
 692          * PCI device.
 693          */
 694         if (dev->bus == &pci_bus_type) {
 695                 pdev = to_pci_dev(dev);
 696                 pci_ctl = pci_bus_to_host(pdev->bus);
 697                 /*
 698                  * make dev point to pci controller device
 699                  * so we can get the LIODN programmed by
 700                  * u-boot.
 701                  */
 702                 dev = pci_ctl->parent;
 703         }
 704
 705         liodn = of_get_property(dev->of_node, "fsl,liodn", &len);
 706         if (liodn) {
 707                 liodn_cnt = len / sizeof(u32);
 708                 ret = handle_attach_device(dma_domain, dev,
 709                                          liodn, liodn_cnt);
 710         } else {
 711                 pr_debug("missing fsl,liodn property at %s\n",
 712                           dev->of_node->full_name);
 713                         ret = -EINVAL;
 714         }
 715
 716         return ret;
 717 }
 718
 719 static void fsl_pamu_detach_device(struct iommu_domain *domain,
 720                                       struct device *dev)
 721 {
 722         struct fsl_dma_domain *dma_domain = domain->priv;
 723         const u32 *prop;
 724         int len;
 725         struct pci_dev *pdev = NULL;
 726         struct pci_controller *pci_ctl;
 727
 728         /*
 729          * Use LIODN of the PCI controller while detaching a
 730          * PCI device.
 731          */
 732         if (dev->bus == &pci_bus_type) {
 733                 pdev = to_pci_dev(dev);
 734                 pci_ctl = pci_bus_to_host(pdev->bus);
 735                 /*
 736                  * make dev point to pci controller device
 737                  * so we can get the LIODN programmed by
 738                  * u-boot.
 739                  */
 740                 dev = pci_ctl->parent;
 741         }
 742
 743         prop = of_get_property(dev->of_node, "fsl,liodn", &len);
 744         if (prop)
 745                 detach_device(dev, dma_domain);
 746         else
 747                 pr_debug("missing fsl,liodn property at %s\n",
 748                           dev->of_node->full_name);
 749 }
 750
 751 static  int configure_domain_geometry(struct iommu_domain *domain, void *data)
 752 {
 753         struct iommu_domain_geometry *geom_attr = data;
 754         struct fsl_dma_domain *dma_domain = domain->priv;
 755         dma_addr_t geom_size;
 756         unsigned long flags;
 757
 758         geom_size = geom_attr->aperture_end - geom_attr->aperture_start + 1;
 759         /*
 760          * Sanity check the geometry size. Also, we do not support
 761          * DMA outside of the geometry.
 762          */
 763         if (check_size(geom_size, geom_attr->aperture_start) ||
 764                 !geom_attr->force_aperture) {
 765                         pr_debug("Invalid PAMU geometry attributes\n");
 766                         return -EINVAL;
 767                 }
 768
 769         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 770         if (dma_domain->enabled) {
 771                 pr_debug("Can't set geometry attributes as domain is active\n");
 772                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 773                 return  -EBUSY;
 774         }
 775
 776         /* Copy the domain geometry information */
 777         memcpy(&domain->geometry, geom_attr,
 778                sizeof(struct iommu_domain_geometry));
 779         dma_domain->geom_size = geom_size;
 780
 781         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 782
 783         return 0;
 784 }
 785
 786 /* Set the domain stash attribute */
 787 static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data)
 788 {
 789         struct pamu_stash_attribute *stash_attr = data;
 790         unsigned long flags;
 791         int ret;
 792
 793         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 794
 795         memcpy(&dma_domain->dma_stash, stash_attr,
 796                  sizeof(struct pamu_stash_attribute));
 797
 798         dma_domain->stash_id = get_stash_id(stash_attr->cache,
 799                                             stash_attr->cpu);
 800         if (dma_domain->stash_id == ~(u32)0) {
 801                 pr_debug("Invalid stash attributes\n");
 802                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 803                 return -EINVAL;
 804         }
 805
 806         ret = update_domain_stash(dma_domain, dma_domain->stash_id);
 807
 808         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 809
 810         return ret;
 811 }
 812
 813 /* Configure domain dma state i.e. enable/disable DMA*/
 814 static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable)
 815 {
 816         struct device_domain_info *info;
 817         unsigned long flags;
 818         int ret;
 819
 820         spin_lock_irqsave(&dma_domain->domain_lock, flags);
 821
 822         if (enable && !dma_domain->mapped) {
 823                 pr_debug("Can't enable DMA domain without valid mapping\n");
 824                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 825                 return -ENODEV;
 826         }
 827
 828         dma_domain->enabled = enable;
 829         list_for_each_entry(info, &dma_domain->devices,
 830                                  link) {
 831                 ret = (enable) ? pamu_enable_liodn(info->liodn) :
 832                         pamu_disable_liodn(info->liodn);
 833                 if (ret)
 834                         pr_debug("Unable to set dma state for liodn %d",
 835                                  info->liodn);
 836         }
 837         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
 838
 839         return 0;
 840 }
 841
 842 static int fsl_pamu_set_domain_attr(struct iommu_domain *domain,
 843                                  enum iommu_attr attr_type, void *data)
 844 {
 845         struct fsl_dma_domain *dma_domain = domain->priv;
 846         int ret = 0;
 847
 848
 849         switch (attr_type) {
 850         case DOMAIN_ATTR_GEOMETRY:
 851                 ret = configure_domain_geometry(domain, data);
 852                 break;
 853         case DOMAIN_ATTR_FSL_PAMU_STASH:
 854                 ret = configure_domain_stash(dma_domain, data);
 855                 break;
 856         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 857                 ret = configure_domain_dma_state(dma_domain, *(int *)data);
 858                 break;
 859         default:
 860                 pr_debug("Unsupported attribute type\n");
 861                 ret = -EINVAL;
 862                 break;
 863         };
 864
 865         return ret;
 866 }
 867
 868 static int fsl_pamu_get_domain_attr(struct iommu_domain *domain,
 869                                  enum iommu_attr attr_type, void *data)
 870 {
 871         struct fsl_dma_domain *dma_domain = domain->priv;
 872         int ret = 0;
 873
 874
 875         switch (attr_type) {
 876         case DOMAIN_ATTR_FSL_PAMU_STASH:
 877                 memcpy((struct pamu_stash_attribute *) data, &dma_domain->dma_stash,
 878                                  sizeof(struct pamu_stash_attribute));
 879                 break;
 880         case DOMAIN_ATTR_FSL_PAMU_ENABLE:
 881                 *(int *)data = dma_domain->enabled;
 882                 break;
 883         case DOMAIN_ATTR_FSL_PAMUV1:
 884                 *(int *)data = DOMAIN_ATTR_FSL_PAMUV1;
 885                 break;
 886         default:
 887                 pr_debug("Unsupported attribute type\n");
 888                 ret = -EINVAL;
 889                 break;
 890         };
 891
 892         return ret;
 893 }
 894
 895 #define REQ_ACS_FLAGS   (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
 896
 897 static struct iommu_group *get_device_iommu_group(struct device *dev)
 898 {
 899         struct iommu_group *group;
 900
 901         group = iommu_group_get(dev);
 902         if (!group)
 903                 group = iommu_group_alloc();
 904
 905         return group;
 906 }
 907
 908 static  bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl)
 909 {
 910         u32 version;
 911
 912         /* Check the PCI controller version number by readding BRR1 register */
 913         version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2));
 914         version &= PCI_FSL_BRR1_VER;
 915         /* If PCI controller version is >= 0x204 we can partition endpoints*/
 916         if (version >= 0x204)
 917                 return 1;
 918
 919         return 0;
 920 }
 921
 922 /* Get iommu group information from peer devices or devices on the parent bus */
 923 static struct iommu_group *get_shared_pci_device_group(struct pci_dev *pdev)
 924 {
 925         struct pci_dev *tmp;
 926         struct iommu_group *group;
 927         struct pci_bus *bus = pdev->bus;
 928
 929         /*
 930          * Traverese the pci bus device list to get
 931          * the shared iommu group.
 932          */
 933         while (bus) {
 934                 list_for_each_entry(tmp, &bus->devices, bus_list) {
 935                         if (tmp == pdev)
 936                                 continue;
 937                         group = iommu_group_get(&tmp->dev);
 938                         if (group)
 939                                 return group;
 940                 }
 941
 942                 bus = bus->parent;
 943         }
 944
 945         return NULL;
 946 }
 947
 948 static struct iommu_group *get_pci_device_group(struct pci_dev *pdev)
 949 {
 950         struct pci_controller *pci_ctl;
 951         bool pci_endpt_partioning;
 952         struct iommu_group *group = NULL;
 953         struct pci_dev *bridge, *dma_pdev = NULL;
 954
 955         pci_ctl = pci_bus_to_host(pdev->bus);
 956         pci_endpt_partioning = check_pci_ctl_endpt_part(pci_ctl);
 957         /* We can partition PCIe devices so assign device group to the device */
 958         if (pci_endpt_partioning) {
 959                 bridge = pci_find_upstream_pcie_bridge(pdev);
 960                 if (bridge) {
 961                         if (pci_is_pcie(bridge))
 962                                 dma_pdev = pci_get_domain_bus_and_slot(
 963                                                 pci_domain_nr(pdev->bus),
 964                                                 bridge->subordinate->number, 0);
 965                         if (!dma_pdev)
 966                                 dma_pdev = pci_dev_get(bridge);
 967                 } else
 968                         dma_pdev = pci_dev_get(pdev);
 969
 970                 /* Account for quirked devices */
 971                 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
 972
 973                 /*
 974                  * If it's a multifunction device that does not support our
 975                  * required ACS flags, add to the same group as lowest numbered
 976                  * function that also does not suport the required ACS flags.
 977                  */
 978                 if (dma_pdev->multifunction &&
 979                     !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) {
 980                         u8 i, slot = PCI_SLOT(dma_pdev->devfn);
 981
 982                         for (i = 0; i < 8; i++) {
 983                                 struct pci_dev *tmp;
 984
 985                                 tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i));
 986                                 if (!tmp)
 987                                         continue;
 988
 989                                 if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) {
 990                                         swap_pci_ref(&dma_pdev, tmp);
 991                                         break;
 992                                 }
 993                                 pci_dev_put(tmp);
 994                         }
 995                 }
 996
 997                 /*
 998                  * Devices on the root bus go through the iommu.  If that's not us,
 999                  * find the next upstream device and test ACS up to the root bus.
1000                  * Finding the next device may require skipping virtual buses.
1001                  */
1002                 while (!pci_is_root_bus(dma_pdev->bus)) {
1003                         struct pci_bus *bus = dma_pdev->bus;
1004
1005                         while (!bus->self) {
1006                                 if (!pci_is_root_bus(bus))
1007                                         bus = bus->parent;
1008                                 else
1009                                         goto root_bus;
1010                         }
1011
1012                         if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
1013                                 break;
1014
1015                         swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
1016                 }
1017
1018 root_bus:
1019                 group = get_device_iommu_group(&dma_pdev->dev);
1020                 pci_dev_put(dma_pdev);
1021                 /*
1022                  * PCIe controller is not a paritionable entity
1023                  * free the controller device iommu_group.
1024                  */
1025                 if (pci_ctl->parent->iommu_group)
1026                         iommu_group_remove_device(pci_ctl->parent);
1027         } else {
1028                 /*
1029                  * All devices connected to the controller will share the
1030                  * PCI controllers device group. If this is the first
1031                  * device to be probed for the pci controller, copy the
1032                  * device group information from the PCI controller device
1033                  * node and remove the PCI controller iommu group.
1034                  * For subsequent devices, the iommu group information can
1035                  * be obtained from sibling devices (i.e. from the bus_devices
1036                  * link list).
1037                  */
1038                 if (pci_ctl->parent->iommu_group) {
1039                         group = get_device_iommu_group(pci_ctl->parent);
1040                         iommu_group_remove_device(pci_ctl->parent);
1041                 } else
1042                         group = get_shared_pci_device_group(pdev);
1043         }
1044
1045         return group;
1046 }
1047
1048 static int fsl_pamu_add_device(struct device *dev)
1049 {
1050         struct iommu_group *group = NULL;
1051         struct pci_dev *pdev;
1052         const u32 *prop;
1053         int ret, len;
1054
1055         /*
1056          * For platform devices we allocate a separate group for
1057          * each of the devices.
1058          */
1059         if (dev->bus == &pci_bus_type) {
1060                 pdev = to_pci_dev(dev);
1061                 /* Don't create device groups for virtual PCI bridges */
1062                 if (pdev->subordinate)
1063                         return 0;
1064
1065                 group = get_pci_device_group(pdev);
1066
1067         } else {
1068                 prop = of_get_property(dev->of_node, "fsl,liodn", &len);
1069                 if (prop)
1070                         group = get_device_iommu_group(dev);
1071         }
1072
1073         if (!group || IS_ERR(group))
1074                 return PTR_ERR(group);
1075
1076         ret = iommu_group_add_device(group, dev);
1077
1078         iommu_group_put(group);
1079         return ret;
1080 }
1081
1082 static void fsl_pamu_remove_device(struct device *dev)
1083 {
1084         iommu_group_remove_device(dev);
1085 }
1086
1087 static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count)
1088 {
1089         struct fsl_dma_domain *dma_domain = domain->priv;
1090         unsigned long flags;
1091         int ret;
1092
1093         spin_lock_irqsave(&dma_domain->domain_lock, flags);
1094         /* Ensure domain is inactive i.e. DMA should be disabled for the domain */
1095         if (dma_domain->enabled) {
1096                 pr_debug("Can't set geometry attributes as domain is active\n");
1097                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1098                 return  -EBUSY;
1099         }
1100
1101         /* Ensure that the geometry has been set for the domain */
1102         if (!dma_domain->geom_size) {
1103                 pr_debug("Please configure geometry before setting the number of windows\n");
1104                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1105                 return -EINVAL;
1106         }
1107
1108         /*
1109          * Ensure we have valid window count i.e. it should be less than
1110          * maximum permissible limit and should be a power of two.
1111          */
1112         if (w_count > pamu_get_max_subwin_cnt() || !is_power_of_2(w_count)) {
1113                 pr_debug("Invalid window count\n");
1114                 spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1115                 return -EINVAL;
1116         }
1117
1118         ret = pamu_set_domain_geometry(dma_domain, &domain->geometry,
1119                                 ((w_count > 1) ? w_count : 0));
1120         if (!ret) {
1121                 if (dma_domain->win_arr)
1122                         kfree(dma_domain->win_arr);
1123                 dma_domain->win_arr = kzalloc(sizeof(struct dma_window) *
1124                                                           w_count, GFP_ATOMIC);
1125                 if (!dma_domain->win_arr) {
1126                         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1127                         return -ENOMEM;
1128                 }
1129                 dma_domain->win_cnt = w_count;
1130         }
1131         spin_unlock_irqrestore(&dma_domain->domain_lock, flags);
1132
1133         return ret;
1134 }
1135
1136 static u32 fsl_pamu_get_windows(struct iommu_domain *domain)
1137 {
1138         struct fsl_dma_domain *dma_domain = domain->priv;
1139
1140         return dma_domain->win_cnt;
1141 }
1142
1143 static struct iommu_ops fsl_pamu_ops = {
1144         .domain_init    = fsl_pamu_domain_init,
1145         .domain_destroy = fsl_pamu_domain_destroy,
1146         .attach_dev     = fsl_pamu_attach_device,
1147         .detach_dev     = fsl_pamu_detach_device,
1148         .domain_window_enable = fsl_pamu_window_enable,
1149         .domain_window_disable = fsl_pamu_window_disable,
1150         .domain_get_windows = fsl_pamu_get_windows,
1151         .domain_set_windows = fsl_pamu_set_windows,
1152         .iova_to_phys   = fsl_pamu_iova_to_phys,
1153         .domain_has_cap = fsl_pamu_domain_has_cap,
1154         .domain_set_attr = fsl_pamu_set_domain_attr,
1155         .domain_get_attr = fsl_pamu_get_domain_attr,
1156         .add_device     = fsl_pamu_add_device,
1157         .remove_device  = fsl_pamu_remove_device,
1158 };
1159
1160 int pamu_domain_init()
1161 {
1162         int ret = 0;
1163
1164         ret = iommu_init_mempool();
1165         if (ret)
1166                 return ret;
1167
1168         bus_set_iommu(&platform_bus_type, &fsl_pamu_ops);
1169         bus_set_iommu(&pci_bus_type, &fsl_pamu_ops);
1170
1171         return ret;
1172 }