#include "i915_vgpu.h"
#include <linux/pm_runtime.h>
-#include <linux/bsearch.h>
#define FORCEWAKE_ACK_TIMEOUT_MS 50
static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
bool restore_forcewake)
{
+ struct intel_device_info *info = mkwrite_device_info(dev_priv);
+
/* clear out unclaimed reg detection bit */
if (check_for_unclaimed_mmio(dev_priv))
DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
GT_FIFO_CTL_RC6_POLICY_STALL);
}
+ /* Enable Decoupled MMIO only on BXT C stepping onwards */
+ if (!IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
+ info->has_decoupled_mmio = false;
+
intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
}
__fwd; \
})
-static int fw_range_cmp(const void *key, const void *elt)
+static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
{
- const struct intel_forcewake_range *entry = elt;
- u32 offset = (u32)((unsigned long)key);
-
if (offset < entry->start)
return -1;
else if (offset > entry->end)
return 0;
}
+/* Copied and "macroized" from lib/bsearch.c */
+#define BSEARCH(key, base, num, cmp) ({ \
+ unsigned int start__ = 0, end__ = (num); \
+ typeof(base) result__ = NULL; \
+ while (start__ < end__) { \
+ unsigned int mid__ = start__ + (end__ - start__) / 2; \
+ int ret__ = (cmp)((key), (base) + mid__); \
+ if (ret__ < 0) { \
+ end__ = mid__; \
+ } else if (ret__ > 0) { \
+ start__ = mid__ + 1; \
+ } else { \
+ result__ = (base) + mid__; \
+ break; \
+ } \
+ } \
+ result__; \
+})
+
static enum forcewake_domains
find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
{
- const struct intel_forcewake_range *table, *entry;
- unsigned int num_entries;
-
- table = dev_priv->uncore.fw_domains_table;
- num_entries = dev_priv->uncore.fw_domains_table_entries;
+ const struct intel_forcewake_range *entry;
- entry = bsearch((void *)(unsigned long)offset, (const void *)table,
- num_entries, sizeof(struct intel_forcewake_range),
+ entry = BSEARCH(offset,
+ dev_priv->uncore.fw_domains_table,
+ dev_priv->uncore.fw_domains_table_entries,
fw_range_cmp);
return entry ? entry->domains : 0;
__fwd; \
})
+/* *Must* be sorted by offset! See intel_shadow_table_check(). */
static const i915_reg_t gen8_shadowed_regs[] = {
- GEN6_RPNSWREQ,
- GEN6_RC_VIDEO_FREQ,
- RING_TAIL(RENDER_RING_BASE),
- RING_TAIL(GEN6_BSD_RING_BASE),
- RING_TAIL(VEBOX_RING_BASE),
- RING_TAIL(BLT_RING_BASE),
+ RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */
+ GEN6_RPNSWREQ, /* 0xA008 */
+ GEN6_RC_VIDEO_FREQ, /* 0xA00C */
+ RING_TAIL(GEN6_BSD_RING_BASE), /* 0x12000 (base) */
+ RING_TAIL(VEBOX_RING_BASE), /* 0x1a000 (base) */
+ RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */
/* TODO: Other registers are not yet used */
};
+static void intel_shadow_table_check(void)
+{
+ const i915_reg_t *reg = gen8_shadowed_regs;
+ s32 prev;
+ u32 offset;
+ unsigned int i;
+
+ if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG))
+ return;
+
+ for (i = 0, prev = -1; i < ARRAY_SIZE(gen8_shadowed_regs); i++, reg++) {
+ offset = i915_mmio_reg_offset(*reg);
+ WARN_ON_ONCE(prev >= (s32)offset);
+ prev = offset;
+ }
+}
+
+static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
+{
+ u32 offset = i915_mmio_reg_offset(*reg);
+
+ if (key < offset)
+ return -1;
+ else if (key > offset)
+ return 1;
+ else
+ return 0;
+}
+
static bool is_gen8_shadowed(u32 offset)
{
- int i;
- for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
- if (offset == gen8_shadowed_regs[i].reg)
- return true;
+ const i915_reg_t *regs = gen8_shadowed_regs;
- return false;
+ return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
+ mmio_reg_cmp);
}
#define __gen8_reg_write_fw_domains(offset) \
GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
};
-#define __chv_reg_write_fw_domains(offset) \
+#define __fwtable_reg_write_fw_domains(offset) \
({ \
enum forcewake_domains __fwd = 0; \
if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
};
-static const i915_reg_t gen9_shadowed_regs[] = {
- RING_TAIL(RENDER_RING_BASE),
- RING_TAIL(GEN6_BSD_RING_BASE),
- RING_TAIL(VEBOX_RING_BASE),
- RING_TAIL(BLT_RING_BASE),
- GEN6_RPNSWREQ,
- GEN6_RC_VIDEO_FREQ,
- /* TODO: Other registers are not yet used */
-};
-
-static bool is_gen9_shadowed(u32 offset)
-{
- int i;
- for (i = 0; i < ARRAY_SIZE(gen9_shadowed_regs); i++)
- if (offset == gen9_shadowed_regs[i].reg)
- return true;
-
- return false;
-}
-
-#define __gen9_reg_write_fw_domains(offset) \
-({ \
- enum forcewake_domains __fwd = 0; \
- if (NEEDS_FORCE_WAKE((offset)) && !is_gen9_shadowed(offset)) \
- __fwd = find_fw_domain(dev_priv, offset); \
- __fwd; \
-})
-
static void
ilk_dummy_write(struct drm_i915_private *dev_priv)
{
__unclaimed_reg_debug(dev_priv, reg, read, before);
}
+static const enum decoupled_power_domain fw2dpd_domain[] = {
+ GEN9_DECOUPLED_PD_RENDER,
+ GEN9_DECOUPLED_PD_BLITTER,
+ GEN9_DECOUPLED_PD_ALL,
+ GEN9_DECOUPLED_PD_MEDIA,
+ GEN9_DECOUPLED_PD_ALL,
+ GEN9_DECOUPLED_PD_ALL,
+ GEN9_DECOUPLED_PD_ALL
+};
+
+/*
+ * Decoupled MMIO access for only 1 DWORD
+ */
+static void __gen9_decoupled_mmio_access(struct drm_i915_private *dev_priv,
+ u32 reg,
+ enum forcewake_domains fw_domain,
+ enum decoupled_ops operation)
+{
+ enum decoupled_power_domain dp_domain;
+ u32 ctrl_reg_data = 0;
+
+ dp_domain = fw2dpd_domain[fw_domain - 1];
+
+ ctrl_reg_data |= reg;
+ ctrl_reg_data |= (operation << GEN9_DECOUPLED_OP_SHIFT);
+ ctrl_reg_data |= (dp_domain << GEN9_DECOUPLED_PD_SHIFT);
+ ctrl_reg_data |= GEN9_DECOUPLED_DW1_GO;
+ __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW1, ctrl_reg_data);
+
+ if (wait_for_atomic((__raw_i915_read32(dev_priv,
+ GEN9_DECOUPLED_REG0_DW1) &
+ GEN9_DECOUPLED_DW1_GO) == 0,
+ FORCEWAKE_ACK_TIMEOUT_MS))
+ DRM_ERROR("Decoupled MMIO wait timed out\n");
+}
+
+static inline u32
+__gen9_decoupled_mmio_read32(struct drm_i915_private *dev_priv,
+ u32 reg,
+ enum forcewake_domains fw_domain)
+{
+ __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
+ GEN9_DECOUPLED_OP_READ);
+
+ return __raw_i915_read32(dev_priv, GEN9_DECOUPLED_REG0_DW0);
+}
+
+static inline void
+__gen9_decoupled_mmio_write(struct drm_i915_private *dev_priv,
+ u32 reg, u32 data,
+ enum forcewake_domains fw_domain)
+{
+
+ __raw_i915_write32(dev_priv, GEN9_DECOUPLED_REG0_DW0, data);
+
+ __gen9_decoupled_mmio_access(dev_priv, reg, fw_domain,
+ GEN9_DECOUPLED_OP_WRITE);
+}
+
+
#define GEN2_READ_HEADER(x) \
u##x val = 0; \
assert_rpm_wakelock_held(dev_priv);
GEN6_READ_FOOTER; \
}
+#define __gen9_decoupled_read(x) \
+static u##x \
+gen9_decoupled_read##x(struct drm_i915_private *dev_priv, \
+ i915_reg_t reg, bool trace) { \
+ enum forcewake_domains fw_engine; \
+ GEN6_READ_HEADER(x); \
+ fw_engine = __fwtable_reg_read_fw_domains(offset); \
+ if (fw_engine & ~dev_priv->uncore.fw_domains_active) { \
+ unsigned i; \
+ u32 *ptr_data = (u32 *) &val; \
+ for (i = 0; i < x/32; i++, offset += sizeof(u32), ptr_data++) \
+ *ptr_data = __gen9_decoupled_mmio_read32(dev_priv, \
+ offset, \
+ fw_engine); \
+ } else { \
+ val = __raw_i915_read##x(dev_priv, reg); \
+ } \
+ GEN6_READ_FOOTER; \
+}
+
+__gen9_decoupled_read(32)
+__gen9_decoupled_read(64)
__fwtable_read(8)
__fwtable_read(16)
__fwtable_read(32)
GEN6_WRITE_FOOTER; \
}
-#define __chv_write(x) \
+#define __fwtable_write(x) \
static void \
-chv_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
+fwtable_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
enum forcewake_domains fw_engine; \
GEN6_WRITE_HEADER; \
- fw_engine = __chv_reg_write_fw_domains(offset); \
+ fw_engine = __fwtable_reg_write_fw_domains(offset); \
if (fw_engine) \
__force_wake_auto(dev_priv, fw_engine); \
__raw_i915_write##x(dev_priv, reg, val); \
GEN6_WRITE_FOOTER; \
}
-#define __gen9_write(x) \
+#define __gen9_decoupled_write(x) \
static void \
-gen9_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, \
+gen9_decoupled_write##x(struct drm_i915_private *dev_priv, \
+ i915_reg_t reg, u##x val, \
bool trace) { \
enum forcewake_domains fw_engine; \
GEN6_WRITE_HEADER; \
- fw_engine = __gen9_reg_write_fw_domains(offset); \
- if (fw_engine) \
- __force_wake_auto(dev_priv, fw_engine); \
- __raw_i915_write##x(dev_priv, reg, val); \
+ fw_engine = __fwtable_reg_write_fw_domains(offset); \
+ if (fw_engine & ~dev_priv->uncore.fw_domains_active) \
+ __gen9_decoupled_mmio_write(dev_priv, \
+ offset, \
+ val, \
+ fw_engine); \
+ else \
+ __raw_i915_write##x(dev_priv, reg, val); \
GEN6_WRITE_FOOTER; \
}
-__gen9_write(8)
-__gen9_write(16)
-__gen9_write(32)
-__chv_write(8)
-__chv_write(16)
-__chv_write(32)
+__gen9_decoupled_write(32)
+__fwtable_write(8)
+__fwtable_write(16)
+__fwtable_write(32)
__gen8_write(8)
__gen8_write(16)
__gen8_write(32)
__gen6_write(16)
__gen6_write(32)
-#undef __gen9_write
-#undef __chv_write
+#undef __fwtable_write
#undef __gen8_write
#undef __gen6_write
#undef GEN6_WRITE_FOOTER
default:
case 9:
ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
- ASSIGN_WRITE_MMIO_VFUNCS(gen9);
+ ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
ASSIGN_READ_MMIO_VFUNCS(fwtable);
+ if (HAS_DECOUPLED_MMIO(dev_priv)) {
+ dev_priv->uncore.funcs.mmio_readl =
+ gen9_decoupled_read32;
+ dev_priv->uncore.funcs.mmio_readq =
+ gen9_decoupled_read64;
+ dev_priv->uncore.funcs.mmio_writel =
+ gen9_decoupled_write32;
+ }
break;
case 8:
if (IS_CHERRYVIEW(dev_priv)) {
ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
- ASSIGN_WRITE_MMIO_VFUNCS(chv);
+ ASSIGN_WRITE_MMIO_VFUNCS(fwtable);
ASSIGN_READ_MMIO_VFUNCS(fwtable);
} else {
}
intel_fw_table_check(dev_priv);
+ if (INTEL_GEN(dev_priv) >= 8)
+ intel_shadow_table_check();
if (intel_vgpu_active(dev_priv)) {
ASSIGN_WRITE_MMIO_VFUNCS(vgpu);
intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
i915_reg_t reg)
{
+ u32 offset = i915_mmio_reg_offset(reg);
enum forcewake_domains fw_domains;
- switch (INTEL_GEN(dev_priv)) {
- case 9:
- fw_domains = __gen9_reg_write_fw_domains(i915_mmio_reg_offset(reg));
- break;
- case 8:
- if (IS_CHERRYVIEW(dev_priv))
- fw_domains = __chv_reg_write_fw_domains(i915_mmio_reg_offset(reg));
- else
- fw_domains = __gen8_reg_write_fw_domains(i915_mmio_reg_offset(reg));
- break;
- case 7:
- case 6:
+ if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
+ fw_domains = __fwtable_reg_write_fw_domains(offset);
+ } else if (IS_GEN8(dev_priv)) {
+ fw_domains = __gen8_reg_write_fw_domains(offset);
+ } else if (IS_GEN(dev_priv, 6, 7)) {
fw_domains = FORCEWAKE_RENDER;
- break;
- default:
- MISSING_CASE(INTEL_INFO(dev_priv)->gen);
- case 5:
- case 4:
- case 3:
- case 2:
- return 0;
+ } else {
+ WARN_ON(!IS_GEN(dev_priv, 2, 5));
+ fw_domains = 0;
}
WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
projects / karo-tx-linux.git / blobdiff