diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
307 files changed, 19913 insertions, 11001 deletions
diff --git a/drivers/gpu/drm/i915/.gitignore b/drivers/gpu/drm/i915/.gitignore index d9a77f3b59b2..81972dce1aff 100644 --- a/drivers/gpu/drm/i915/.gitignore +++ b/drivers/gpu/drm/i915/.gitignore @@ -1 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only *.hdrtest diff --git a/drivers/gpu/drm/i915/Kconfig.profile b/drivers/gpu/drm/i915/Kconfig.profile index c280b6ae38eb..35bbe2b80596 100644 --- a/drivers/gpu/drm/i915/Kconfig.profile +++ b/drivers/gpu/drm/i915/Kconfig.profile @@ -1,3 +1,15 @@ +config DRM_I915_FENCE_TIMEOUT + int "Timeout for unsignaled foreign fences (ms, jiffy granularity)" + default 10000 # milliseconds + help + When listening to a foreign fence, we install a supplementary timer + to ensure that we are always signaled and our userspace is able to + make forward progress. This value specifies the timeout used for an + unsignaled foreign fence. + + May be 0 to disable the timeout, and rely on the foreign fence being + eventually signaled. + config DRM_I915_USERFAULT_AUTOSUSPEND int "Runtime autosuspend delay for userspace GGTT mmaps (ms)" default 250 # milliseconds @@ -20,6 +32,9 @@ config DRM_I915_HEARTBEAT_INTERVAL check the health of the GPU and undertake regular house-keeping of internal driver state. + This is adjustable via + /sys/class/drm/card?/engine/*/heartbeat_interval_ms + May be 0 to disable heartbeats and therefore disable automatic GPU hang detection. @@ -33,11 +48,18 @@ config DRM_I915_PREEMPT_TIMEOUT expires, the HW will be reset to allow the more important context to execute. + This is adjustable via + /sys/class/drm/card?/engine/*/preempt_timeout_ms + May be 0 to disable the timeout. -config DRM_I915_SPIN_REQUEST - int "Busywait for request completion (us)" - default 5 # microseconds + The compiled in default may get overridden at driver probe time on + certain platforms and certain engines which will be reflected in the + sysfs control. + +config DRM_I915_MAX_REQUEST_BUSYWAIT + int "Busywait for request completion limit (ns)" + default 8000 # nanoseconds help Before sleeping waiting for a request (GPU operation) to complete, we may spend some time polling for its completion. As the IRQ may @@ -45,6 +67,9 @@ config DRM_I915_SPIN_REQUEST check if the request will complete in the time it would have taken us to enable the interrupt. + This is adjustable via + /sys/class/drm/card?/engine/*/max_busywait_duration_ns + May be 0 to disable the initial spin. In practice, we estimate the cost of enabling the interrupt (if currently disabled) to be a few microseconds. @@ -60,6 +85,9 @@ config DRM_I915_STOP_TIMEOUT that the reset itself may take longer and so be more disruptive to interactive or low latency workloads. + This is adjustable via + /sys/class/drm/card?/engine/*/stop_timeout_ms + config DRM_I915_TIMESLICE_DURATION int "Scheduling quantum for userspace batches (ms, jiffy granularity)" default 1 # milliseconds @@ -73,4 +101,7 @@ config DRM_I915_TIMESLICE_DURATION is scheduled for execution for the timeslice duration, before switching to the next context. + This is adjustable via + /sys/class/drm/card?/engine/*/timeslice_duration_ms + May be 0 to disable timeslicing. diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index b314d44ded5e..b0da6ea6e3f1 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -22,21 +22,20 @@ subdir-ccflags-y += $(call cc-disable-warning, sign-compare) subdir-ccflags-y += $(call cc-disable-warning, sometimes-uninitialized) subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides) subdir-ccflags-y += $(call cc-disable-warning, uninitialized) +subdir-ccflags-y += $(call cc-disable-warning, frame-address) subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror # Fine grained warnings disable CFLAGS_i915_pci.o = $(call cc-disable-warning, override-init) CFLAGS_display/intel_fbdev.o = $(call cc-disable-warning, override-init) -subdir-ccflags-y += \ - $(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA) - subdir-ccflags-y += -I$(srctree)/$(src) # Please keep these build lists sorted! # core driver code i915-y += i915_drv.o \ + i915_config.o \ i915_irq.o \ i915_getparam.o \ i915_params.o \ @@ -47,6 +46,7 @@ i915-y += i915_drv.o \ i915_sysfs.o \ i915_utils.o \ intel_device_info.o \ + intel_dram.o \ intel_memory_region.o \ intel_pch.o \ intel_pm.o \ @@ -79,17 +79,21 @@ gt-y += \ gt/debugfs_gt.o \ gt/debugfs_gt_pm.o \ gt/gen6_ppgtt.o \ + gt/gen7_renderclear.o \ gt/gen8_ppgtt.o \ gt/intel_breadcrumbs.o \ gt/intel_context.o \ + gt/intel_context_param.o \ gt/intel_context_sseu.o \ gt/intel_engine_cs.o \ gt/intel_engine_heartbeat.o \ gt/intel_engine_pm.o \ - gt/intel_engine_pool.o \ gt/intel_engine_user.o \ gt/intel_ggtt.o \ + gt/intel_ggtt_fencing.o \ gt/intel_gt.o \ + gt/intel_gt_buffer_pool.o \ + gt/intel_gt_clock_utils.o \ gt/intel_gt_irq.o \ gt/intel_gt_pm.o \ gt/intel_gt_pm_irq.o \ @@ -107,7 +111,9 @@ gt-y += \ gt/intel_rps.o \ gt/intel_sseu.o \ gt/intel_timeline.o \ - gt/intel_workarounds.o + gt/intel_workarounds.o \ + gt/shmem_utils.o \ + gt/sysfs_engines.o # autogenerated null render state gt-y += \ gt/gen6_renderstate.o \ @@ -149,7 +155,6 @@ i915-y += \ i915_buddy.o \ i915_cmd_parser.o \ i915_gem_evict.o \ - i915_gem_fence_reg.o \ i915_gem_gtt.o \ i915_gem.o \ i915_globals.o \ @@ -163,14 +168,18 @@ i915-y += \ # general-purpose microcontroller (GuC) support i915-y += gt/uc/intel_uc.o \ + gt/uc/intel_uc_debugfs.o \ gt/uc/intel_uc_fw.o \ gt/uc/intel_guc.o \ gt/uc/intel_guc_ads.o \ gt/uc/intel_guc_ct.o \ + gt/uc/intel_guc_debugfs.o \ gt/uc/intel_guc_fw.o \ gt/uc/intel_guc_log.o \ + gt/uc/intel_guc_log_debugfs.o \ gt/uc/intel_guc_submission.o \ gt/uc/intel_huc.o \ + gt/uc/intel_huc_debugfs.o \ gt/uc/intel_huc_fw.o # modesetting core code @@ -239,23 +248,6 @@ i915-y += \ display/vlv_dsi.o \ display/vlv_dsi_pll.o -# perf code -i915-y += \ - oa/i915_oa_hsw.o \ - oa/i915_oa_bdw.o \ - oa/i915_oa_chv.o \ - oa/i915_oa_sklgt2.o \ - oa/i915_oa_sklgt3.o \ - oa/i915_oa_sklgt4.o \ - oa/i915_oa_bxt.o \ - oa/i915_oa_kblgt2.o \ - oa/i915_oa_kblgt3.o \ - oa/i915_oa_glk.o \ - oa/i915_oa_cflgt2.o \ - oa/i915_oa_cflgt3.o \ - oa/i915_oa_cnl.o \ - oa/i915_oa_icl.o \ - oa/i915_oa_tgl.o i915-y += i915_perf.o # Post-mortem debug and GPU hang state capture @@ -269,7 +261,8 @@ i915-$(CONFIG_DRM_I915_SELFTEST) += \ selftests/igt_live_test.o \ selftests/igt_mmap.o \ selftests/igt_reset.o \ - selftests/igt_spinner.o + selftests/igt_spinner.o \ + selftests/librapl.o # virtual gpu code i915-y += i915_vgpu.o @@ -301,7 +294,7 @@ extra-$(CONFIG_DRM_I915_WERROR) += \ $(shell cd $(srctree)/$(src) && find * -name '*.h'))) quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) - cmd_hdrtest = $(CC) $(c_flags) -S -o /dev/null -x c /dev/null -include $<; touch $@ + cmd_hdrtest = $(CC) $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@ $(obj)/%.hdrtest: $(src)/%.h FORCE $(call if_changed_dep,hdrtest) diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index d0f7419fb02b..4fec5bd64920 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -36,15 +36,15 @@ #include "intel_panel.h" #include "intel_vdsc.h" -static inline int header_credits_available(struct drm_i915_private *dev_priv, - enum transcoder dsi_trans) +static int header_credits_available(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans) { return (intel_de_read(dev_priv, DSI_CMD_TXCTL(dsi_trans)) & FREE_HEADER_CREDIT_MASK) >> FREE_HEADER_CREDIT_SHIFT; } -static inline int payload_credits_available(struct drm_i915_private *dev_priv, - enum transcoder dsi_trans) +static int payload_credits_available(struct drm_i915_private *dev_priv, + enum transcoder dsi_trans) { return (intel_de_read(dev_priv, DSI_CMD_TXCTL(dsi_trans)) & FREE_PLOAD_CREDIT_MASK) >> FREE_PLOAD_CREDIT_SHIFT; @@ -186,16 +186,19 @@ static int dsi_send_pkt_hdr(struct intel_dsi_host *host, static int dsi_send_pkt_payld(struct intel_dsi_host *host, struct mipi_dsi_packet pkt) { + struct intel_dsi *intel_dsi = host->intel_dsi; + struct drm_i915_private *i915 = to_i915(intel_dsi->base.base.dev); + /* payload queue can accept *256 bytes*, check limit */ if (pkt.payload_length > MAX_PLOAD_CREDIT * 4) { - DRM_ERROR("payload size exceeds max queue limit\n"); + drm_err(&i915->drm, "payload size exceeds max queue limit\n"); return -1; } /* load data into command payload queue */ if (!add_payld_to_queue(host, pkt.payload, pkt.payload_length)) { - DRM_ERROR("adding payload to queue failed\n"); + drm_err(&i915->drm, "adding payload to queue failed\n"); return -1; } @@ -599,13 +602,13 @@ static void gen11_dsi_gate_clocks(struct intel_encoder *encoder) u32 tmp; enum phy phy; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); tmp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); for_each_dsi_phy(phy, intel_dsi->phys) tmp |= ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, tmp); - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder) @@ -615,13 +618,13 @@ static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder) u32 tmp; enum phy phy; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); tmp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); for_each_dsi_phy(phy, intel_dsi->phys) tmp &= ~ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, tmp); - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static void gen11_dsi_map_pll(struct intel_encoder *encoder, @@ -633,7 +636,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder, enum phy phy; u32 val; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); val = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); for_each_dsi_phy(phy, intel_dsi->phys) { @@ -652,7 +655,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder, intel_de_posting_read(dev_priv, ICL_DPCLKA_CFGCR0); - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static void @@ -744,6 +747,18 @@ gen11_dsi_configure_transcoder(struct intel_encoder *encoder, tmp |= VIDEO_MODE_SYNC_PULSE; break; } + } else { + /* + * FIXME: Retrieve this info from VBT. + * As per the spec when dsi transcoder is operating + * in TE GATE mode, TE comes from GPIO + * which is UTIL PIN for DSI 0. + * Also this GPIO would not be used for other + * purposes is an assumption. + */ + tmp &= ~OP_MODE_MASK; + tmp |= CMD_MODE_TE_GATE; + tmp |= TE_SOURCE_GPIO; } intel_de_write(dev_priv, DSI_TRANS_FUNC_CONF(dsi_trans), tmp); @@ -837,14 +852,33 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, } hactive = adjusted_mode->crtc_hdisplay; - htotal = DIV_ROUND_UP(adjusted_mode->crtc_htotal * mul, div); + + if (is_vid_mode(intel_dsi)) + htotal = DIV_ROUND_UP(adjusted_mode->crtc_htotal * mul, div); + else + htotal = DIV_ROUND_UP((hactive + 160) * mul, div); + hsync_start = DIV_ROUND_UP(adjusted_mode->crtc_hsync_start * mul, div); hsync_end = DIV_ROUND_UP(adjusted_mode->crtc_hsync_end * mul, div); hsync_size = hsync_end - hsync_start; hback_porch = (adjusted_mode->crtc_htotal - adjusted_mode->crtc_hsync_end); vactive = adjusted_mode->crtc_vdisplay; - vtotal = adjusted_mode->crtc_vtotal; + + if (is_vid_mode(intel_dsi)) { + vtotal = adjusted_mode->crtc_vtotal; + } else { + int bpp, line_time_us, byte_clk_period_ns; + + if (crtc_state->dsc.compression_enable) + bpp = crtc_state->dsc.compressed_bpp; + else + bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format); + + byte_clk_period_ns = 1000000 / afe_clk(encoder, crtc_state); + line_time_us = (htotal * (bpp / 8) * byte_clk_period_ns) / (1000 * intel_dsi->lane_count); + vtotal = vactive + DIV_ROUND_UP(400, line_time_us); + } vsync_start = adjusted_mode->crtc_vsync_start; vsync_end = adjusted_mode->crtc_vsync_end; vsync_shift = hsync_start - htotal / 2; @@ -873,7 +907,7 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, } /* TRANS_HSYNC register to be programmed only for video mode */ - if (intel_dsi->operation_mode == INTEL_DSI_VIDEO_MODE) { + if (is_vid_mode(intel_dsi)) { if (intel_dsi->video_mode_format == VIDEO_MODE_NON_BURST_WITH_SYNC_PULSE) { /* BSPEC: hsync size should be atleast 16 pixels */ @@ -916,22 +950,27 @@ gen11_dsi_set_transcoder_timings(struct intel_encoder *encoder, if (vsync_start < vactive) drm_err(&dev_priv->drm, "vsync_start less than vactive\n"); - /* program TRANS_VSYNC register */ - for_each_dsi_port(port, intel_dsi->ports) { - dsi_trans = dsi_port_to_transcoder(port); - intel_de_write(dev_priv, VSYNC(dsi_trans), - (vsync_start - 1) | ((vsync_end - 1) << 16)); + /* program TRANS_VSYNC register for video mode only */ + if (is_vid_mode(intel_dsi)) { + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + intel_de_write(dev_priv, VSYNC(dsi_trans), + (vsync_start - 1) | ((vsync_end - 1) << 16)); + } } /* - * FIXME: It has to be programmed only for interlaced + * FIXME: It has to be programmed only for video modes and interlaced * modes. Put the check condition here once interlaced * info available as described above. * program TRANS_VSYNCSHIFT register */ - for_each_dsi_port(port, intel_dsi->ports) { - dsi_trans = dsi_port_to_transcoder(port); - intel_de_write(dev_priv, VSYNCSHIFT(dsi_trans), vsync_shift); + if (is_vid_mode(intel_dsi)) { + for_each_dsi_port(port, intel_dsi->ports) { + dsi_trans = dsi_port_to_transcoder(port); + intel_de_write(dev_priv, VSYNCSHIFT(dsi_trans), + vsync_shift); + } } /* program TRANS_VBLANK register, should be same as vtotal programmed */ @@ -1016,6 +1055,32 @@ static void gen11_dsi_setup_timeouts(struct intel_encoder *encoder, } } +static void gen11_dsi_config_util_pin(struct intel_encoder *encoder, + bool enable) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); + u32 tmp; + + /* + * used as TE i/p for DSI0, + * for dual link/DSI1 TE is from slave DSI1 + * through GPIO. + */ + if (is_vid_mode(intel_dsi) || (intel_dsi->ports & BIT(PORT_B))) + return; + + tmp = intel_de_read(dev_priv, UTIL_PIN_CTL); + + if (enable) { + tmp |= UTIL_PIN_DIRECTION_INPUT; + tmp |= UTIL_PIN_ENABLE; + } else { + tmp &= ~UTIL_PIN_ENABLE; + } + intel_de_write(dev_priv, UTIL_PIN_CTL, tmp); +} + static void gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) @@ -1037,6 +1102,9 @@ gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder, /* setup D-PHY timings */ gen11_dsi_setup_dphy_timings(encoder, crtc_state); + /* Since transcoder is configured to take events from GPIO */ + gen11_dsi_config_util_pin(encoder, true); + /* step 4h: setup DSI protocol timeouts */ gen11_dsi_setup_timeouts(encoder, crtc_state); @@ -1088,7 +1156,8 @@ static void gen11_dsi_powerup_panel(struct intel_encoder *encoder) wait_for_cmds_dispatched_to_panel(encoder); } -static void gen11_dsi_pre_pll_enable(struct intel_encoder *encoder, +static void gen11_dsi_pre_pll_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -1099,7 +1168,8 @@ static void gen11_dsi_pre_pll_enable(struct intel_encoder *encoder, gen11_dsi_program_esc_clk_div(encoder, crtc_state); } -static void gen11_dsi_pre_enable(struct intel_encoder *encoder, +static void gen11_dsi_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -1118,13 +1188,14 @@ static void gen11_dsi_pre_enable(struct intel_encoder *encoder, gen11_dsi_set_transcoder_timings(encoder, pipe_config); } -static void gen11_dsi_enable(struct intel_encoder *encoder, +static void gen11_dsi_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); - WARN_ON(crtc_state->has_pch_encoder); + drm_WARN_ON(state->base.dev, crtc_state->has_pch_encoder); /* step6d: enable dsi transcoder */ gen11_dsi_enable_transcoder(encoder); @@ -1180,6 +1251,15 @@ static void gen11_dsi_deconfigure_trancoder(struct intel_encoder *encoder) enum transcoder dsi_trans; u32 tmp; + /* disable periodic update mode */ + if (is_cmd_mode(intel_dsi)) { + for_each_dsi_port(port, intel_dsi->ports) { + tmp = intel_de_read(dev_priv, DSI_CMD_FRMCTL(port)); + tmp &= ~DSI_PERIODIC_FRAME_UPDATE_ENABLE; + intel_de_write(dev_priv, DSI_CMD_FRMCTL(port), tmp); + } + } + /* put dsi link in ULPS */ for_each_dsi_port(port, intel_dsi->ports) { dsi_trans = dsi_port_to_transcoder(port); @@ -1264,7 +1344,8 @@ static void gen11_dsi_disable_io_power(struct intel_encoder *encoder) } } -static void gen11_dsi_disable(struct intel_encoder *encoder, +static void gen11_dsi_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -1286,11 +1367,14 @@ static void gen11_dsi_disable(struct intel_encoder *encoder, /* step3: disable port */ gen11_dsi_disable_port(encoder); + gen11_dsi_config_util_pin(encoder, false); + /* step4: disable IO power */ gen11_dsi_disable_io_power(encoder); } -static void gen11_dsi_post_disable(struct intel_encoder *encoder, +static void gen11_dsi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -1347,18 +1431,34 @@ static void gen11_dsi_get_timings(struct intel_encoder *encoder, adjusted_mode->crtc_vblank_end = adjusted_mode->crtc_vtotal; } +static bool gen11_dsi_is_periodic_cmd_mode(struct intel_dsi *intel_dsi) +{ + struct drm_device *dev = intel_dsi->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + enum transcoder dsi_trans; + u32 val; + + if (intel_dsi->ports == BIT(PORT_B)) + dsi_trans = TRANSCODER_DSI_1; + else + dsi_trans = TRANSCODER_DSI_0; + + val = intel_de_read(dev_priv, DSI_TRANS_FUNC_CONF(dsi_trans)); + return (val & DSI_PERIODIC_FRAME_UPDATE_ENABLE); +} + static void gen11_dsi_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); intel_dsc_get_config(encoder, pipe_config); /* FIXME: adapt icl_ddi_clock_get() for DSI and use that? */ - pipe_config->port_clock = - cnl_calc_wrpll_link(dev_priv, &pipe_config->dpll_hw_state); + pipe_config->port_clock = intel_dpll_get_freq(i915, + pipe_config->shared_dpll); pipe_config->hw.adjusted_mode.crtc_clock = intel_dsi->pclk; if (intel_dsi->dual_link) @@ -1367,6 +1467,10 @@ static void gen11_dsi_get_config(struct intel_encoder *encoder, gen11_dsi_get_timings(encoder, pipe_config); pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI); pipe_config->pipe_bpp = bdw_get_pipemisc_bpp(crtc); + + if (gen11_dsi_is_periodic_cmd_mode(intel_dsi)) + pipe_config->hw.adjusted_mode.private_flags |= + I915_MODE_FLAG_DSI_PERIODIC_CMD_MODE; } static int gen11_dsi_dsc_compute_config(struct intel_encoder *encoder, @@ -1417,18 +1521,22 @@ static int gen11_dsi_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, base); struct intel_connector *intel_connector = intel_dsi->attached_connector; - struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); const struct drm_display_mode *fixed_mode = - intel_connector->panel.fixed_mode; + intel_connector->panel.fixed_mode; struct drm_display_mode *adjusted_mode = - &pipe_config->hw.adjusted_mode; + &pipe_config->hw.adjusted_mode; + int ret; pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; intel_fixed_panel_mode(fixed_mode, adjusted_mode); - intel_pch_panel_fitting(crtc, pipe_config, conn_state->scaling_mode); + + ret = intel_pch_panel_fitting(pipe_config, conn_state); + if (ret) + return ret; adjusted_mode->flags = 0; @@ -1446,10 +1554,32 @@ static int gen11_dsi_compute_config(struct intel_encoder *encoder, pipe_config->clock_set = true; if (gen11_dsi_dsc_compute_config(encoder, pipe_config)) - DRM_DEBUG_KMS("Attempting to use DSC failed\n"); + drm_dbg_kms(&i915->drm, "Attempting to use DSC failed\n"); pipe_config->port_clock = afe_clk(encoder, pipe_config) / 5; + /* We would not operate in periodic command mode */ + pipe_config->hw.adjusted_mode.private_flags &= + ~I915_MODE_FLAG_DSI_PERIODIC_CMD_MODE; + + /* + * In case of TE GATE cmd mode, we + * receive TE from the slave if + * dual link is enabled + */ + if (is_cmd_mode(intel_dsi)) { + if (intel_dsi->ports == (BIT(PORT_B) | BIT(PORT_A))) + pipe_config->hw.adjusted_mode.private_flags |= + I915_MODE_FLAG_DSI_USE_TE1 | + I915_MODE_FLAG_DSI_USE_TE0; + else if (intel_dsi->ports == BIT(PORT_B)) + pipe_config->hw.adjusted_mode.private_flags |= + I915_MODE_FLAG_DSI_USE_TE1; + else + pipe_config->hw.adjusted_mode.private_flags |= + I915_MODE_FLAG_DSI_USE_TE0; + } + return 0; } @@ -1718,9 +1848,8 @@ static void icl_dsi_add_properties(struct intel_connector *connector) connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT; - connector->base.display_info.panel_orientation = - intel_dsi_get_panel_orientation(connector); - drm_connector_init_panel_orientation_property(&connector->base, + drm_connector_set_panel_orientation_with_quirk(&connector->base, + intel_dsi_get_panel_orientation(connector), connector->panel.fixed_mode->hdisplay, connector->panel.fixed_mode->vdisplay); } diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c index c86d7a35c816..79032701873a 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c @@ -125,7 +125,7 @@ intel_plane_destroy_state(struct drm_plane *plane, struct drm_plane_state *state) { struct intel_plane_state *plane_state = to_intel_plane_state(state); - WARN_ON(plane_state->vma); + drm_WARN_ON(plane->dev, plane_state->vma); __drm_atomic_helper_plane_destroy_state(&plane_state->uapi); if (plane_state->hw.fb) @@ -133,15 +133,37 @@ intel_plane_destroy_state(struct drm_plane *plane, kfree(plane_state); } +unsigned int intel_plane_pixel_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state) +{ + unsigned int src_w, src_h, dst_w, dst_h; + unsigned int pixel_rate = crtc_state->pixel_rate; + + src_w = drm_rect_width(&plane_state->uapi.src) >> 16; + src_h = drm_rect_height(&plane_state->uapi.src) >> 16; + dst_w = drm_rect_width(&plane_state->uapi.dst); + dst_h = drm_rect_height(&plane_state->uapi.dst); + + /* Downscaling limits the maximum pixel rate */ + dst_w = min(src_w, dst_w); + dst_h = min(src_h, dst_h); + + return DIV_ROUND_UP_ULL(mul_u32_u32(pixel_rate, src_w * src_h), + dst_w * dst_h); +} + unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { const struct drm_framebuffer *fb = plane_state->hw.fb; unsigned int cpp; + unsigned int pixel_rate; if (!plane_state->uapi.visible) return 0; + pixel_rate = intel_plane_pixel_rate(crtc_state, plane_state); + cpp = fb->format->cpp[0]; /* @@ -153,7 +175,7 @@ unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state, if (fb->format->is_yuv && fb->format->num_planes > 1) cpp *= 4; - return cpp * crtc_state->pixel_rate; + return pixel_rate * cpp; } int intel_plane_calc_min_cdclk(struct intel_atomic_state *state, @@ -242,6 +264,20 @@ void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, plane_state->hw.color_range = from_plane_state->uapi.color_range; } +void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state) +{ + struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane); + + crtc_state->active_planes &= ~BIT(plane->id); + crtc_state->nv12_planes &= ~BIT(plane->id); + crtc_state->c8_planes &= ~BIT(plane->id); + crtc_state->data_rate[plane->id] = 0; + crtc_state->min_cdclk[plane->id] = 0; + + plane_state->uapi.visible = false; +} + int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_state, struct intel_crtc_state *new_crtc_state, const struct intel_plane_state *old_plane_state, @@ -251,12 +287,7 @@ int intel_plane_atomic_check_with_state(const struct intel_crtc_state *old_crtc_ const struct drm_framebuffer *fb = new_plane_state->hw.fb; int ret; - new_crtc_state->active_planes &= ~BIT(plane->id); - new_crtc_state->nv12_planes &= ~BIT(plane->id); - new_crtc_state->c8_planes &= ~BIT(plane->id); - new_crtc_state->data_rate[plane->id] = 0; - new_crtc_state->min_cdclk[plane->id] = 0; - new_plane_state->uapi.visible = false; + intel_plane_set_invisible(new_crtc_state, new_plane_state); if (!new_plane_state->hw.crtc && !old_plane_state->hw.crtc) return 0; @@ -365,7 +396,7 @@ skl_next_plane_to_commit(struct intel_atomic_state *state, } /* should never happen */ - WARN_ON(1); + drm_WARN_ON(state->base.dev, 1); return NULL; } diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.h b/drivers/gpu/drm/i915/display/intel_atomic_plane.h index 2bcf15e34728..59dd1fbb02ea 100644 --- a/drivers/gpu/drm/i915/display/intel_atomic_plane.h +++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.h @@ -18,6 +18,9 @@ struct intel_plane_state; extern const struct drm_plane_helper_funcs intel_plane_helper_funcs; +unsigned int intel_plane_pixel_rate(const struct intel_crtc_state *crtc_state, + const struct intel_plane_state *plane_state); + unsigned int intel_plane_data_rate(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, @@ -49,5 +52,7 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat int intel_plane_calc_min_cdclk(struct intel_atomic_state *state, struct intel_plane *plane, bool *need_cdclk_calc); +void intel_plane_set_invisible(struct intel_crtc_state *crtc_state, + struct intel_plane_state *plane_state); #endif /* __INTEL_ATOMIC_PLANE_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c index 30fb7c887ff0..ad4aa66fd676 100644 --- a/drivers/gpu/drm/i915/display/intel_audio.c +++ b/drivers/gpu/drm/i915/display/intel_audio.c @@ -149,6 +149,10 @@ static const struct { { 74250, AUD_CONFIG_PIXEL_CLOCK_HDMI_74250 }, { 148352, AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 }, { 148500, AUD_CONFIG_PIXEL_CLOCK_HDMI_148500 }, + { 296703, AUD_CONFIG_PIXEL_CLOCK_HDMI_296703 }, + { 297000, AUD_CONFIG_PIXEL_CLOCK_HDMI_297000 }, + { 593407, AUD_CONFIG_PIXEL_CLOCK_HDMI_593407 }, + { 594000, AUD_CONFIG_PIXEL_CLOCK_HDMI_594000 }, }; /* HDMI N/CTS table */ @@ -234,6 +238,7 @@ static const struct hdmi_aud_ncts hdmi_aud_ncts_36bpp[] = { /* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */ static u32 audio_config_hdmi_pixel_clock(const struct intel_crtc_state *crtc_state) { + struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; int i; @@ -243,15 +248,20 @@ static u32 audio_config_hdmi_pixel_clock(const struct intel_crtc_state *crtc_sta break; } + if (INTEL_GEN(dev_priv) < 12 && adjusted_mode->crtc_clock > 148500) + i = ARRAY_SIZE(hdmi_audio_clock); + if (i == ARRAY_SIZE(hdmi_audio_clock)) { - DRM_DEBUG_KMS("HDMI audio pixel clock setting for %d not found, falling back to defaults\n", - adjusted_mode->crtc_clock); + drm_dbg_kms(&dev_priv->drm, + "HDMI audio pixel clock setting for %d not found, falling back to defaults\n", + adjusted_mode->crtc_clock); i = 1; } - DRM_DEBUG_KMS("Configuring HDMI audio for pixel clock %d (0x%08x)\n", - hdmi_audio_clock[i].clock, - hdmi_audio_clock[i].config); + drm_dbg_kms(&dev_priv->drm, + "Configuring HDMI audio for pixel clock %d (0x%08x)\n", + hdmi_audio_clock[i].clock, + hdmi_audio_clock[i].config); return hdmi_audio_clock[i].config; } @@ -504,6 +514,124 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder, mutex_unlock(&dev_priv->av_mutex); } +static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + unsigned int link_clks_available, link_clks_required; + unsigned int tu_data, tu_line, link_clks_active; + unsigned int h_active, h_total, hblank_delta, pixel_clk; + unsigned int fec_coeff, cdclk, vdsc_bpp; + unsigned int link_clk, lanes; + unsigned int hblank_rise; + + h_active = crtc_state->hw.adjusted_mode.crtc_hdisplay; + h_total = crtc_state->hw.adjusted_mode.crtc_htotal; + pixel_clk = crtc_state->hw.adjusted_mode.crtc_clock; + vdsc_bpp = crtc_state->dsc.compressed_bpp; + cdclk = i915->cdclk.hw.cdclk; + /* fec= 0.972261, using rounding multiplier of 1000000 */ + fec_coeff = 972261; + link_clk = crtc_state->port_clock; + lanes = crtc_state->lane_count; + + drm_dbg_kms(&i915->drm, "h_active = %u link_clk = %u :" + "lanes = %u vdsc_bpp = %u cdclk = %u\n", + h_active, link_clk, lanes, vdsc_bpp, cdclk); + + if (WARN_ON(!link_clk || !pixel_clk || !lanes || !vdsc_bpp || !cdclk)) + return 0; + + link_clks_available = (h_total - h_active) * link_clk / pixel_clk - 28; + link_clks_required = DIV_ROUND_UP(192000 * h_total, 1000 * pixel_clk) * (48 / lanes + 2); + + if (link_clks_available > link_clks_required) + hblank_delta = 32; + else + hblank_delta = DIV64_U64_ROUND_UP(mul_u32_u32(5 * (link_clk + cdclk), pixel_clk), + mul_u32_u32(link_clk, cdclk)); + + tu_data = div64_u64(mul_u32_u32(pixel_clk * vdsc_bpp * 8, 1000000), + mul_u32_u32(link_clk * lanes, fec_coeff)); + tu_line = div64_u64(h_active * mul_u32_u32(link_clk, fec_coeff), + mul_u32_u32(64 * pixel_clk, 1000000)); + link_clks_active = (tu_line - 1) * 64 + tu_data; + + hblank_rise = (link_clks_active + 6 * DIV_ROUND_UP(link_clks_active, 250) + 4) * pixel_clk / link_clk; + + return h_active - hblank_rise + hblank_delta; +} + +static unsigned int calc_samples_room(const struct intel_crtc_state *crtc_state) +{ + unsigned int h_active, h_total, pixel_clk; + unsigned int link_clk, lanes; + + h_active = crtc_state->hw.adjusted_mode.hdisplay; + h_total = crtc_state->hw.adjusted_mode.htotal; + pixel_clk = crtc_state->hw.adjusted_mode.clock; + link_clk = crtc_state->port_clock; + lanes = crtc_state->lane_count; + + return ((h_total - h_active) * link_clk - 12 * pixel_clk) / + (pixel_clk * (48 / lanes + 2)); +} + +static void enable_audio_dsc_wa(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + enum pipe pipe = crtc->pipe; + unsigned int hblank_early_prog, samples_room; + unsigned int val; + + if (INTEL_GEN(i915) < 11) + return; + + val = intel_de_read(i915, AUD_CONFIG_BE); + + if (INTEL_GEN(i915) == 11) + val |= HBLANK_EARLY_ENABLE_ICL(pipe); + else if (INTEL_GEN(i915) >= 12) + val |= HBLANK_EARLY_ENABLE_TGL(pipe); + + if (crtc_state->dsc.compression_enable && + (crtc_state->hw.adjusted_mode.hdisplay >= 3840 && + crtc_state->hw.adjusted_mode.vdisplay >= 2160)) { + /* Get hblank early enable value required */ + hblank_early_prog = calc_hblank_early_prog(encoder, crtc_state); + if (hblank_early_prog < 32) { + val &= ~HBLANK_START_COUNT_MASK(pipe); + val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_32); + } else if (hblank_early_prog < 64) { + val &= ~HBLANK_START_COUNT_MASK(pipe); + val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_64); + } else if (hblank_early_prog < 96) { + val &= ~HBLANK_START_COUNT_MASK(pipe); + val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_96); + } else { + val &= ~HBLANK_START_COUNT_MASK(pipe); + val |= HBLANK_START_COUNT(pipe, HBLANK_START_COUNT_128); + } + + /* Get samples room value required */ + samples_room = calc_samples_room(crtc_state); + if (samples_room < 3) { + val &= ~NUMBER_SAMPLES_PER_LINE_MASK(pipe); + val |= NUMBER_SAMPLES_PER_LINE(pipe, samples_room); + } else { + /* Program 0 i.e "All Samples available in buffer" */ + val &= ~NUMBER_SAMPLES_PER_LINE_MASK(pipe); + val |= NUMBER_SAMPLES_PER_LINE(pipe, 0x0); + } + } + + intel_de_write(i915, AUD_CONFIG_BE, val); +} + +#undef ROUNDING_FACTOR + static void hsw_audio_codec_enable(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) @@ -521,6 +649,10 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder, mutex_lock(&dev_priv->av_mutex); + /* Enable Audio WA for 4k DSC usecases */ + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP)) + enable_audio_dsc_wa(encoder, crtc_state); + /* Enable audio presence detect, invalidate ELD */ tmp = intel_de_read(dev_priv, HSW_AUD_PIN_ELD_CP_VLD); tmp |= AUDIO_OUTPUT_ENABLE(cpu_transcoder); @@ -844,7 +976,7 @@ static void glk_force_audio_cdclk(struct drm_i915_private *dev_priv, struct intel_crtc *crtc; int ret; - crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A); + crtc = intel_get_first_crtc(dev_priv); if (!crtc) return; @@ -883,7 +1015,7 @@ static unsigned long i915_audio_component_get_power(struct device *kdev) ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO); if (dev_priv->audio_power_refcount++ == 0) { - if (IS_TIGERLAKE(dev_priv) || IS_ICELAKE(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { intel_de_write(dev_priv, AUD_FREQ_CNTRL, dev_priv->audio_freq_cntrl); drm_dbg_kms(&dev_priv->drm, @@ -923,7 +1055,7 @@ static void i915_audio_component_codec_wake_override(struct device *kdev, unsigned long cookie; u32 tmp; - if (!IS_GEN(dev_priv, 9)) + if (INTEL_GEN(dev_priv) < 9) return; cookie = i915_audio_component_get_power(kdev); @@ -1128,6 +1260,10 @@ static void i915_audio_component_unbind(struct device *i915_kdev, drm_modeset_unlock_all(&dev_priv->drm); device_link_remove(hda_kdev, i915_kdev); + + if (dev_priv->audio_power_refcount) + drm_err(&dev_priv->drm, "audio power refcount %d after unbind\n", + dev_priv->audio_power_refcount); } static const struct component_ops i915_audio_component_bind_ops = { @@ -1165,7 +1301,7 @@ static void i915_audio_component_init(struct drm_i915_private *dev_priv) return; } - if (IS_TIGERLAKE(dev_priv) || IS_ICELAKE(dev_priv)) { + if (INTEL_GEN(dev_priv) >= 9) { dev_priv->audio_freq_cntrl = intel_de_read(dev_priv, AUD_FREQ_CNTRL); drm_dbg_kms(&dev_priv->drm, diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index 2049cf5b54f3..839124647202 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -26,7 +26,6 @@ */ #include <drm/drm_dp_helper.h> -#include <drm/i915_drm.h> #include "display/intel_display.h" #include "display/intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_bios.h b/drivers/gpu/drm/i915/display/intel_bios.h index c17199caeff8..e29e79faa01b 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.h +++ b/drivers/gpu/drm/i915/display/intel_bios.h @@ -32,8 +32,6 @@ #include <linux/types.h> -#include <drm/i915_drm.h> - struct drm_i915_private; struct intel_crtc_state; struct intel_encoder; diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c index 58b264bc318d..6e7cc3a4f1aa 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.c +++ b/drivers/gpu/drm/i915/display/intel_bw.c @@ -338,16 +338,17 @@ void intel_bw_crtc_update(struct intel_bw_state *bw_state, const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); bw_state->data_rate[crtc->pipe] = intel_bw_crtc_data_rate(crtc_state); bw_state->num_active_planes[crtc->pipe] = intel_bw_crtc_num_active_planes(crtc_state); - DRM_DEBUG_KMS("pipe %c data rate %u num active planes %u\n", - pipe_name(crtc->pipe), - bw_state->data_rate[crtc->pipe], - bw_state->num_active_planes[crtc->pipe]); + drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n", + pipe_name(crtc->pipe), + bw_state->data_rate[crtc->pipe], + bw_state->num_active_planes[crtc->pipe]); } static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv, @@ -374,7 +375,29 @@ static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv, return data_rate; } -static struct intel_bw_state * +struct intel_bw_state * +intel_atomic_get_old_bw_state(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_global_state *bw_state; + + bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj); + + return to_intel_bw_state(bw_state); +} + +struct intel_bw_state * +intel_atomic_get_new_bw_state(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + struct intel_global_state *bw_state; + + bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj); + + return to_intel_bw_state(bw_state); +} + +struct intel_bw_state * intel_atomic_get_bw_state(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); @@ -391,7 +414,7 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); struct intel_crtc_state *new_crtc_state, *old_crtc_state; - struct intel_bw_state *bw_state = NULL; + struct intel_bw_state *new_bw_state = NULL; unsigned int data_rate, max_data_rate; unsigned int num_active_planes; struct intel_crtc *crtc; @@ -420,29 +443,29 @@ int intel_bw_atomic_check(struct intel_atomic_state *state) old_active_planes == new_active_planes) continue; - bw_state = intel_atomic_get_bw_state(state); - if (IS_ERR(bw_state)) - return PTR_ERR(bw_state); + new_bw_state = intel_atomic_get_bw_state(state); + if (IS_ERR(new_bw_state)) + return PTR_ERR(new_bw_state); - bw_state->data_rate[crtc->pipe] = new_data_rate; - bw_state->num_active_planes[crtc->pipe] = new_active_planes; + new_bw_state->data_rate[crtc->pipe] = new_data_rate; + new_bw_state->num_active_planes[crtc->pipe] = new_active_planes; drm_dbg_kms(&dev_priv->drm, "pipe %c data rate %u num active planes %u\n", pipe_name(crtc->pipe), - bw_state->data_rate[crtc->pipe], - bw_state->num_active_planes[crtc->pipe]); + new_bw_state->data_rate[crtc->pipe], + new_bw_state->num_active_planes[crtc->pipe]); } - if (!bw_state) + if (!new_bw_state) return 0; - ret = intel_atomic_lock_global_state(&bw_state->base); + ret = intel_atomic_lock_global_state(&new_bw_state->base); if (ret) return ret; - data_rate = intel_bw_data_rate(dev_priv, bw_state); - num_active_planes = intel_bw_num_active_planes(dev_priv, bw_state); + data_rate = intel_bw_data_rate(dev_priv, new_bw_state); + num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state); max_data_rate = intel_max_data_rate(dev_priv, num_active_planes); diff --git a/drivers/gpu/drm/i915/display/intel_bw.h b/drivers/gpu/drm/i915/display/intel_bw.h index a8aa7624c5aa..898b4a85ccab 100644 --- a/drivers/gpu/drm/i915/display/intel_bw.h +++ b/drivers/gpu/drm/i915/display/intel_bw.h @@ -18,12 +18,30 @@ struct intel_crtc_state; struct intel_bw_state { struct intel_global_state base; + /* + * Contains a bit mask, used to determine, whether correspondent + * pipe allows SAGV or not. + */ + u8 pipe_sagv_reject; + unsigned int data_rate[I915_MAX_PIPES]; u8 num_active_planes[I915_MAX_PIPES]; + + /* bitmask of active pipes */ + u8 active_pipes; }; #define to_intel_bw_state(x) container_of((x), struct intel_bw_state, base) +struct intel_bw_state * +intel_atomic_get_old_bw_state(struct intel_atomic_state *state); + +struct intel_bw_state * +intel_atomic_get_new_bw_state(struct intel_atomic_state *state); + +struct intel_bw_state * +intel_atomic_get_bw_state(struct intel_atomic_state *state); + void intel_bw_init_hw(struct drm_i915_private *dev_priv); int intel_bw_init(struct drm_i915_private *dev_priv); int intel_bw_atomic_check(struct intel_atomic_state *state); diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 146c2b9bb7fb..979a0241fdcb 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -525,7 +525,8 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv) * FIXME is this guaranteed to clear * immediately or should we poll for it? */ - WARN_ON(intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, GCI_CONTROL) & PFI_CREDIT_RESEND); } static void vlv_set_cdclk(struct drm_i915_private *dev_priv, @@ -727,12 +728,13 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv, u32 val; int ret; - if (WARN((intel_de_read(dev_priv, LCPLL_CTL) & - (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK | - LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE | - LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW | - LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK, - "trying to change cdclk frequency with cdclk not enabled\n")) + if (drm_WARN(&dev_priv->drm, + (intel_de_read(dev_priv, LCPLL_CTL) & + (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK | + LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE | + LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW | + LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK, + "trying to change cdclk frequency with cdclk not enabled\n")) return; ret = sandybridge_pcode_write(dev_priv, @@ -842,15 +844,16 @@ static void skl_dpll0_update(struct drm_i915_private *dev_priv, if ((val & LCPLL_PLL_ENABLE) == 0) return; - if (WARN_ON((val & LCPLL_PLL_LOCK) == 0)) + if (drm_WARN_ON(&dev_priv->drm, (val & LCPLL_PLL_LOCK) == 0)) return; val = intel_de_read(dev_priv, DPLL_CTRL1); - if (WARN_ON((val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | - DPLL_CTRL1_SSC(SKL_DPLL0) | - DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) != - DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) + if (drm_WARN_ON(&dev_priv->drm, + (val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | + DPLL_CTRL1_SSC(SKL_DPLL0) | + DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) != + DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) return; switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) { @@ -952,7 +955,7 @@ static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco) { u32 val; - WARN_ON(vco != 8100000 && vco != 8640000); + drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000); /* * We always enable DPLL0 with the lowest link rate possible, but still @@ -1017,7 +1020,8 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, * use the corresponding VCO freq as that always leads to using the * minimum 308MHz CDCLK. */ - WARN_ON_ONCE(IS_SKYLAKE(dev_priv) && vco == 8640000); + drm_WARN_ON_ONCE(&dev_priv->drm, + IS_SKYLAKE(dev_priv) && vco == 8640000); ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL, SKL_CDCLK_PREPARE_FOR_CHANGE, @@ -1032,8 +1036,9 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, /* Choose frequency for this cdclk */ switch (cdclk) { default: - WARN_ON(cdclk != dev_priv->cdclk.hw.bypass); - WARN_ON(vco != 0); + drm_WARN_ON(&dev_priv->drm, + cdclk != dev_priv->cdclk.hw.bypass); + drm_WARN_ON(&dev_priv->drm, vco != 0); /* fall through */ case 308571: case 337500: @@ -1235,8 +1240,9 @@ static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk) table[i].cdclk >= min_cdclk) return table[i].cdclk; - WARN(1, "Cannot satisfy minimum cdclk %d with refclk %u\n", - min_cdclk, dev_priv->cdclk.hw.ref); + drm_WARN(&dev_priv->drm, 1, + "Cannot satisfy minimum cdclk %d with refclk %u\n", + min_cdclk, dev_priv->cdclk.hw.ref); return 0; } @@ -1253,8 +1259,8 @@ static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk) table[i].cdclk == cdclk) return dev_priv->cdclk.hw.ref * table[i].ratio; - WARN(1, "cdclk %d not valid for refclk %u\n", - cdclk, dev_priv->cdclk.hw.ref); + drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n", + cdclk, dev_priv->cdclk.hw.ref); return 0; } @@ -1399,15 +1405,17 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv, div = 2; break; case BXT_CDCLK_CD2X_DIV_SEL_1_5: - WARN(IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10, - "Unsupported divider\n"); + drm_WARN(&dev_priv->drm, + IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10, + "Unsupported divider\n"); div = 3; break; case BXT_CDCLK_CD2X_DIV_SEL_2: div = 4; break; case BXT_CDCLK_CD2X_DIV_SEL_4: - WARN(INTEL_GEN(dev_priv) >= 10, "Unsupported divider\n"); + drm_WARN(&dev_priv->drm, INTEL_GEN(dev_priv) >= 10, + "Unsupported divider\n"); div = 8; break; default: @@ -1547,22 +1555,25 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv, /* cdclk = vco / 2 / div{1,1.5,2,4} */ switch (DIV_ROUND_CLOSEST(vco, cdclk)) { default: - WARN_ON(cdclk != dev_priv->cdclk.hw.bypass); - WARN_ON(vco != 0); + drm_WARN_ON(&dev_priv->drm, + cdclk != dev_priv->cdclk.hw.bypass); + drm_WARN_ON(&dev_priv->drm, vco != 0); /* fall through */ case 2: divider = BXT_CDCLK_CD2X_DIV_SEL_1; break; case 3: - WARN(IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10, - "Unsupported divider\n"); + drm_WARN(&dev_priv->drm, + IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10, + "Unsupported divider\n"); divider = BXT_CDCLK_CD2X_DIV_SEL_1_5; break; case 4: divider = BXT_CDCLK_CD2X_DIV_SEL_2; break; case 8: - WARN(INTEL_GEN(dev_priv) >= 10, "Unsupported divider\n"); + drm_WARN(&dev_priv->drm, INTEL_GEN(dev_priv) >= 10, + "Unsupported divider\n"); divider = BXT_CDCLK_CD2X_DIV_SEL_4; break; } @@ -1857,18 +1868,41 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv, const struct intel_cdclk_config *cdclk_config, enum pipe pipe) { + struct intel_encoder *encoder; + if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config)) return; - if (WARN_ON_ONCE(!dev_priv->display.set_cdclk)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.set_cdclk)) return; intel_dump_cdclk_config(cdclk_config, "Changing CDCLK to"); + /* + * Lock aux/gmbus while we change cdclk in case those + * functions use cdclk. Not all platforms/ports do, + * but we'll lock them all for simplicity. + */ + mutex_lock(&dev_priv->gmbus_mutex); + for_each_intel_dp(&dev_priv->drm, encoder) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + mutex_lock_nest_lock(&intel_dp->aux.hw_mutex, + &dev_priv->gmbus_mutex); + } + dev_priv->display.set_cdclk(dev_priv, cdclk_config, pipe); - if (WARN(intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config), - "cdclk state doesn't match!\n")) { + for_each_intel_dp(&dev_priv->drm, encoder) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + + mutex_unlock(&intel_dp->aux.hw_mutex); + } + mutex_unlock(&dev_priv->gmbus_mutex); + + if (drm_WARN(&dev_priv->drm, + intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config), + "cdclk state doesn't match!\n")) { intel_dump_cdclk_config(&dev_priv->cdclk.hw, "[hw state]"); intel_dump_cdclk_config(cdclk_config, "[sw state]"); } @@ -1897,7 +1931,7 @@ intel_set_cdclk_pre_plane_update(struct intel_atomic_state *state) if (pipe == INVALID_PIPE || old_cdclk_state->actual.cdclk <= new_cdclk_state->actual.cdclk) { - WARN_ON(!new_cdclk_state->base.changed); + drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed); intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe); } @@ -1926,7 +1960,7 @@ intel_set_cdclk_post_plane_update(struct intel_atomic_state *state) if (pipe != INVALID_PIPE && old_cdclk_state->actual.cdclk > new_cdclk_state->actual.cdclk) { - WARN_ON(!new_cdclk_state->base.changed); + drm_WARN_ON(&dev_priv->drm, !new_cdclk_state->base.changed); intel_set_cdclk(dev_priv, &new_cdclk_state->actual, pipe); } @@ -2550,7 +2584,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv) int max_cdclk, vco; vco = dev_priv->skl_preferred_vco_freq; - WARN_ON(vco != 8100000 && vco != 8640000); + drm_WARN_ON(&dev_priv->drm, vco != 8100000 && vco != 8640000); /* * Use the lower (vco 8640) cdclk values as a @@ -2809,8 +2843,8 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv) else if (IS_I845G(dev_priv)) dev_priv->display.get_cdclk = fixed_200mhz_get_cdclk; else { /* 830 */ - WARN(!IS_I830(dev_priv), - "Unknown platform. Assuming 133 MHz CDCLK\n"); + drm_WARN(&dev_priv->drm, !IS_I830(dev_priv), + "Unknown platform. Assuming 133 MHz CDCLK\n"); dev_priv->display.get_cdclk = fixed_133mhz_get_cdclk; } } diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index d44bd8287801..98ece9cd7cdd 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -348,48 +348,70 @@ static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state) crtc_state->csc_mode); } -/* - * Set up the pipe CSC unit on CherryView. - */ -static void cherryview_load_csc_matrix(const struct intel_crtc_state *crtc_state) +static void chv_load_cgm_csc(struct intel_crtc *crtc, + const struct drm_property_blob *blob) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_color_ctm *ctm = blob->data; enum pipe pipe = crtc->pipe; + u16 coeffs[9]; + int i; - if (crtc_state->hw.ctm) { - const struct drm_color_ctm *ctm = crtc_state->hw.ctm->data; - u16 coeffs[9] = {}; - int i; - - for (i = 0; i < ARRAY_SIZE(coeffs); i++) { - u64 abs_coeff = - ((1ULL << 63) - 1) & ctm->matrix[i]; - - /* Round coefficient. */ - abs_coeff += 1 << (32 - 13); - /* Clamp to hardware limits. */ - abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_8_0 - 1); - - /* Write coefficients in S3.12 format. */ - if (ctm->matrix[i] & (1ULL << 63)) - coeffs[i] = 1 << 15; - coeffs[i] |= ((abs_coeff >> 32) & 7) << 12; - coeffs[i] |= (abs_coeff >> 20) & 0xfff; - } + for (i = 0; i < ARRAY_SIZE(coeffs); i++) { + u64 abs_coeff = ((1ULL << 63) - 1) & ctm->matrix[i]; + + /* Round coefficient. */ + abs_coeff += 1 << (32 - 13); + /* Clamp to hardware limits. */ + abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_8_0 - 1); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF01(pipe), - coeffs[1] << 16 | coeffs[0]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF23(pipe), - coeffs[3] << 16 | coeffs[2]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF45(pipe), - coeffs[5] << 16 | coeffs[4]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF67(pipe), - coeffs[7] << 16 | coeffs[6]); - intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF8(pipe), coeffs[8]); + coeffs[i] = 0; + + /* Write coefficients in S3.12 format. */ + if (ctm->matrix[i] & (1ULL << 63)) + coeffs[i] |= 1 << 15; + + coeffs[i] |= ((abs_coeff >> 32) & 7) << 12; + coeffs[i] |= (abs_coeff >> 20) & 0xfff; } - intel_de_write(dev_priv, CGM_PIPE_MODE(pipe), crtc_state->cgm_mode); + intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF01(pipe), + coeffs[1] << 16 | coeffs[0]); + intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF23(pipe), + coeffs[3] << 16 | coeffs[2]); + intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF45(pipe), + coeffs[5] << 16 | coeffs[4]); + intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF67(pipe), + coeffs[7] << 16 | coeffs[6]); + intel_de_write(dev_priv, CGM_PIPE_CSC_COEFF8(pipe), + coeffs[8]); +} + +/* convert hw value with given bit_precision to lut property val */ +static u32 intel_color_lut_pack(u32 val, int bit_precision) +{ + u32 max = 0xffff >> (16 - bit_precision); + + val = clamp_val(val, 0, max); + + if (bit_precision < 16) + val <<= 16 - bit_precision; + + return val; +} + +static u32 i9xx_lut_8(const struct drm_color_lut *color) +{ + return drm_color_lut_extract(color->red, 8) << 16 | + drm_color_lut_extract(color->green, 8) << 8 | + drm_color_lut_extract(color->blue, 8); +} + +static void i9xx_lut_8_pack(struct drm_color_lut *entry, u32 val) +{ + entry->red = intel_color_lut_pack(REG_FIELD_GET(LGC_PALETTE_RED_MASK, val), 8); + entry->green = intel_color_lut_pack(REG_FIELD_GET(LGC_PALETTE_GREEN_MASK, val), 8); + entry->blue = intel_color_lut_pack(REG_FIELD_GET(LGC_PALETTE_BLUE_MASK, val), 8); } /* i965+ "10.6" bit interpolated format "even DW" (low 8 bits) */ @@ -408,6 +430,22 @@ static u32 i965_lut_10p6_udw(const struct drm_color_lut *color) (color->blue >> 8); } +static void i965_lut_10p6_pack(struct drm_color_lut *entry, u32 ldw, u32 udw) +{ + entry->red = REG_FIELD_GET(PALETTE_RED_MASK, udw) << 8 | + REG_FIELD_GET(PALETTE_RED_MASK, ldw); + entry->green = REG_FIELD_GET(PALETTE_GREEN_MASK, udw) << 8 | + REG_FIELD_GET(PALETTE_GREEN_MASK, ldw); + entry->blue = REG_FIELD_GET(PALETTE_BLUE_MASK, udw) << 8 | + REG_FIELD_GET(PALETTE_BLUE_MASK, ldw); +} + +static u16 i965_lut_11p6_max_pack(u32 val) +{ + /* PIPEGCMAX is 11.6, clamp to 10.6 */ + return clamp_val(val, 0, 0xffff); +} + static u32 ilk_lut_10(const struct drm_color_lut *color) { return drm_color_lut_extract(color->red, 10) << 20 | @@ -415,44 +453,21 @@ static u32 ilk_lut_10(const struct drm_color_lut *color) drm_color_lut_extract(color->blue, 10); } -/* Loads the legacy palette/gamma unit for the CRTC. */ -static void i9xx_load_luts_internal(const struct intel_crtc_state *crtc_state, - const struct drm_property_blob *blob) +static void ilk_lut_10_pack(struct drm_color_lut *entry, u32 val) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; - int i; - - if (HAS_GMCH(dev_priv)) { - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) - assert_dsi_pll_enabled(dev_priv); - else - assert_pll_enabled(dev_priv, pipe); - } - - if (blob) { - const struct drm_color_lut *lut = blob->data; - - for (i = 0; i < 256; i++) { - u32 word = - (drm_color_lut_extract(lut[i].red, 8) << 16) | - (drm_color_lut_extract(lut[i].green, 8) << 8) | - drm_color_lut_extract(lut[i].blue, 8); - - if (HAS_GMCH(dev_priv)) - intel_de_write(dev_priv, PALETTE(pipe, i), - word); - else - intel_de_write(dev_priv, LGC_PALETTE(pipe, i), - word); - } - } + entry->red = intel_color_lut_pack(REG_FIELD_GET(PREC_PALETTE_RED_MASK, val), 10); + entry->green = intel_color_lut_pack(REG_FIELD_GET(PREC_PALETTE_GREEN_MASK, val), 10); + entry->blue = intel_color_lut_pack(REG_FIELD_GET(PREC_PALETTE_BLUE_MASK, val), 10); } -static void i9xx_load_luts(const struct intel_crtc_state *crtc_state) +static void icl_lut_multi_seg_pack(struct drm_color_lut *entry, u32 ldw, u32 udw) { - i9xx_load_luts_internal(crtc_state, crtc_state->hw.gamma_lut); + entry->red = REG_FIELD_GET(PAL_PREC_MULTI_SEG_RED_UDW_MASK, udw) << 6 | + REG_FIELD_GET(PAL_PREC_MULTI_SEG_RED_LDW_MASK, ldw); + entry->green = REG_FIELD_GET(PAL_PREC_MULTI_SEG_GREEN_UDW_MASK, udw) << 6 | + REG_FIELD_GET(PAL_PREC_MULTI_SEG_GREEN_LDW_MASK, ldw); + entry->blue = REG_FIELD_GET(PAL_PREC_MULTI_SEG_BLUE_UDW_MASK, udw) << 6 | + REG_FIELD_GET(PAL_PREC_MULTI_SEG_BLUE_LDW_MASK, ldw); } static void i9xx_color_commit(const struct intel_crtc_state *crtc_state) @@ -521,6 +536,35 @@ static void skl_color_commit(const struct intel_crtc_state *crtc_state) ilk_load_csc_matrix(crtc_state); } +static void i9xx_load_lut_8(struct intel_crtc *crtc, + const struct drm_property_blob *blob) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_color_lut *lut; + enum pipe pipe = crtc->pipe; + int i; + + if (!blob) + return; + + lut = blob->data; + + for (i = 0; i < 256; i++) + intel_de_write(dev_priv, PALETTE(pipe, i), + i9xx_lut_8(&lut[i])); +} + +static void i9xx_load_luts(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_property_blob *gamma_lut = crtc_state->hw.gamma_lut; + + assert_pll_enabled(dev_priv, crtc->pipe); + + i9xx_load_lut_8(crtc, gamma_lut); +} + static void i965_load_lut_10p6(struct intel_crtc *crtc, const struct drm_property_blob *blob) { @@ -544,14 +588,38 @@ static void i965_load_lut_10p6(struct intel_crtc *crtc, static void i965_load_luts(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct drm_property_blob *gamma_lut = crtc_state->hw.gamma_lut; + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) + assert_dsi_pll_enabled(dev_priv); + else + assert_pll_enabled(dev_priv, crtc->pipe); + if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) - i9xx_load_luts(crtc_state); + i9xx_load_lut_8(crtc, gamma_lut); else i965_load_lut_10p6(crtc, gamma_lut); } +static void ilk_load_lut_8(struct intel_crtc *crtc, + const struct drm_property_blob *blob) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_color_lut *lut; + enum pipe pipe = crtc->pipe; + int i; + + if (!blob) + return; + + lut = blob->data; + + for (i = 0; i < 256; i++) + intel_de_write(dev_priv, LGC_PALETTE(pipe, i), + i9xx_lut_8(&lut[i])); +} + static void ilk_load_lut_10(struct intel_crtc *crtc, const struct drm_property_blob *blob) { @@ -562,7 +630,7 @@ static void ilk_load_lut_10(struct intel_crtc *crtc, for (i = 0; i < lut_size; i++) intel_de_write(dev_priv, PREC_PALETTE(pipe, i), - ilk_lut_10(&lut[i])); + ilk_lut_10(&lut[i])); } static void ilk_load_luts(const struct intel_crtc_state *crtc_state) @@ -571,7 +639,7 @@ static void ilk_load_luts(const struct intel_crtc_state *crtc_state) const struct drm_property_blob *gamma_lut = crtc_state->hw.gamma_lut; if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) - i9xx_load_luts(crtc_state); + ilk_load_lut_8(crtc, gamma_lut); else ilk_load_lut_10(crtc, gamma_lut); } @@ -681,7 +749,7 @@ static void ivb_load_luts(const struct intel_crtc_state *crtc_state) const struct drm_property_blob *degamma_lut = crtc_state->hw.degamma_lut; if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) { - i9xx_load_luts(crtc_state); + ilk_load_lut_8(crtc, gamma_lut); } else if (crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) { ivb_load_lut_10(crtc, degamma_lut, PAL_PREC_SPLIT_MODE | PAL_PREC_INDEX_VALUE(0)); @@ -704,7 +772,7 @@ static void bdw_load_luts(const struct intel_crtc_state *crtc_state) const struct drm_property_blob *degamma_lut = crtc_state->hw.degamma_lut; if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) { - i9xx_load_luts(crtc_state); + ilk_load_lut_8(crtc, gamma_lut); } else if (crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT) { bdw_load_lut_10(crtc, degamma_lut, PAL_PREC_SPLIT_MODE | PAL_PREC_INDEX_VALUE(0)); @@ -725,9 +793,8 @@ static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state) struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - const u32 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; + int i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; const struct drm_color_lut *lut = crtc_state->hw.degamma_lut->data; - u32 i; /* * When setting the auto-increment bit, the hardware seems to @@ -766,8 +833,7 @@ static void glk_load_degamma_lut_linear(const struct intel_crtc_state *crtc_stat struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - const u32 lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; - u32 i; + int i, lut_size = INTEL_INFO(dev_priv)->color.degamma_lut_size; /* * When setting the auto-increment bit, the hardware seems to @@ -808,7 +874,7 @@ static void glk_load_luts(const struct intel_crtc_state *crtc_state) glk_load_degamma_lut_linear(crtc_state); if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) { - i9xx_load_luts(crtc_state); + ilk_load_lut_8(crtc, gamma_lut); } else { bdw_load_lut_10(crtc, gamma_lut, PAL_PREC_INDEX_VALUE(0)); ivb_load_lut_ext_max(crtc); @@ -837,7 +903,7 @@ icl_load_gcmax(const struct intel_crtc_state *crtc_state, struct intel_dsb *dsb = intel_dsb_get(crtc); enum pipe pipe = crtc->pipe; - /* Fixme: LUT entries are 16 bit only, so we can prog 0xFFFF max */ + /* FIXME LUT entries are 16 bit only, so we can prog 0xFFFF max */ intel_dsb_reg_write(dsb, PREC_PAL_GC_MAX(pipe, 0), color->red); intel_dsb_reg_write(dsb, PREC_PAL_GC_MAX(pipe, 1), color->green); intel_dsb_reg_write(dsb, PREC_PAL_GC_MAX(pipe, 2), color->blue); @@ -852,7 +918,7 @@ icl_program_gamma_superfine_segment(const struct intel_crtc_state *crtc_state) const struct drm_color_lut *lut = blob->data; struct intel_dsb *dsb = intel_dsb_get(crtc); enum pipe pipe = crtc->pipe; - u32 i; + int i; /* * Program Super Fine segment (let's call it seg1)... @@ -885,7 +951,7 @@ icl_program_gamma_multi_segment(const struct intel_crtc_state *crtc_state) const struct drm_color_lut *entry; struct intel_dsb *dsb = intel_dsb_get(crtc); enum pipe pipe = crtc->pipe; - u32 i; + int i; /* * Program Fine segment (let's call it seg2)... @@ -944,7 +1010,7 @@ static void icl_load_luts(const struct intel_crtc_state *crtc_state) switch (crtc_state->gamma_mode & GAMMA_MODE_MODE_MASK) { case GAMMA_MODE_MODE_8BIT: - i9xx_load_luts(crtc_state); + ilk_load_lut_8(crtc, gamma_lut); break; case GAMMA_MODE_MODE_12BIT_MULTI_SEGMENTED: icl_program_gamma_superfine_segment(crtc_state); @@ -970,6 +1036,13 @@ static u32 chv_cgm_degamma_udw(const struct drm_color_lut *color) return drm_color_lut_extract(color->red, 14); } +static void chv_cgm_gamma_pack(struct drm_color_lut *entry, u32 ldw, u32 udw) +{ + entry->green = intel_color_lut_pack(REG_FIELD_GET(CGM_PIPE_GAMMA_GREEN_MASK, ldw), 10); + entry->blue = intel_color_lut_pack(REG_FIELD_GET(CGM_PIPE_GAMMA_BLUE_MASK, ldw), 10); + entry->red = intel_color_lut_pack(REG_FIELD_GET(CGM_PIPE_GAMMA_RED_MASK, udw), 10); +} + static void chv_load_cgm_degamma(struct intel_crtc *crtc, const struct drm_property_blob *blob) { @@ -1016,21 +1089,24 @@ static void chv_load_cgm_gamma(struct intel_crtc *crtc, static void chv_load_luts(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - const struct drm_property_blob *gamma_lut = crtc_state->hw.gamma_lut; + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); const struct drm_property_blob *degamma_lut = crtc_state->hw.degamma_lut; + const struct drm_property_blob *gamma_lut = crtc_state->hw.gamma_lut; + const struct drm_property_blob *ctm = crtc_state->hw.ctm; - cherryview_load_csc_matrix(crtc_state); - - if (crtc_state_is_legacy_gamma(crtc_state)) { - i9xx_load_luts(crtc_state); - return; - } + if (crtc_state->cgm_mode & CGM_PIPE_MODE_CSC) + chv_load_cgm_csc(crtc, ctm); - if (degamma_lut) + if (crtc_state->cgm_mode & CGM_PIPE_MODE_DEGAMMA) chv_load_cgm_degamma(crtc, degamma_lut); - if (gamma_lut) + if (crtc_state->cgm_mode & CGM_PIPE_MODE_GAMMA) chv_load_cgm_gamma(crtc, gamma_lut); + else + i965_load_luts(crtc_state); + + intel_de_write(dev_priv, CGM_PIPE_MODE(crtc->pipe), + crtc_state->cgm_mode); } void intel_color_load_luts(const struct intel_crtc_state *crtc_state) @@ -1564,6 +1640,24 @@ static int glk_gamma_precision(const struct intel_crtc_state *crtc_state) } } +static int icl_gamma_precision(const struct intel_crtc_state *crtc_state) +{ + if ((crtc_state->gamma_mode & POST_CSC_GAMMA_ENABLE) == 0) + return 0; + + switch (crtc_state->gamma_mode & GAMMA_MODE_MODE_MASK) { + case GAMMA_MODE_MODE_8BIT: + return 8; + case GAMMA_MODE_MODE_10BIT: + return 10; + case GAMMA_MODE_MODE_12BIT_MULTI_SEGMENTED: + return 16; + default: + MISSING_CASE(crtc_state->gamma_mode); + return 0; + } +} + int intel_color_get_gamma_bit_precision(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); @@ -1575,7 +1669,9 @@ int intel_color_get_gamma_bit_precision(const struct intel_crtc_state *crtc_stat else return i9xx_gamma_precision(crtc_state); } else { - if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) + if (INTEL_GEN(dev_priv) >= 11) + return icl_gamma_precision(crtc_state); + else if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) return glk_gamma_precision(crtc_state); else if (IS_IRONLAKE(dev_priv)) return ilk_gamma_precision(crtc_state); @@ -1592,9 +1688,9 @@ static bool err_check(struct drm_color_lut *lut1, ((abs((long)lut2->green - lut1->green)) <= err); } -static bool intel_color_lut_entry_equal(struct drm_color_lut *lut1, - struct drm_color_lut *lut2, - int lut_size, u32 err) +static bool intel_color_lut_entries_equal(struct drm_color_lut *lut1, + struct drm_color_lut *lut2, + int lut_size, u32 err) { int i; @@ -1624,16 +1720,8 @@ bool intel_color_lut_equal(struct drm_property_blob *blob1, lut_size2 = drm_color_lut_size(blob2); /* check sw and hw lut size */ - switch (gamma_mode) { - case GAMMA_MODE_MODE_8BIT: - case GAMMA_MODE_MODE_10BIT: - if (lut_size1 != lut_size2) - return false; - break; - default: - MISSING_CASE(gamma_mode); - return false; - } + if (lut_size1 != lut_size2) + return false; lut1 = blob1->data; lut2 = blob2->data; @@ -1641,11 +1729,16 @@ bool intel_color_lut_equal(struct drm_property_blob *blob1, err = 0xffff >> bit_precision; /* check sw and hw lut entry to be equal */ - switch (gamma_mode) { + switch (gamma_mode & GAMMA_MODE_MODE_MASK) { case GAMMA_MODE_MODE_8BIT: case GAMMA_MODE_MODE_10BIT: - if (!intel_color_lut_entry_equal(lut1, lut2, - lut_size2, err)) + if (!intel_color_lut_entries_equal(lut1, lut2, + lut_size2, err)) + return false; + break; + case GAMMA_MODE_MODE_12BIT_MULTI_SEGMENTED: + if (!intel_color_lut_entries_equal(lut1, lut2, + 9, err)) return false; break; default: @@ -1656,28 +1749,13 @@ bool intel_color_lut_equal(struct drm_property_blob *blob1, return true; } -/* convert hw value with given bit_precision to lut property val */ -static u32 intel_color_lut_pack(u32 val, u32 bit_precision) -{ - u32 max = 0xffff >> (16 - bit_precision); - - val = clamp_val(val, 0, max); - - if (bit_precision < 16) - val <<= 16 - bit_precision; - - return val; -} - -static struct drm_property_blob * -i9xx_read_lut_8(const struct intel_crtc_state *crtc_state) +static struct drm_property_blob *i9xx_read_lut_8(struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; - struct drm_color_lut *blob_data; - u32 i, val; + struct drm_color_lut *lut; + int i; blob = drm_property_create_blob(&dev_priv->drm, sizeof(struct drm_color_lut) * LEGACY_LUT_LENGTH, @@ -1685,20 +1763,12 @@ i9xx_read_lut_8(const struct intel_crtc_state *crtc_state) if (IS_ERR(blob)) return NULL; - blob_data = blob->data; + lut = blob->data; for (i = 0; i < LEGACY_LUT_LENGTH; i++) { - if (HAS_GMCH(dev_priv)) - val = intel_de_read(dev_priv, PALETTE(pipe, i)); - else - val = intel_de_read(dev_priv, LGC_PALETTE(pipe, i)); - - blob_data[i].red = intel_color_lut_pack(REG_FIELD_GET( - LGC_PALETTE_RED_MASK, val), 8); - blob_data[i].green = intel_color_lut_pack(REG_FIELD_GET( - LGC_PALETTE_GREEN_MASK, val), 8); - blob_data[i].blue = intel_color_lut_pack(REG_FIELD_GET( - LGC_PALETTE_BLUE_MASK, val), 8); + u32 val = intel_de_read(dev_priv, PALETTE(pipe, i)); + + i9xx_lut_8_pack(&lut[i], val); } return blob; @@ -1706,22 +1776,21 @@ i9xx_read_lut_8(const struct intel_crtc_state *crtc_state) static void i9xx_read_luts(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + if (!crtc_state->gamma_enable) return; - crtc_state->hw.gamma_lut = i9xx_read_lut_8(crtc_state); + crtc_state->hw.gamma_lut = i9xx_read_lut_8(crtc); } -static struct drm_property_blob * -i965_read_lut_10p6(const struct intel_crtc_state *crtc_state) +static struct drm_property_blob *i965_read_lut_10p6(struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + int i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; - struct drm_color_lut *blob_data; - u32 i, val1, val2; + struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, sizeof(struct drm_color_lut) * lut_size, @@ -1729,51 +1798,42 @@ i965_read_lut_10p6(const struct intel_crtc_state *crtc_state) if (IS_ERR(blob)) return NULL; - blob_data = blob->data; + lut = blob->data; for (i = 0; i < lut_size - 1; i++) { - val1 = intel_de_read(dev_priv, PALETTE(pipe, 2 * i + 0)); - val2 = intel_de_read(dev_priv, PALETTE(pipe, 2 * i + 1)); - - blob_data[i].red = REG_FIELD_GET(PALETTE_RED_MASK, val2) << 8 | - REG_FIELD_GET(PALETTE_RED_MASK, val1); - blob_data[i].green = REG_FIELD_GET(PALETTE_GREEN_MASK, val2) << 8 | - REG_FIELD_GET(PALETTE_GREEN_MASK, val1); - blob_data[i].blue = REG_FIELD_GET(PALETTE_BLUE_MASK, val2) << 8 | - REG_FIELD_GET(PALETTE_BLUE_MASK, val1); + u32 ldw = intel_de_read(dev_priv, PALETTE(pipe, 2 * i + 0)); + u32 udw = intel_de_read(dev_priv, PALETTE(pipe, 2 * i + 1)); + + i965_lut_10p6_pack(&lut[i], ldw, udw); } - blob_data[i].red = REG_FIELD_GET(PIPEGCMAX_RGB_MASK, - intel_de_read(dev_priv, PIPEGCMAX(pipe, 0))); - blob_data[i].green = REG_FIELD_GET(PIPEGCMAX_RGB_MASK, - intel_de_read(dev_priv, PIPEGCMAX(pipe, 1))); - blob_data[i].blue = REG_FIELD_GET(PIPEGCMAX_RGB_MASK, - intel_de_read(dev_priv, PIPEGCMAX(pipe, 2))); + lut[i].red = i965_lut_11p6_max_pack(intel_de_read(dev_priv, PIPEGCMAX(pipe, 0))); + lut[i].green = i965_lut_11p6_max_pack(intel_de_read(dev_priv, PIPEGCMAX(pipe, 1))); + lut[i].blue = i965_lut_11p6_max_pack(intel_de_read(dev_priv, PIPEGCMAX(pipe, 2))); return blob; } static void i965_read_luts(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + if (!crtc_state->gamma_enable) return; if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) - crtc_state->hw.gamma_lut = i9xx_read_lut_8(crtc_state); + crtc_state->hw.gamma_lut = i9xx_read_lut_8(crtc); else - crtc_state->hw.gamma_lut = i965_read_lut_10p6(crtc_state); + crtc_state->hw.gamma_lut = i965_read_lut_10p6(crtc); } -static struct drm_property_blob * -chv_read_cgm_lut(const struct intel_crtc_state *crtc_state) +static struct drm_property_blob *chv_read_cgm_gamma(struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + int i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; - struct drm_color_lut *blob_data; - u32 i, val; + struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, sizeof(struct drm_color_lut) * lut_size, @@ -1781,18 +1841,13 @@ chv_read_cgm_lut(const struct intel_crtc_state *crtc_state) if (IS_ERR(blob)) return NULL; - blob_data = blob->data; + lut = blob->data; for (i = 0; i < lut_size; i++) { - val = intel_de_read(dev_priv, CGM_PIPE_GAMMA(pipe, i, 0)); - blob_data[i].green = intel_color_lut_pack(REG_FIELD_GET( - CGM_PIPE_GAMMA_GREEN_MASK, val), 10); - blob_data[i].blue = intel_color_lut_pack(REG_FIELD_GET( - CGM_PIPE_GAMMA_BLUE_MASK, val), 10); - - val = intel_de_read(dev_priv, CGM_PIPE_GAMMA(pipe, i, 1)); - blob_data[i].red = intel_color_lut_pack(REG_FIELD_GET( - CGM_PIPE_GAMMA_RED_MASK, val), 10); + u32 ldw = intel_de_read(dev_priv, CGM_PIPE_GAMMA(pipe, i, 0)); + u32 udw = intel_de_read(dev_priv, CGM_PIPE_GAMMA(pipe, i, 1)); + + chv_cgm_gamma_pack(&lut[i], ldw, udw); } return blob; @@ -1800,22 +1855,46 @@ chv_read_cgm_lut(const struct intel_crtc_state *crtc_state) static void chv_read_luts(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + if (crtc_state->cgm_mode & CGM_PIPE_MODE_GAMMA) - crtc_state->hw.gamma_lut = chv_read_cgm_lut(crtc_state); + crtc_state->hw.gamma_lut = chv_read_cgm_gamma(crtc); else i965_read_luts(crtc_state); } -static struct drm_property_blob * -ilk_read_lut_10(const struct intel_crtc_state *crtc_state) +static struct drm_property_blob *ilk_read_lut_8(struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; - struct drm_color_lut *blob_data; - u32 i, val; + struct drm_color_lut *lut; + int i; + + blob = drm_property_create_blob(&dev_priv->drm, + sizeof(struct drm_color_lut) * LEGACY_LUT_LENGTH, + NULL); + if (IS_ERR(blob)) + return NULL; + + lut = blob->data; + + for (i = 0; i < LEGACY_LUT_LENGTH; i++) { + u32 val = intel_de_read(dev_priv, LGC_PALETTE(pipe, i)); + + i9xx_lut_8_pack(&lut[i], val); + } + + return blob; +} + +static struct drm_property_blob *ilk_read_lut_10(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + int i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + enum pipe pipe = crtc->pipe; + struct drm_property_blob *blob; + struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, sizeof(struct drm_color_lut) * lut_size, @@ -1823,17 +1902,12 @@ ilk_read_lut_10(const struct intel_crtc_state *crtc_state) if (IS_ERR(blob)) return NULL; - blob_data = blob->data; + lut = blob->data; for (i = 0; i < lut_size; i++) { - val = intel_de_read(dev_priv, PREC_PALETTE(pipe, i)); - - blob_data[i].red = intel_color_lut_pack(REG_FIELD_GET( - PREC_PALETTE_RED_MASK, val), 10); - blob_data[i].green = intel_color_lut_pack(REG_FIELD_GET( - PREC_PALETTE_GREEN_MASK, val), 10); - blob_data[i].blue = intel_color_lut_pack(REG_FIELD_GET( - PREC_PALETTE_BLUE_MASK, val), 10); + u32 val = intel_de_read(dev_priv, PREC_PALETTE(pipe, i)); + + ilk_lut_10_pack(&lut[i], val); } return blob; @@ -1841,6 +1915,8 @@ ilk_read_lut_10(const struct intel_crtc_state *crtc_state) static void ilk_read_luts(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + if (!crtc_state->gamma_enable) return; @@ -1848,21 +1924,19 @@ static void ilk_read_luts(struct intel_crtc_state *crtc_state) return; if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) - crtc_state->hw.gamma_lut = i9xx_read_lut_8(crtc_state); + crtc_state->hw.gamma_lut = ilk_read_lut_8(crtc); else - crtc_state->hw.gamma_lut = ilk_read_lut_10(crtc_state); + crtc_state->hw.gamma_lut = ilk_read_lut_10(crtc); } -static struct drm_property_blob * -glk_read_lut_10(const struct intel_crtc_state *crtc_state, u32 prec_index) +static struct drm_property_blob *glk_read_lut_10(struct intel_crtc *crtc, + u32 prec_index) { - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - int hw_lut_size = ivb_lut_10_size(prec_index); + int i, hw_lut_size = ivb_lut_10_size(prec_index); enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; - struct drm_color_lut *blob_data; - u32 i, val; + struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, sizeof(struct drm_color_lut) * hw_lut_size, @@ -1870,20 +1944,15 @@ glk_read_lut_10(const struct intel_crtc_state *crtc_state, u32 prec_index) if (IS_ERR(blob)) return NULL; - blob_data = blob->data; + lut = blob->data; intel_de_write(dev_priv, PREC_PAL_INDEX(pipe), prec_index | PAL_PREC_AUTO_INCREMENT); for (i = 0; i < hw_lut_size; i++) { - val = intel_de_read(dev_priv, PREC_PAL_DATA(pipe)); - - blob_data[i].red = intel_color_lut_pack(REG_FIELD_GET( - PREC_PAL_DATA_RED_MASK, val), 10); - blob_data[i].green = intel_color_lut_pack(REG_FIELD_GET( - PREC_PAL_DATA_GREEN_MASK, val), 10); - blob_data[i].blue = intel_color_lut_pack(REG_FIELD_GET( - PREC_PAL_DATA_BLUE_MASK, val), 10); + u32 val = intel_de_read(dev_priv, PREC_PAL_DATA(pipe)); + + ilk_lut_10_pack(&lut[i], val); } intel_de_write(dev_priv, PREC_PAL_INDEX(pipe), 0); @@ -1893,13 +1962,72 @@ glk_read_lut_10(const struct intel_crtc_state *crtc_state, u32 prec_index) static void glk_read_luts(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + if (!crtc_state->gamma_enable) return; if (crtc_state->gamma_mode == GAMMA_MODE_MODE_8BIT) - crtc_state->hw.gamma_lut = i9xx_read_lut_8(crtc_state); + crtc_state->hw.gamma_lut = ilk_read_lut_8(crtc); else - crtc_state->hw.gamma_lut = glk_read_lut_10(crtc_state, PAL_PREC_INDEX_VALUE(0)); + crtc_state->hw.gamma_lut = glk_read_lut_10(crtc, PAL_PREC_INDEX_VALUE(0)); +} + +static struct drm_property_blob * +icl_read_lut_multi_segment(struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + int i, lut_size = INTEL_INFO(dev_priv)->color.gamma_lut_size; + enum pipe pipe = crtc->pipe; + struct drm_property_blob *blob; + struct drm_color_lut *lut; + + blob = drm_property_create_blob(&dev_priv->drm, + sizeof(struct drm_color_lut) * lut_size, + NULL); + if (IS_ERR(blob)) + return NULL; + + lut = blob->data; + + intel_de_write(dev_priv, PREC_PAL_MULTI_SEG_INDEX(pipe), + PAL_PREC_AUTO_INCREMENT); + + for (i = 0; i < 9; i++) { + u32 ldw = intel_de_read(dev_priv, PREC_PAL_MULTI_SEG_DATA(pipe)); + u32 udw = intel_de_read(dev_priv, PREC_PAL_MULTI_SEG_DATA(pipe)); + + icl_lut_multi_seg_pack(&lut[i], ldw, udw); + } + + intel_de_write(dev_priv, PREC_PAL_MULTI_SEG_INDEX(pipe), 0); + + /* + * FIXME readouts from PAL_PREC_DATA register aren't giving + * correct values in the case of fine and coarse segments. + * Restricting readouts only for super fine segment as of now. + */ + + return blob; +} + +static void icl_read_luts(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + + if ((crtc_state->gamma_mode & POST_CSC_GAMMA_ENABLE) == 0) + return; + + switch (crtc_state->gamma_mode & GAMMA_MODE_MODE_MASK) { + case GAMMA_MODE_MODE_8BIT: + crtc_state->hw.gamma_lut = ilk_read_lut_8(crtc); + break; + case GAMMA_MODE_MODE_12BIT_MULTI_SEGMENTED: + crtc_state->hw.gamma_lut = icl_read_lut_multi_segment(crtc); + break; + default: + crtc_state->hw.gamma_lut = glk_read_lut_10(crtc, PAL_PREC_INDEX_VALUE(0)); + } } void intel_color_init(struct intel_crtc *crtc) @@ -1945,6 +2073,7 @@ void intel_color_init(struct intel_crtc *crtc) if (INTEL_GEN(dev_priv) >= 11) { dev_priv->display.load_luts = icl_load_luts; + dev_priv->display.read_luts = icl_read_luts; } else if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) { dev_priv->display.load_luts = glk_load_luts; dev_priv->display.read_luts = glk_read_luts; diff --git a/drivers/gpu/drm/i915/display/intel_connector.c b/drivers/gpu/drm/i915/display/intel_connector.c index 903e49659f56..406e96785c76 100644 --- a/drivers/gpu/drm/i915/display/intel_connector.c +++ b/drivers/gpu/drm/i915/display/intel_connector.c @@ -33,6 +33,7 @@ #include "i915_drv.h" #include "intel_connector.h" +#include "intel_display_debugfs.h" #include "intel_display_types.h" #include "intel_hdcp.h" @@ -123,6 +124,8 @@ int intel_connector_register(struct drm_connector *connector) goto err_backlight; } + intel_connector_debugfs_add(connector); + return 0; err_backlight: @@ -290,7 +293,7 @@ intel_attach_colorspace_property(struct drm_connector *connector) return; break; default: - DRM_DEBUG_KMS("Colorspace property not supported\n"); + MISSING_CASE(connector->connector_type); return; } diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c index 45ecc7d9c829..2f5b9a4baafd 100644 --- a/drivers/gpu/drm/i915/display/intel_crt.c +++ b/drivers/gpu/drm/i915/display/intel_crt.c @@ -32,7 +32,6 @@ #include <drm/drm_crtc.h> #include <drm/drm_edid.h> #include <drm/drm_probe_helper.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_connector.h" @@ -204,27 +203,31 @@ static void intel_crt_set_dpms(struct intel_encoder *encoder, intel_de_write(dev_priv, crt->adpa_reg, adpa); } -static void intel_disable_crt(struct intel_encoder *encoder, +static void intel_disable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { intel_crt_set_dpms(encoder, old_crtc_state, DRM_MODE_DPMS_OFF); } -static void pch_disable_crt(struct intel_encoder *encoder, +static void pch_disable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { } -static void pch_post_disable_crt(struct intel_encoder *encoder, +static void pch_post_disable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_disable_crt(encoder, old_crtc_state, old_conn_state); + intel_disable_crt(state, encoder, old_crtc_state, old_conn_state); } -static void hsw_disable_crt(struct intel_encoder *encoder, +static void hsw_disable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -235,7 +238,8 @@ static void hsw_disable_crt(struct intel_encoder *encoder, intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); } -static void hsw_post_disable_crt(struct intel_encoder *encoder, +static void hsw_post_disable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -251,19 +255,20 @@ static void hsw_post_disable_crt(struct intel_encoder *encoder, intel_ddi_disable_pipe_clock(old_crtc_state); - pch_post_disable_crt(encoder, old_crtc_state, old_conn_state); + pch_post_disable_crt(state, encoder, old_crtc_state, old_conn_state); lpt_disable_pch_transcoder(dev_priv); lpt_disable_iclkip(dev_priv); - intel_ddi_fdi_post_disable(encoder, old_crtc_state, old_conn_state); + intel_ddi_fdi_post_disable(state, encoder, old_crtc_state, old_conn_state); drm_WARN_ON(&dev_priv->drm, !old_crtc_state->has_pch_encoder); intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } -static void hsw_pre_pll_enable_crt(struct intel_encoder *encoder, +static void hsw_pre_pll_enable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -274,7 +279,8 @@ static void hsw_pre_pll_enable_crt(struct intel_encoder *encoder, intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false); } -static void hsw_pre_enable_crt(struct intel_encoder *encoder, +static void hsw_pre_enable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -288,10 +294,11 @@ static void hsw_pre_enable_crt(struct intel_encoder *encoder, hsw_fdi_link_train(encoder, crtc_state); - intel_ddi_enable_pipe_clock(crtc_state); + intel_ddi_enable_pipe_clock(encoder, crtc_state); } -static void hsw_enable_crt(struct intel_encoder *encoder, +static void hsw_enable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -301,6 +308,8 @@ static void hsw_enable_crt(struct intel_encoder *encoder, drm_WARN_ON(&dev_priv->drm, !crtc_state->has_pch_encoder); + intel_ddi_enable_transcoder_func(encoder, crtc_state); + intel_enable_pipe(crtc_state); lpt_pch_enable(crtc_state); @@ -315,7 +324,8 @@ static void hsw_enable_crt(struct intel_encoder *encoder, intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true); } -static void intel_enable_crt(struct intel_encoder *encoder, +static void intel_enable_crt(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -595,7 +605,8 @@ static struct edid *intel_crt_get_edid(struct drm_connector *connector, edid = drm_get_edid(connector, i2c); if (!edid && !intel_gmbus_is_forced_bit(i2c)) { - DRM_DEBUG_KMS("CRT GMBUS EDID read failed, retry using GPIO bit-banging\n"); + drm_dbg_kms(connector->dev, + "CRT GMBUS EDID read failed, retry using GPIO bit-banging\n"); intel_gmbus_force_bit(i2c, true); edid = drm_get_edid(connector, i2c); intel_gmbus_force_bit(i2c, false); @@ -1046,6 +1057,7 @@ void intel_crt_init(struct drm_i915_private *dev_priv) !dmi_check_system(intel_spurious_crt_detect)) { crt->base.hpd_pin = HPD_CRT; crt->base.hotplug = intel_encoder_hotplug; + intel_connector->polled = DRM_CONNECTOR_POLL_HPD; } else { intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT; } diff --git a/drivers/gpu/drm/i915/display/intel_csr.c b/drivers/gpu/drm/i915/display/intel_csr.c index 57320c12839f..3112572cfb7d 100644 --- a/drivers/gpu/drm/i915/display/intel_csr.c +++ b/drivers/gpu/drm/i915/display/intel_csr.c @@ -40,8 +40,8 @@ #define GEN12_CSR_MAX_FW_SIZE ICL_CSR_MAX_FW_SIZE -#define TGL_CSR_PATH "i915/tgl_dmc_ver2_04.bin" -#define TGL_CSR_VERSION_REQUIRED CSR_VERSION(2, 4) +#define TGL_CSR_PATH "i915/tgl_dmc_ver2_06.bin" +#define TGL_CSR_VERSION_REQUIRED CSR_VERSION(2, 6) #define TGL_CSR_MAX_FW_SIZE 0x6000 MODULE_FIRMWARE(TGL_CSR_PATH); diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index ff292dfe2dd3..aa22465bb56e 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -568,7 +568,7 @@ static const struct cnl_ddi_buf_trans icl_combo_phy_ddi_translations_hdmi[] = { { 0x6, 0x7F, 0x35, 0x00, 0x0A }, /* 600 850 3.0 */ }; -static const struct cnl_ddi_buf_trans ehl_combo_phy_ddi_translations_hbr2_hbr3[] = { +static const struct cnl_ddi_buf_trans ehl_combo_phy_ddi_translations_dp[] = { /* NT mV Trans mV db */ { 0xA, 0x33, 0x3F, 0x00, 0x00 }, /* 350 350 0.0 */ { 0xA, 0x47, 0x36, 0x00, 0x09 }, /* 350 500 3.1 */ @@ -583,23 +583,51 @@ static const struct cnl_ddi_buf_trans ehl_combo_phy_ddi_translations_hbr2_hbr3[] }; struct icl_mg_phy_ddi_buf_trans { - u32 cri_txdeemph_override_5_0; u32 cri_txdeemph_override_11_6; + u32 cri_txdeemph_override_5_0; u32 cri_txdeemph_override_17_12; }; -static const struct icl_mg_phy_ddi_buf_trans icl_mg_phy_ddi_translations[] = { +static const struct icl_mg_phy_ddi_buf_trans icl_mg_phy_ddi_translations_rbr_hbr[] = { /* Voltage swing pre-emphasis */ - { 0x0, 0x1B, 0x00 }, /* 0 0 */ - { 0x0, 0x23, 0x08 }, /* 0 1 */ - { 0x0, 0x2D, 0x12 }, /* 0 2 */ - { 0x0, 0x00, 0x00 }, /* 0 3 */ - { 0x0, 0x23, 0x00 }, /* 1 0 */ - { 0x0, 0x2B, 0x09 }, /* 1 1 */ - { 0x0, 0x2E, 0x11 }, /* 1 2 */ - { 0x0, 0x2F, 0x00 }, /* 2 0 */ - { 0x0, 0x33, 0x0C }, /* 2 1 */ - { 0x0, 0x00, 0x00 }, /* 3 0 */ + { 0x18, 0x00, 0x00 }, /* 0 0 */ + { 0x1D, 0x00, 0x05 }, /* 0 1 */ + { 0x24, 0x00, 0x0C }, /* 0 2 */ + { 0x2B, 0x00, 0x14 }, /* 0 3 */ + { 0x21, 0x00, 0x00 }, /* 1 0 */ + { 0x2B, 0x00, 0x08 }, /* 1 1 */ + { 0x30, 0x00, 0x0F }, /* 1 2 */ + { 0x31, 0x00, 0x03 }, /* 2 0 */ + { 0x34, 0x00, 0x0B }, /* 2 1 */ + { 0x3F, 0x00, 0x00 }, /* 3 0 */ +}; + +static const struct icl_mg_phy_ddi_buf_trans icl_mg_phy_ddi_translations_hbr2_hbr3[] = { + /* Voltage swing pre-emphasis */ + { 0x18, 0x00, 0x00 }, /* 0 0 */ + { 0x1D, 0x00, 0x05 }, /* 0 1 */ + { 0x24, 0x00, 0x0C }, /* 0 2 */ + { 0x2B, 0x00, 0x14 }, /* 0 3 */ + { 0x26, 0x00, 0x00 }, /* 1 0 */ + { 0x2C, 0x00, 0x07 }, /* 1 1 */ + { 0x33, 0x00, 0x0C }, /* 1 2 */ + { 0x2E, 0x00, 0x00 }, /* 2 0 */ + { 0x36, 0x00, 0x09 }, /* 2 1 */ + { 0x3F, 0x00, 0x00 }, /* 3 0 */ +}; + +static const struct icl_mg_phy_ddi_buf_trans icl_mg_phy_ddi_translations_hdmi[] = { + /* HDMI Preset VS Pre-emph */ + { 0x1A, 0x0, 0x0 }, /* 1 400mV 0dB */ + { 0x20, 0x0, 0x0 }, /* 2 500mV 0dB */ + { 0x29, 0x0, 0x0 }, /* 3 650mV 0dB */ + { 0x32, 0x0, 0x0 }, /* 4 800mV 0dB */ + { 0x3F, 0x0, 0x0 }, /* 5 1000mV 0dB */ + { 0x3A, 0x0, 0x5 }, /* 6 Full -1.5 dB */ + { 0x39, 0x0, 0x6 }, /* 7 Full -1.8 dB */ + { 0x38, 0x0, 0x7 }, /* 8 Full -2 dB */ + { 0x37, 0x0, 0x8 }, /* 9 Full -2.5 dB */ + { 0x36, 0x0, 0x9 }, /* 10 Full -3 dB */ }; struct tgl_dkl_phy_ddi_buf_trans { @@ -943,13 +971,29 @@ icl_get_combo_buf_trans(struct drm_i915_private *dev_priv, int type, int rate, return icl_combo_phy_ddi_translations_dp_hbr2; } +static const struct icl_mg_phy_ddi_buf_trans * +icl_get_mg_buf_trans(struct drm_i915_private *dev_priv, int type, int rate, + int *n_entries) +{ + if (type == INTEL_OUTPUT_HDMI) { + *n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations_hdmi); + return icl_mg_phy_ddi_translations_hdmi; + } else if (rate > 270000) { + *n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations_hbr2_hbr3); + return icl_mg_phy_ddi_translations_hbr2_hbr3; + } + + *n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations_rbr_hbr); + return icl_mg_phy_ddi_translations_rbr_hbr; +} + static const struct cnl_ddi_buf_trans * ehl_get_combo_buf_trans(struct drm_i915_private *dev_priv, int type, int rate, int *n_entries) { - if (type == INTEL_OUTPUT_DP && rate > 270000) { - *n_entries = ARRAY_SIZE(ehl_combo_phy_ddi_translations_hbr2_hbr3); - return ehl_combo_phy_ddi_translations_hbr2_hbr3; + if (type != INTEL_OUTPUT_HDMI && type != INTEL_OUTPUT_EDP) { + *n_entries = ARRAY_SIZE(ehl_combo_phy_ddi_translations_dp); + return ehl_combo_phy_ddi_translations_dp; } return icl_get_combo_buf_trans(dev_priv, type, rate, n_entries); @@ -959,7 +1003,7 @@ static const struct cnl_ddi_buf_trans * tgl_get_combo_buf_trans(struct drm_i915_private *dev_priv, int type, int rate, int *n_entries) { - if (type != INTEL_OUTPUT_DP) { + if (type == INTEL_OUTPUT_HDMI || type == INTEL_OUTPUT_EDP) { return icl_get_combo_buf_trans(dev_priv, type, rate, n_entries); } else if (rate > 270000) { *n_entries = ARRAY_SIZE(tgl_combo_phy_ddi_translations_dp_hbr2); @@ -988,7 +1032,8 @@ static int intel_ddi_hdmi_level(struct intel_encoder *encoder) icl_get_combo_buf_trans(dev_priv, INTEL_OUTPUT_HDMI, 0, &n_entries); else - n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); + icl_get_mg_buf_trans(dev_priv, INTEL_OUTPUT_HDMI, 0, + &n_entries); default_entry = n_entries - 1; } else if (IS_CANNONLAKE(dev_priv)) { cnl_get_buf_trans_hdmi(dev_priv, &n_entries); @@ -1006,18 +1051,18 @@ static int intel_ddi_hdmi_level(struct intel_encoder *encoder) intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); default_entry = 6; } else { - WARN(1, "ddi translation table missing\n"); + drm_WARN(&dev_priv->drm, 1, "ddi translation table missing\n"); return 0; } - if (WARN_ON_ONCE(n_entries == 0)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, n_entries == 0)) return 0; level = intel_bios_hdmi_level_shift(encoder); if (level < 0) level = default_entry; - if (WARN_ON_ONCE(level >= n_entries)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) level = n_entries - 1; return level; @@ -1075,9 +1120,9 @@ static void intel_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder, ddi_translations = intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries); - if (WARN_ON_ONCE(!ddi_translations)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) return; - if (WARN_ON_ONCE(level >= n_entries)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) level = n_entries - 1; /* If we're boosting the current, set bit 31 of trans1 */ @@ -1102,7 +1147,8 @@ static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv, if (intel_de_read(dev_priv, reg) & DDI_BUF_IS_IDLE) return; } - DRM_ERROR("Timeout waiting for DDI BUF %c idle bit\n", port_name(port)); + drm_err(&dev_priv->drm, "Timeout waiting for DDI BUF %c idle bit\n", + port_name(port)); } static u32 hsw_pll_to_ddi_pll_sel(const struct intel_shared_dpll *pll) @@ -1208,14 +1254,17 @@ void hsw_fdi_link_train(struct intel_encoder *encoder, /* Configure Port Clock Select */ ddi_pll_sel = hsw_pll_to_ddi_pll_sel(crtc_state->shared_dpll); intel_de_write(dev_priv, PORT_CLK_SEL(PORT_E), ddi_pll_sel); - WARN_ON(ddi_pll_sel != PORT_CLK_SEL_SPLL); + drm_WARN_ON(&dev_priv->drm, ddi_pll_sel != PORT_CLK_SEL_SPLL); /* Start the training iterating through available voltages and emphasis, * testing each value twice. */ for (i = 0; i < ARRAY_SIZE(hsw_ddi_translations_fdi) * 2; i++) { /* Configure DP_TP_CTL with auto-training */ intel_de_write(dev_priv, DP_TP_CTL(PORT_E), - DP_TP_CTL_FDI_AUTOTRAIN | DP_TP_CTL_ENHANCED_FRAME_ENABLE | DP_TP_CTL_LINK_TRAIN_PAT1 | DP_TP_CTL_ENABLE); + DP_TP_CTL_FDI_AUTOTRAIN | + DP_TP_CTL_ENHANCED_FRAME_ENABLE | + DP_TP_CTL_LINK_TRAIN_PAT1 | + DP_TP_CTL_ENABLE); /* Configure and enable DDI_BUF_CTL for DDI E with next voltage. * DDI E does not support port reversal, the functionality is @@ -1249,7 +1298,8 @@ void hsw_fdi_link_train(struct intel_encoder *encoder, temp = intel_de_read(dev_priv, DP_TP_STATUS(PORT_E)); if (temp & DP_TP_STATUS_AUTOTRAIN_DONE) { - DRM_DEBUG_KMS("FDI link training done on step %d\n", i); + drm_dbg_kms(&dev_priv->drm, + "FDI link training done on step %d\n", i); break; } @@ -1258,7 +1308,7 @@ void hsw_fdi_link_train(struct intel_encoder *encoder, * Results in less fireworks from the state checker. */ if (i == ARRAY_SIZE(hsw_ddi_translations_fdi) * 2 - 1) { - DRM_ERROR("FDI link training failed!\n"); + drm_err(&dev_priv->drm, "FDI link training failed!\n"); break; } @@ -1290,7 +1340,10 @@ void hsw_fdi_link_train(struct intel_encoder *encoder, /* Enable normal pixel sending for FDI */ intel_de_write(dev_priv, DP_TP_CTL(PORT_E), - DP_TP_CTL_FDI_AUTOTRAIN | DP_TP_CTL_LINK_TRAIN_NORMAL | DP_TP_CTL_ENHANCED_FRAME_ENABLE | DP_TP_CTL_ENABLE); + DP_TP_CTL_FDI_AUTOTRAIN | + DP_TP_CTL_LINK_TRAIN_NORMAL | + DP_TP_CTL_ENHANCED_FRAME_ENABLE | + DP_TP_CTL_ENABLE); } static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder) @@ -1304,184 +1357,6 @@ static void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder) intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count); } -static struct intel_encoder * -intel_ddi_get_crtc_encoder(struct intel_crtc *crtc) -{ - struct drm_device *dev = crtc->base.dev; - struct intel_encoder *encoder, *ret = NULL; - int num_encoders = 0; - - for_each_encoder_on_crtc(dev, &crtc->base, encoder) { - ret = encoder; - num_encoders++; - } - - if (num_encoders != 1) - WARN(1, "%d encoders on crtc for pipe %c\n", num_encoders, - pipe_name(crtc->pipe)); - - BUG_ON(ret == NULL); - return ret; -} - -static int hsw_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv, - i915_reg_t reg) -{ - int refclk; - int n, p, r; - u32 wrpll; - - wrpll = intel_de_read(dev_priv, reg); - switch (wrpll & WRPLL_REF_MASK) { - case WRPLL_REF_SPECIAL_HSW: - /* - * muxed-SSC for BDW. - * non-SSC for non-ULT HSW. Check FUSE_STRAP3 - * for the non-SSC reference frequency. - */ - if (IS_HASWELL(dev_priv) && !IS_HSW_ULT(dev_priv)) { - if (intel_de_read(dev_priv, FUSE_STRAP3) & HSW_REF_CLK_SELECT) - refclk = 24; - else - refclk = 135; - break; - } - /* fall through */ - case WRPLL_REF_PCH_SSC: - /* - * We could calculate spread here, but our checking - * code only cares about 5% accuracy, and spread is a max of - * 0.5% downspread. - */ - refclk = 135; - break; - case WRPLL_REF_LCPLL: - refclk = 2700; - break; - default: - MISSING_CASE(wrpll); - return 0; - } - - r = wrpll & WRPLL_DIVIDER_REF_MASK; - p = (wrpll & WRPLL_DIVIDER_POST_MASK) >> WRPLL_DIVIDER_POST_SHIFT; - n = (wrpll & WRPLL_DIVIDER_FB_MASK) >> WRPLL_DIVIDER_FB_SHIFT; - - /* Convert to KHz, p & r have a fixed point portion */ - return (refclk * n * 100) / (p * r); -} - -static int skl_calc_wrpll_link(const struct intel_dpll_hw_state *pll_state) -{ - u32 p0, p1, p2, dco_freq; - - p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK; - p2 = pll_state->cfgcr2 & DPLL_CFGCR2_KDIV_MASK; - - if (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_MODE(1)) - p1 = (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8; - else - p1 = 1; - - - switch (p0) { - case DPLL_CFGCR2_PDIV_1: - p0 = 1; - break; - case DPLL_CFGCR2_PDIV_2: - p0 = 2; - break; - case DPLL_CFGCR2_PDIV_3: - p0 = 3; - break; - case DPLL_CFGCR2_PDIV_7: - p0 = 7; - break; - } - - switch (p2) { - case DPLL_CFGCR2_KDIV_5: - p2 = 5; - break; - case DPLL_CFGCR2_KDIV_2: - p2 = 2; - break; - case DPLL_CFGCR2_KDIV_3: - p2 = 3; - break; - case DPLL_CFGCR2_KDIV_1: - p2 = 1; - break; - } - - dco_freq = (pll_state->cfgcr1 & DPLL_CFGCR1_DCO_INTEGER_MASK) - * 24 * 1000; - - dco_freq += (((pll_state->cfgcr1 & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) - * 24 * 1000) / 0x8000; - - if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0)) - return 0; - - return dco_freq / (p0 * p1 * p2 * 5); -} - -int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, - struct intel_dpll_hw_state *pll_state) -{ - u32 p0, p1, p2, dco_freq, ref_clock; - - p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK; - p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK; - - if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1)) - p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >> - DPLL_CFGCR1_QDIV_RATIO_SHIFT; - else - p1 = 1; - - - switch (p0) { - case DPLL_CFGCR1_PDIV_2: - p0 = 2; - break; - case DPLL_CFGCR1_PDIV_3: - p0 = 3; - break; - case DPLL_CFGCR1_PDIV_5: - p0 = 5; - break; - case DPLL_CFGCR1_PDIV_7: - p0 = 7; - break; - } - - switch (p2) { - case DPLL_CFGCR1_KDIV_1: - p2 = 1; - break; - case DPLL_CFGCR1_KDIV_2: - p2 = 2; - break; - case DPLL_CFGCR1_KDIV_3: - p2 = 3; - break; - } - - ref_clock = cnl_hdmi_pll_ref_clock(dev_priv); - - dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) - * ref_clock; - - dco_freq += (((pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >> - DPLL_CFGCR0_DCO_FRACTION_SHIFT) * ref_clock) / 0x8000; - - if (WARN_ON(p0 == 0 || p1 == 0 || p2 == 0)) - return 0; - - return dco_freq / (p0 * p1 * p2 * 5); -} - static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv, enum port port) { @@ -1504,77 +1379,6 @@ static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv, } } -static int icl_calc_mg_pll_link(struct drm_i915_private *dev_priv, - const struct intel_dpll_hw_state *pll_state) -{ - u32 m1, m2_int, m2_frac, div1, div2, ref_clock; - u64 tmp; - - ref_clock = dev_priv->cdclk.hw.ref; - - if (INTEL_GEN(dev_priv) >= 12) { - m1 = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBPREDIV_MASK; - m1 = m1 >> DKL_PLL_DIV0_FBPREDIV_SHIFT; - m2_int = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBDIV_INT_MASK; - - if (pll_state->mg_pll_bias & DKL_PLL_BIAS_FRAC_EN_H) { - m2_frac = pll_state->mg_pll_bias & - DKL_PLL_BIAS_FBDIV_FRAC_MASK; - m2_frac = m2_frac >> DKL_PLL_BIAS_FBDIV_SHIFT; - } else { - m2_frac = 0; - } - } else { - m1 = pll_state->mg_pll_div1 & MG_PLL_DIV1_FBPREDIV_MASK; - m2_int = pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK; - - if (pll_state->mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) { - m2_frac = pll_state->mg_pll_div0 & - MG_PLL_DIV0_FBDIV_FRAC_MASK; - m2_frac = m2_frac >> MG_PLL_DIV0_FBDIV_FRAC_SHIFT; - } else { - m2_frac = 0; - } - } - - switch (pll_state->mg_clktop2_hsclkctl & - MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK) { - case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2: - div1 = 2; - break; - case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3: - div1 = 3; - break; - case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5: - div1 = 5; - break; - case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7: - div1 = 7; - break; - default: - MISSING_CASE(pll_state->mg_clktop2_hsclkctl); - return 0; - } - - div2 = (pll_state->mg_clktop2_hsclkctl & - MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK) >> - MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT; - - /* div2 value of 0 is same as 1 means no div */ - if (div2 == 0) - div2 = 1; - - /* - * Adjust the original formula to delay the division by 2^22 in order to - * minimize possible rounding errors. - */ - tmp = (u64)m1 * m2_int * ref_clock + - (((u64)m1 * m2_frac * ref_clock) >> 22); - tmp = div_u64(tmp, 5 * div1 * div2); - - return tmp; -} - static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) { int dotclock; @@ -1600,215 +1404,22 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config) pipe_config->hw.adjusted_mode.crtc_clock = dotclock; } -static void icl_ddi_clock_get(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state; - enum port port = encoder->port; - enum phy phy = intel_port_to_phy(dev_priv, port); - int link_clock; - - if (intel_phy_is_combo(dev_priv, phy)) { - link_clock = cnl_calc_wrpll_link(dev_priv, pll_state); - } else { - enum intel_dpll_id pll_id = intel_get_shared_dpll_id(dev_priv, - pipe_config->shared_dpll); - - if (pll_id == DPLL_ID_ICL_TBTPLL) - link_clock = icl_calc_tbt_pll_link(dev_priv, port); - else - link_clock = icl_calc_mg_pll_link(dev_priv, pll_state); - } - - pipe_config->port_clock = link_clock; - - ddi_dotclock_get(pipe_config); -} - -static void cnl_ddi_clock_get(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state; - int link_clock; - - if (pll_state->cfgcr0 & DPLL_CFGCR0_HDMI_MODE) { - link_clock = cnl_calc_wrpll_link(dev_priv, pll_state); - } else { - link_clock = pll_state->cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK; - - switch (link_clock) { - case DPLL_CFGCR0_LINK_RATE_810: - link_clock = 81000; - break; - case DPLL_CFGCR0_LINK_RATE_1080: - link_clock = 108000; - break; - case DPLL_CFGCR0_LINK_RATE_1350: - link_clock = 135000; - break; - case DPLL_CFGCR0_LINK_RATE_1620: - link_clock = 162000; - break; - case DPLL_CFGCR0_LINK_RATE_2160: - link_clock = 216000; - break; - case DPLL_CFGCR0_LINK_RATE_2700: - link_clock = 270000; - break; - case DPLL_CFGCR0_LINK_RATE_3240: - link_clock = 324000; - break; - case DPLL_CFGCR0_LINK_RATE_4050: - link_clock = 405000; - break; - default: - WARN(1, "Unsupported link rate\n"); - break; - } - link_clock *= 2; - } - - pipe_config->port_clock = link_clock; - - ddi_dotclock_get(pipe_config); -} - -static void skl_ddi_clock_get(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct intel_dpll_hw_state *pll_state = &pipe_config->dpll_hw_state; - int link_clock; - - /* - * ctrl1 register is already shifted for each pll, just use 0 to get - * the internal shift for each field - */ - if (pll_state->ctrl1 & DPLL_CTRL1_HDMI_MODE(0)) { - link_clock = skl_calc_wrpll_link(pll_state); - } else { - link_clock = pll_state->ctrl1 & DPLL_CTRL1_LINK_RATE_MASK(0); - link_clock >>= DPLL_CTRL1_LINK_RATE_SHIFT(0); - - switch (link_clock) { - case DPLL_CTRL1_LINK_RATE_810: - link_clock = 81000; - break; - case DPLL_CTRL1_LINK_RATE_1080: - link_clock = 108000; - break; - case DPLL_CTRL1_LINK_RATE_1350: - link_clock = 135000; - break; - case DPLL_CTRL1_LINK_RATE_1620: - link_clock = 162000; - break; - case DPLL_CTRL1_LINK_RATE_2160: - link_clock = 216000; - break; - case DPLL_CTRL1_LINK_RATE_2700: - link_clock = 270000; - break; - default: - drm_WARN(encoder->base.dev, 1, - "Unsupported link rate\n"); - break; - } - link_clock *= 2; - } - - pipe_config->port_clock = link_clock; - - ddi_dotclock_get(pipe_config); -} - -static void hsw_ddi_clock_get(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - int link_clock = 0; - u32 val, pll; - - val = hsw_pll_to_ddi_pll_sel(pipe_config->shared_dpll); - switch (val & PORT_CLK_SEL_MASK) { - case PORT_CLK_SEL_LCPLL_810: - link_clock = 81000; - break; - case PORT_CLK_SEL_LCPLL_1350: - link_clock = 135000; - break; - case PORT_CLK_SEL_LCPLL_2700: - link_clock = 270000; - break; - case PORT_CLK_SEL_WRPLL1: - link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(0)); - break; - case PORT_CLK_SEL_WRPLL2: - link_clock = hsw_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL(1)); - break; - case PORT_CLK_SEL_SPLL: - pll = intel_de_read(dev_priv, SPLL_CTL) & SPLL_FREQ_MASK; - if (pll == SPLL_FREQ_810MHz) - link_clock = 81000; - else if (pll == SPLL_FREQ_1350MHz) - link_clock = 135000; - else if (pll == SPLL_FREQ_2700MHz) - link_clock = 270000; - else { - WARN(1, "bad spll freq\n"); - return; - } - break; - default: - WARN(1, "bad port clock sel\n"); - return; - } - - pipe_config->port_clock = link_clock * 2; - - ddi_dotclock_get(pipe_config); -} - -static int bxt_calc_pll_link(const struct intel_dpll_hw_state *pll_state) -{ - struct dpll clock; - - clock.m1 = 2; - clock.m2 = (pll_state->pll0 & PORT_PLL_M2_MASK) << 22; - if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE) - clock.m2 |= pll_state->pll2 & PORT_PLL_M2_FRAC_MASK; - clock.n = (pll_state->pll1 & PORT_PLL_N_MASK) >> PORT_PLL_N_SHIFT; - clock.p1 = (pll_state->ebb0 & PORT_PLL_P1_MASK) >> PORT_PLL_P1_SHIFT; - clock.p2 = (pll_state->ebb0 & PORT_PLL_P2_MASK) >> PORT_PLL_P2_SHIFT; - - return chv_calc_dpll_params(100000, &clock); -} - -static void bxt_ddi_clock_get(struct intel_encoder *encoder, - struct intel_crtc_state *pipe_config) -{ - pipe_config->port_clock = - bxt_calc_pll_link(&pipe_config->dpll_hw_state); - - ddi_dotclock_get(pipe_config); -} - static void intel_ddi_clock_get(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - if (INTEL_GEN(dev_priv) >= 11) - icl_ddi_clock_get(encoder, pipe_config); - else if (IS_CANNONLAKE(dev_priv)) - cnl_ddi_clock_get(encoder, pipe_config); - else if (IS_GEN9_LP(dev_priv)) - bxt_ddi_clock_get(encoder, pipe_config); - else if (IS_GEN9_BC(dev_priv)) - skl_ddi_clock_get(encoder, pipe_config); - else if (INTEL_GEN(dev_priv) <= 8) - hsw_ddi_clock_get(encoder, pipe_config); + if (intel_phy_is_tc(dev_priv, phy) && + intel_get_shared_dpll_id(dev_priv, pipe_config->shared_dpll) == + DPLL_ID_ICL_TBTPLL) + pipe_config->port_clock = icl_calc_tbt_pll_link(dev_priv, + encoder->port); + else + pipe_config->port_clock = + intel_dpll_get_freq(dev_priv, pipe_config->shared_dpll); + + ddi_dotclock_get(pipe_config); } void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state, @@ -1822,7 +1433,7 @@ void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state, if (!intel_crtc_has_dp_encoder(crtc_state)) return; - WARN_ON(transcoder_is_dsi(cpu_transcoder)); + drm_WARN_ON(&dev_priv->drm, transcoder_is_dsi(cpu_transcoder)); temp = DP_MSA_MISC_SYNC_CLOCK; @@ -1845,8 +1456,8 @@ void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state, } /* nonsense combination */ - WARN_ON(crtc_state->limited_color_range && - crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB); + drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range && + crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB); if (crtc_state->limited_color_range) temp |= DP_MSA_MISC_COLOR_CEA_RGB; @@ -1871,6 +1482,14 @@ void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state, intel_de_write(dev_priv, TRANS_MSA_MISC(cpu_transcoder), temp); } +static u32 bdw_trans_port_sync_master_select(enum transcoder master_transcoder) +{ + if (master_transcoder == TRANSCODER_EDP) + return 0; + else + return master_transcoder + 1; +} + /* * Returns the TRANS_DDI_FUNC_CTL value based on CRTC state. * @@ -1878,10 +1497,10 @@ void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state, * intel_ddi_config_transcoder_func(). */ static u32 -intel_ddi_transcoder_func_reg_val_get(const struct intel_crtc_state *crtc_state) +intel_ddi_transcoder_func_reg_val_get(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; @@ -1962,7 +1581,8 @@ intel_ddi_transcoder_func_reg_val_get(const struct intel_crtc_state *crtc_state) enum transcoder master; master = crtc_state->mst_master_transcoder; - WARN_ON(master == INVALID_TRANSCODER); + drm_WARN_ON(&dev_priv->drm, + master == INVALID_TRANSCODER); temp |= TRANS_DDI_MST_TRANSPORT_SELECT(master); } } else { @@ -1970,20 +1590,46 @@ intel_ddi_transcoder_func_reg_val_get(const struct intel_crtc_state *crtc_state) temp |= DDI_PORT_WIDTH(crtc_state->lane_count); } + if (IS_GEN_RANGE(dev_priv, 8, 10) && + crtc_state->master_transcoder != INVALID_TRANSCODER) { + u8 master_select = + bdw_trans_port_sync_master_select(crtc_state->master_transcoder); + + temp |= TRANS_DDI_PORT_SYNC_ENABLE | + TRANS_DDI_PORT_SYNC_MASTER_SELECT(master_select); + } + return temp; } -void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state) +void intel_ddi_enable_transcoder_func(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - u32 temp; + u32 ctl; + + if (INTEL_GEN(dev_priv) >= 11) { + enum transcoder master_transcoder = crtc_state->master_transcoder; + u32 ctl2 = 0; + + if (master_transcoder != INVALID_TRANSCODER) { + u8 master_select = + bdw_trans_port_sync_master_select(master_transcoder); - temp = intel_ddi_transcoder_func_reg_val_get(crtc_state); + ctl2 |= PORT_SYNC_MODE_ENABLE | + PORT_SYNC_MODE_MASTER_SELECT(master_select); + } + + intel_de_write(dev_priv, + TRANS_DDI_FUNC_CTL2(cpu_transcoder), ctl2); + } + + ctl = intel_ddi_transcoder_func_reg_val_get(encoder, crtc_state); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) - temp |= TRANS_DDI_DP_VC_PAYLOAD_ALLOC; - intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), temp); + ctl |= TRANS_DDI_DP_VC_PAYLOAD_ALLOC; + intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), ctl); } /* @@ -1991,16 +1637,17 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state) * bit. */ static void -intel_ddi_config_transcoder_func(const struct intel_crtc_state *crtc_state) +intel_ddi_config_transcoder_func(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - u32 temp; + u32 ctl; - temp = intel_ddi_transcoder_func_reg_val_get(crtc_state); - temp &= ~TRANS_DDI_FUNC_ENABLE; - intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), temp); + ctl = intel_ddi_transcoder_func_reg_val_get(encoder, crtc_state); + ctl &= ~TRANS_DDI_FUNC_ENABLE; + intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), ctl); } void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state) @@ -2008,24 +1655,35 @@ void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; - u32 val; + u32 ctl; - val = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); - val &= ~TRANS_DDI_FUNC_ENABLE; + if (INTEL_GEN(dev_priv) >= 11) + intel_de_write(dev_priv, + TRANS_DDI_FUNC_CTL2(cpu_transcoder), 0); + + ctl = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); + + ctl &= ~TRANS_DDI_FUNC_ENABLE; + + if (IS_GEN_RANGE(dev_priv, 8, 10)) + ctl &= ~(TRANS_DDI_PORT_SYNC_ENABLE | + TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK); if (INTEL_GEN(dev_priv) >= 12) { if (!intel_dp_mst_is_master_trans(crtc_state)) { - val &= ~(TGL_TRANS_DDI_PORT_MASK | + ctl &= ~(TGL_TRANS_DDI_PORT_MASK | TRANS_DDI_MODE_SELECT_MASK); } } else { - val &= ~(TRANS_DDI_PORT_MASK | TRANS_DDI_MODE_SELECT_MASK); + ctl &= ~(TRANS_DDI_PORT_MASK | TRANS_DDI_MODE_SELECT_MASK); } - intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), val); + + intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), ctl); if (dev_priv->quirks & QUIRK_INCREASE_DDI_DISABLED_TIME && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { - DRM_DEBUG_KMS("Quirk Increase DDI disabled time\n"); + drm_dbg_kms(&dev_priv->drm, + "Quirk Increase DDI disabled time\n"); /* Quirk time at 100ms for reliable operation */ msleep(100); } @@ -2043,10 +1701,11 @@ int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, wakeref = intel_display_power_get_if_enabled(dev_priv, intel_encoder->power_domain); - if (WARN_ON(!wakeref)) + if (drm_WARN_ON(dev, !wakeref)) return -ENXIO; - if (WARN_ON(!intel_encoder->get_hw_state(intel_encoder, &pipe))) { + if (drm_WARN_ON(dev, + !intel_encoder->get_hw_state(intel_encoder, &pipe))) { ret = -EIO; goto out; } @@ -2085,7 +1744,7 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector) goto out; } - if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) + if (HAS_TRANSCODER(dev_priv, TRANSCODER_EDP) && port == PORT_A) cpu_transcoder = TRANSCODER_EDP; else cpu_transcoder = (enum transcoder) pipe; @@ -2147,7 +1806,7 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder, if (!(tmp & DDI_BUF_CTL_ENABLE)) goto out; - if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) { + if (HAS_TRANSCODER(dev_priv, TRANSCODER_EDP) && port == PORT_A) { tmp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(TRANSCODER_EDP)); @@ -2205,20 +1864,23 @@ static void intel_ddi_get_encoder_pipes(struct intel_encoder *encoder, } if (!*pipe_mask) - DRM_DEBUG_KMS("No pipe for [ENCODER:%d:%s] found\n", - encoder->base.base.id, encoder->base.name); + drm_dbg_kms(&dev_priv->drm, + "No pipe for [ENCODER:%d:%s] found\n", + encoder->base.base.id, encoder->base.name); if (!mst_pipe_mask && hweight8(*pipe_mask) > 1) { - DRM_DEBUG_KMS("Multiple pipes for [ENCODER:%d:%s] (pipe_mask %02x)\n", - encoder->base.base.id, encoder->base.name, - *pipe_mask); + drm_dbg_kms(&dev_priv->drm, + "Multiple pipes for [ENCODER:%d:%s] (pipe_mask %02x)\n", + encoder->base.base.id, encoder->base.name, + *pipe_mask); *pipe_mask = BIT(ffs(*pipe_mask) - 1); } if (mst_pipe_mask && mst_pipe_mask != *pipe_mask) - DRM_DEBUG_KMS("Conflicting MST and non-MST state for [ENCODER:%d:%s] (pipe_mask %02x mst_pipe_mask %02x)\n", - encoder->base.base.id, encoder->base.name, - *pipe_mask, mst_pipe_mask); + drm_dbg_kms(&dev_priv->drm, + "Conflicting MST and non-MST state for [ENCODER:%d:%s] (pipe_mask %02x mst_pipe_mask %02x)\n", + encoder->base.base.id, encoder->base.name, + *pipe_mask, mst_pipe_mask); else *is_dp_mst = mst_pipe_mask; @@ -2228,9 +1890,9 @@ out: if ((tmp & (BXT_PHY_CMNLANE_POWERDOWN_ACK | BXT_PHY_LANE_POWERDOWN_ACK | BXT_PHY_LANE_ENABLED)) != BXT_PHY_LANE_ENABLED) - DRM_ERROR("[ENCODER:%d:%s] enabled but PHY powered down? " - "(PHY_CTL %08x)\n", encoder->base.base.id, - encoder->base.name, tmp); + drm_err(&dev_priv->drm, + "[ENCODER:%d:%s] enabled but PHY powered down? (PHY_CTL %08x)\n", + encoder->base.base.id, encoder->base.name, tmp); } intel_display_power_put(dev_priv, encoder->power_domain, wakeref); @@ -2252,7 +1914,7 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder, return true; } -static inline enum intel_display_power_domain +static enum intel_display_power_domain intel_ddi_main_link_aux_domain(struct intel_digital_port *dig_port) { /* CNL+ HW requires corresponding AUX IOs to be powered up for PSR with @@ -2283,11 +1945,16 @@ static void intel_ddi_get_power_domains(struct intel_encoder *encoder, * happen since fake-MST encoders don't set their get_power_domains() * hook. */ - if (WARN_ON(intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))) + if (drm_WARN_ON(&dev_priv->drm, + intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))) return; dig_port = enc_to_dig_port(encoder); - intel_display_power_get(dev_priv, dig_port->ddi_io_power_domain); + + if (!intel_phy_is_tc(dev_priv, phy) || + dig_port->tc_mode != TC_PORT_TBT_ALT) + intel_display_power_get(dev_priv, + dig_port->ddi_io_power_domain); /* * AUX power is only needed for (e)DP mode, and for HDMI mode on TC @@ -2306,11 +1973,11 @@ static void intel_ddi_get_power_domains(struct intel_encoder *encoder, intel_dsc_power_domain(crtc_state)); } -void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state) +void intel_ddi_enable_pipe_clock(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc); enum port port = encoder->port; enum transcoder cpu_transcoder = crtc_state->cpu_transcoder; @@ -2381,9 +2048,9 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, else ddi_translations = intel_ddi_get_buf_trans_dp(dev_priv, port, &n_entries); - if (WARN_ON_ONCE(!ddi_translations)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) return; - if (WARN_ON_ONCE(level >= n_entries)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) level = n_entries - 1; iboost = ddi_translations[level].i_boost; @@ -2391,7 +2058,7 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder, /* Make sure that the requested I_boost is valid */ if (iboost && iboost != 0x1 && iboost != 0x3 && iboost != 0x7) { - DRM_ERROR("Invalid I_boost value %u\n", iboost); + drm_err(&dev_priv->drm, "Invalid I_boost value %u\n", iboost); return; } @@ -2416,9 +2083,9 @@ static void bxt_ddi_vswing_sequence(struct intel_encoder *encoder, else ddi_translations = bxt_get_buf_trans_dp(dev_priv, &n_entries); - if (WARN_ON_ONCE(!ddi_translations)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) return; - if (WARN_ON_ONCE(level >= n_entries)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) level = n_entries - 1; bxt_ddi_phy_set_signal_level(dev_priv, port, @@ -2450,7 +2117,8 @@ u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder) icl_get_combo_buf_trans(dev_priv, encoder->type, intel_dp->link_rate, &n_entries); else - n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); + icl_get_mg_buf_trans(dev_priv, encoder->type, + intel_dp->link_rate, &n_entries); } else if (IS_CANNONLAKE(dev_priv)) { if (encoder->type == INTEL_OUTPUT_EDP) cnl_get_buf_trans_edp(dev_priv, &n_entries); @@ -2468,9 +2136,10 @@ u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder) intel_ddi_get_buf_trans_dp(dev_priv, port, &n_entries); } - if (WARN_ON(n_entries < 1)) + if (drm_WARN_ON(&dev_priv->drm, n_entries < 1)) n_entries = 1; - if (WARN_ON(n_entries > ARRAY_SIZE(index_to_dp_signal_levels))) + if (drm_WARN_ON(&dev_priv->drm, + n_entries > ARRAY_SIZE(index_to_dp_signal_levels))) n_entries = ARRAY_SIZE(index_to_dp_signal_levels); return index_to_dp_signal_levels[n_entries - 1] & @@ -2513,9 +2182,9 @@ static void cnl_ddi_vswing_program(struct intel_encoder *encoder, else ddi_translations = cnl_get_buf_trans_dp(dev_priv, &n_entries); - if (WARN_ON_ONCE(!ddi_translations)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, !ddi_translations)) return; - if (WARN_ON_ONCE(level >= n_entries)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, level >= n_entries)) level = n_entries - 1; /* Set PORT_TX_DW5 Scaling Mode Sel to 010b. */ @@ -2649,7 +2318,9 @@ static void icl_ddi_combo_vswing_program(struct drm_i915_private *dev_priv, return; if (level >= n_entries) { - DRM_DEBUG_KMS("DDI translation not found for level %d. Using %d instead.", level, n_entries - 1); + drm_dbg_kms(&dev_priv->drm, + "DDI translation not found for level %d. Using %d instead.", + level, n_entries - 1); level = n_entries - 1; } @@ -2762,21 +2433,28 @@ static void icl_combo_phy_ddi_vswing_sequence(struct intel_encoder *encoder, } static void icl_mg_phy_ddi_vswing_sequence(struct intel_encoder *encoder, - int link_clock, - u32 level) + int link_clock, u32 level, + enum intel_output_type type) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); enum tc_port tc_port = intel_port_to_tc(dev_priv, encoder->port); const struct icl_mg_phy_ddi_buf_trans *ddi_translations; u32 n_entries, val; - int ln; + int ln, rate = 0; + + if (type != INTEL_OUTPUT_HDMI) { + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - n_entries = ARRAY_SIZE(icl_mg_phy_ddi_translations); - ddi_translations = icl_mg_phy_ddi_translations; + rate = intel_dp->link_rate; + } + + ddi_translations = icl_get_mg_buf_trans(dev_priv, type, rate, + &n_entries); /* The table does not have values for level 3 and level 9. */ if (level >= n_entries || level == 3 || level == 9) { - DRM_DEBUG_KMS("DDI translation not found for level %d. Using %d instead.", - level, n_entries - 2); + drm_dbg_kms(&dev_priv->drm, + "DDI translation not found for level %d. Using %d instead.", + level, n_entries - 2); level = n_entries - 2; } @@ -2895,7 +2573,8 @@ static void icl_ddi_vswing_sequence(struct intel_encoder *encoder, if (intel_phy_is_combo(dev_priv, phy)) icl_combo_phy_ddi_vswing_sequence(encoder, level, type); else - icl_mg_phy_ddi_vswing_sequence(encoder, link_clock, level); + icl_mg_phy_ddi_vswing_sequence(encoder, link_clock, level, + type); } static void @@ -2962,8 +2641,9 @@ static void tgl_ddi_vswing_sequence(struct intel_encoder *encoder, tgl_dkl_phy_ddi_vswing_sequence(encoder, link_clock, level); } -static u32 translate_signal_level(int signal_levels) +static u32 translate_signal_level(struct intel_dp *intel_dp, int signal_levels) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); int i; for (i = 0; i < ARRAY_SIZE(index_to_dp_signal_levels); i++) { @@ -2971,8 +2651,9 @@ static u32 translate_signal_level(int signal_levels) return i; } - WARN(1, "Unsupported voltage swing/pre-emphasis level: 0x%x\n", - signal_levels); + drm_WARN(&i915->drm, 1, + "Unsupported voltage swing/pre-emphasis level: 0x%x\n", + signal_levels); return 0; } @@ -2983,46 +2664,73 @@ static u32 intel_ddi_dp_level(struct intel_dp *intel_dp) int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | DP_TRAIN_PRE_EMPHASIS_MASK); - return translate_signal_level(signal_levels); + return translate_signal_level(intel_dp, signal_levels); } -u32 bxt_signal_levels(struct intel_dp *intel_dp) +static void +tgl_set_signal_levels(struct intel_dp *intel_dp) { - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); - struct intel_encoder *encoder = &dport->base; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; int level = intel_ddi_dp_level(intel_dp); - if (INTEL_GEN(dev_priv) >= 12) - tgl_ddi_vswing_sequence(encoder, intel_dp->link_rate, - level, encoder->type); - else if (INTEL_GEN(dev_priv) >= 11) - icl_ddi_vswing_sequence(encoder, intel_dp->link_rate, - level, encoder->type); - else if (IS_CANNONLAKE(dev_priv)) - cnl_ddi_vswing_sequence(encoder, level, encoder->type); - else - bxt_ddi_vswing_sequence(encoder, level, encoder->type); + tgl_ddi_vswing_sequence(encoder, intel_dp->link_rate, + level, encoder->type); +} - return 0; +static void +icl_set_signal_levels(struct intel_dp *intel_dp) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + int level = intel_ddi_dp_level(intel_dp); + + icl_ddi_vswing_sequence(encoder, intel_dp->link_rate, + level, encoder->type); } -u32 ddi_signal_levels(struct intel_dp *intel_dp) +static void +cnl_set_signal_levels(struct intel_dp *intel_dp) { - struct intel_digital_port *dport = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev); - struct intel_encoder *encoder = &dport->base; + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + int level = intel_ddi_dp_level(intel_dp); + + cnl_ddi_vswing_sequence(encoder, level, encoder->type); +} + +static void +bxt_set_signal_levels(struct intel_dp *intel_dp) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + int level = intel_ddi_dp_level(intel_dp); + + bxt_ddi_vswing_sequence(encoder, level, encoder->type); +} + +static void +hsw_set_signal_levels(struct intel_dp *intel_dp) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); int level = intel_ddi_dp_level(intel_dp); + enum port port = encoder->port; + u32 signal_levels; + + signal_levels = DDI_BUF_TRANS_SELECT(level); + + drm_dbg_kms(&dev_priv->drm, "Using signal levels %08x\n", + signal_levels); + + intel_dp->DP &= ~DDI_BUF_EMP_MASK; + intel_dp->DP |= signal_levels; if (IS_GEN9_BC(dev_priv)) skl_ddi_set_iboost(encoder, level, encoder->type); - return DDI_BUF_TRANS_SELECT(level); + intel_de_write(dev_priv, DDI_BUF_CTL(port), intel_dp->DP); + intel_de_posting_read(dev_priv, DDI_BUF_CTL(port)); } -static inline -u32 icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, - enum phy phy) +static u32 icl_dpclka_cfgcr0_clk_off(struct drm_i915_private *dev_priv, + enum phy phy) { if (intel_phy_is_combo(dev_priv, phy)) { return ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy); @@ -3044,10 +2752,11 @@ static void icl_map_plls_to_ports(struct intel_encoder *encoder, enum phy phy = intel_port_to_phy(dev_priv, encoder->port); u32 val; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); val = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); - WARN_ON((val & icl_dpclka_cfgcr0_clk_off(dev_priv, phy)) == 0); + drm_WARN_ON(&dev_priv->drm, + (val & icl_dpclka_cfgcr0_clk_off(dev_priv, phy)) == 0); if (intel_phy_is_combo(dev_priv, phy)) { /* @@ -3069,7 +2778,7 @@ static void icl_map_plls_to_ports(struct intel_encoder *encoder, val &= ~icl_dpclka_cfgcr0_clk_off(dev_priv, phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, val); - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static void icl_unmap_plls_to_ports(struct intel_encoder *encoder) @@ -3078,13 +2787,13 @@ static void icl_unmap_plls_to_ports(struct intel_encoder *encoder) enum phy phy = intel_port_to_phy(dev_priv, encoder->port); u32 val; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); val = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0); val |= icl_dpclka_cfgcr0_clk_off(dev_priv, phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, val); - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static void icl_sanitize_port_clk_off(struct drm_i915_private *dev_priv, @@ -3106,11 +2815,12 @@ static void icl_sanitize_port_clk_off(struct drm_i915_private *dev_priv, * Punt on the case now where clock is gated, but it would * be needed by the port. Something else is really broken then. */ - if (WARN_ON(ddi_clk_needed)) + if (drm_WARN_ON(&dev_priv->drm, ddi_clk_needed)) continue; - DRM_NOTE("PHY %c is disabled/in DSI mode with an ungated DDI clock, gate it\n", - phy_name(phy)); + drm_notice(&dev_priv->drm, + "PHY %c is disabled/in DSI mode with an ungated DDI clock, gate it\n", + phy_name(phy)); val |= icl_dpclka_cfgcr0_clk_off(dev_priv, phy); intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, val); } @@ -3138,7 +2848,7 @@ void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder) * In the unlikely case that BIOS enables DP in MST mode, just * warn since our MST HW readout is incomplete. */ - if (WARN_ON(is_mst)) + if (drm_WARN_ON(&dev_priv->drm, is_mst)) return; } @@ -3157,7 +2867,8 @@ void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder) if (other_encoder == encoder) continue; - if (WARN_ON(port_mask & BIT(other_encoder->port))) + if (drm_WARN_ON(&dev_priv->drm, + port_mask & BIT(other_encoder->port))) return; } /* @@ -3179,10 +2890,10 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder, u32 val; const struct intel_shared_dpll *pll = crtc_state->shared_dpll; - if (WARN_ON(!pll)) + if (drm_WARN_ON(&dev_priv->drm, !pll)) return; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); if (INTEL_GEN(dev_priv) >= 11) { if (!intel_phy_is_combo(dev_priv, phy)) @@ -3226,7 +2937,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder, hsw_pll_to_ddi_pll_sel(pll)); } - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static void intel_ddi_clk_disable(struct intel_encoder *encoder) @@ -3285,7 +2996,8 @@ icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port, switch (pin_assignment) { case 0x0: - WARN_ON(intel_dig_port->tc_mode != TC_PORT_LEGACY); + drm_WARN_ON(&dev_priv->drm, + intel_dig_port->tc_mode != TC_PORT_LEGACY); if (width == 1) { ln1 |= MG_DP_MODE_CFG_DP_X1_MODE; } else { @@ -3345,11 +3057,14 @@ icl_program_mg_dp_mode(struct intel_digital_port *intel_dig_port, static void intel_dp_sink_set_fec_ready(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + if (!crtc_state->fec_enable) return; if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_FEC_CONFIGURATION, DP_FEC_READY) <= 0) - DRM_DEBUG_KMS("Failed to set FEC_READY in the sink\n"); + drm_dbg_kms(&i915->drm, + "Failed to set FEC_READY in the sink\n"); } static void intel_ddi_enable_fec(struct intel_encoder *encoder, @@ -3369,7 +3084,8 @@ static void intel_ddi_enable_fec(struct intel_encoder *encoder, if (intel_de_wait_for_set(dev_priv, intel_dp->regs.dp_tp_status, DP_TP_STATUS_FEC_ENABLE_LIVE, 1)) - DRM_ERROR("Timed out waiting for FEC Enable Status\n"); + drm_err(&dev_priv->drm, + "Timed out waiting for FEC Enable Status\n"); } static void intel_ddi_disable_fec_state(struct intel_encoder *encoder, @@ -3389,7 +3105,8 @@ static void intel_ddi_disable_fec_state(struct intel_encoder *encoder, intel_de_posting_read(dev_priv, intel_dp->regs.dp_tp_ctl); } -static void tgl_ddi_pre_enable_dp(struct intel_encoder *encoder, +static void tgl_ddi_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -3457,13 +3174,13 @@ static void tgl_ddi_pre_enable_dp(struct intel_encoder *encoder, * 7.a Configure Transcoder Clock Select to direct the Port clock to the * Transcoder. */ - intel_ddi_enable_pipe_clock(crtc_state); + intel_ddi_enable_pipe_clock(encoder, crtc_state); /* * 7.b Configure TRANS_DDI_FUNC_CTL DDI Select, DDI Mode Select & MST * Transport Select */ - intel_ddi_config_transcoder_func(crtc_state); + intel_ddi_config_transcoder_func(encoder, crtc_state); /* * 7.c Configure & enable DP_TP_CTL with link training pattern 1 @@ -3529,7 +3246,8 @@ static void tgl_ddi_pre_enable_dp(struct intel_encoder *encoder, intel_dsc_enable(encoder, crtc_state); } -static void hsw_ddi_pre_enable_dp(struct intel_encoder *encoder, +static void hsw_ddi_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -3542,16 +3260,14 @@ static void hsw_ddi_pre_enable_dp(struct intel_encoder *encoder, int level = intel_ddi_dp_level(intel_dp); if (INTEL_GEN(dev_priv) < 11) - WARN_ON(is_mst && (port == PORT_A || port == PORT_E)); + drm_WARN_ON(&dev_priv->drm, + is_mst && (port == PORT_A || port == PORT_E)); else - WARN_ON(is_mst && port == PORT_A); + drm_WARN_ON(&dev_priv->drm, is_mst && port == PORT_A); intel_dp_set_link_params(intel_dp, crtc_state->port_clock, crtc_state->lane_count, is_mst); - intel_dp->regs.dp_tp_ctl = DP_TP_CTL(port); - intel_dp->regs.dp_tp_status = DP_TP_STATUS(port); - intel_edp_panel_on(intel_dp); intel_ddi_clk_select(encoder, crtc_state); @@ -3596,21 +3312,22 @@ static void hsw_ddi_pre_enable_dp(struct intel_encoder *encoder, intel_ddi_enable_fec(encoder, crtc_state); if (!is_mst) - intel_ddi_enable_pipe_clock(crtc_state); + intel_ddi_enable_pipe_clock(encoder, crtc_state); intel_dsc_enable(encoder, crtc_state); } -static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder, +static void intel_ddi_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); if (INTEL_GEN(dev_priv) >= 12) - tgl_ddi_pre_enable_dp(encoder, crtc_state, conn_state); + tgl_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); else - hsw_ddi_pre_enable_dp(encoder, crtc_state, conn_state); + hsw_ddi_pre_enable_dp(state, encoder, crtc_state, conn_state); /* MST will call a setting of MSA after an allocating of Virtual Channel * from MST encoder pre_enable callback. @@ -3622,7 +3339,8 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder, } } -static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder, +static void intel_ddi_pre_enable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -3655,14 +3373,15 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder, if (IS_GEN9_BC(dev_priv)) skl_ddi_set_iboost(encoder, level, INTEL_OUTPUT_HDMI); - intel_ddi_enable_pipe_clock(crtc_state); + intel_ddi_enable_pipe_clock(encoder, crtc_state); intel_dig_port->set_infoframes(encoder, crtc_state->has_infoframe, crtc_state, conn_state); } -static void intel_ddi_pre_enable(struct intel_encoder *encoder, +static void intel_ddi_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -3683,7 +3402,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder, * the DP link parameteres */ - WARN_ON(crtc_state->has_pch_encoder); + drm_WARN_ON(&dev_priv->drm, crtc_state->has_pch_encoder); if (INTEL_GEN(dev_priv) >= 11) icl_map_plls_to_ports(encoder, crtc_state); @@ -3691,12 +3410,14 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder, intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { - intel_ddi_pre_enable_hdmi(encoder, crtc_state, conn_state); + intel_ddi_pre_enable_hdmi(state, encoder, crtc_state, + conn_state); } else { struct intel_lspcon *lspcon = enc_to_intel_lspcon(encoder); - intel_ddi_pre_enable_dp(encoder, crtc_state, conn_state); + intel_ddi_pre_enable_dp(state, encoder, crtc_state, + conn_state); if (lspcon->active) { struct intel_digital_port *dig_port = enc_to_dig_port(encoder); @@ -3739,7 +3460,8 @@ static void intel_disable_ddi_buf(struct intel_encoder *encoder, intel_wait_ddi_buf_idle(dev_priv, port); } -static void intel_ddi_post_disable_dp(struct intel_encoder *encoder, +static void intel_ddi_post_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3750,6 +3472,8 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder, INTEL_OUTPUT_DP_MST); enum phy phy = intel_port_to_phy(dev_priv, encoder->port); + intel_dp_set_infoframes(encoder, false, old_crtc_state, old_conn_state); + /* * Power down sink before disabling the port, otherwise we end * up getting interrupts from the sink on detecting link loss. @@ -3795,7 +3519,8 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder, intel_ddi_clk_disable(encoder); } -static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder, +static void intel_ddi_post_disable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3818,22 +3543,8 @@ static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder, intel_dp_dual_mode_set_tmds_output(intel_hdmi, false); } -static void icl_disable_transcoder_port_sync(const struct intel_crtc_state *old_crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - - if (old_crtc_state->master_transcoder == INVALID_TRANSCODER) - return; - - DRM_DEBUG_KMS("Disabling Transcoder Port Sync on Slave Transcoder %s\n", - transcoder_name(old_crtc_state->cpu_transcoder)); - - intel_de_write(dev_priv, - TRANS_DDI_FUNC_CTL2(old_crtc_state->cpu_transcoder), 0); -} - -static void intel_ddi_post_disable(struct intel_encoder *encoder, +static void intel_ddi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3847,9 +3558,6 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder, intel_disable_pipe(old_crtc_state); - if (INTEL_GEN(dev_priv) >= 11) - icl_disable_transcoder_port_sync(old_crtc_state); - intel_ddi_disable_transcoder_func(old_crtc_state); intel_dsc_disable(old_crtc_state); @@ -3874,11 +3582,11 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder, */ if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI)) - intel_ddi_post_disable_hdmi(encoder, - old_crtc_state, old_conn_state); + intel_ddi_post_disable_hdmi(state, encoder, old_crtc_state, + old_conn_state); else - intel_ddi_post_disable_dp(encoder, - old_crtc_state, old_conn_state); + intel_ddi_post_disable_dp(state, encoder, old_crtc_state, + old_conn_state); if (INTEL_GEN(dev_priv) >= 11) icl_unmap_plls_to_ports(encoder); @@ -3891,7 +3599,8 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder, intel_tc_port_put_link(dig_port); } -void intel_ddi_fdi_post_disable(struct intel_encoder *encoder, +void intel_ddi_fdi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3925,7 +3634,43 @@ void intel_ddi_fdi_post_disable(struct intel_encoder *encoder, intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), val); } -static void intel_enable_ddi_dp(struct intel_encoder *encoder, +static void trans_port_sync_stop_link_train(struct intel_atomic_state *state, + struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state) +{ + const struct drm_connector_state *conn_state; + struct drm_connector *conn; + int i; + + if (!crtc_state->sync_mode_slaves_mask) + return; + + for_each_new_connector_in_state(&state->base, conn, conn_state, i) { + struct intel_encoder *slave_encoder = + to_intel_encoder(conn_state->best_encoder); + struct intel_crtc *slave_crtc = to_intel_crtc(conn_state->crtc); + const struct intel_crtc_state *slave_crtc_state; + + if (!slave_crtc) + continue; + + slave_crtc_state = + intel_atomic_get_new_crtc_state(state, slave_crtc); + + if (slave_crtc_state->master_transcoder != + crtc_state->cpu_transcoder) + continue; + + intel_dp_stop_link_train(enc_to_intel_dp(slave_encoder)); + } + + usleep_range(200, 400); + + intel_dp_stop_link_train(enc_to_intel_dp(encoder)); +} + +static void intel_enable_ddi_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -3937,13 +3682,14 @@ static void intel_enable_ddi_dp(struct intel_encoder *encoder, intel_dp_stop_link_train(intel_dp); intel_edp_backlight_on(crtc_state, conn_state); - intel_psr_enable(intel_dp, crtc_state); - intel_dp_vsc_enable(intel_dp, crtc_state, conn_state); - intel_dp_hdr_metadata_enable(intel_dp, crtc_state, conn_state); + intel_psr_enable(intel_dp, crtc_state, conn_state); + intel_dp_set_infoframes(encoder, true, crtc_state, conn_state); intel_edp_drrs_enable(intel_dp, crtc_state); if (crtc_state->has_audio) intel_audio_codec_enable(encoder, crtc_state, conn_state); + + trans_port_sync_stop_link_train(state, encoder, crtc_state); } static i915_reg_t @@ -3958,15 +3704,16 @@ gen9_chicken_trans_reg_by_port(struct drm_i915_private *dev_priv, [PORT_E] = TRANSCODER_A, }; - WARN_ON(INTEL_GEN(dev_priv) < 9); + drm_WARN_ON(&dev_priv->drm, INTEL_GEN(dev_priv) < 9); - if (WARN_ON(port < PORT_A || port > PORT_E)) + if (drm_WARN_ON(&dev_priv->drm, port < PORT_A || port > PORT_E)) port = PORT_A; return CHICKEN_TRANS(trans[port]); } -static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, +static void intel_enable_ddi_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -3978,8 +3725,9 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, if (!intel_hdmi_handle_sink_scrambling(encoder, connector, crtc_state->hdmi_high_tmds_clock_ratio, crtc_state->hdmi_scrambling)) - DRM_ERROR("[CONNECTOR:%d:%s] Failed to configure sink scrambling/TMDS bit clock ratio\n", - connector->base.id, connector->name); + drm_dbg_kms(&dev_priv->drm, + "[CONNECTOR:%d:%s] Failed to configure sink scrambling/TMDS bit clock ratio\n", + connector->base.id, connector->name); /* Display WA #1143: skl,kbl,cfl */ if (IS_GEN9_BC(dev_priv)) { @@ -4027,20 +3775,23 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder, intel_audio_codec_enable(encoder, crtc_state, conn_state); } -static void intel_enable_ddi(struct intel_encoder *encoder, +static void intel_enable_ddi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - WARN_ON(crtc_state->has_pch_encoder); + drm_WARN_ON(state->base.dev, crtc_state->has_pch_encoder); + + intel_ddi_enable_transcoder_func(encoder, crtc_state); intel_enable_pipe(crtc_state); intel_crtc_vblank_on(crtc_state); if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - intel_enable_ddi_hdmi(encoder, crtc_state, conn_state); + intel_enable_ddi_hdmi(state, encoder, crtc_state, conn_state); else - intel_enable_ddi_dp(encoder, crtc_state, conn_state); + intel_enable_ddi_dp(state, encoder, crtc_state, conn_state); /* Enable hdcp if it's desired */ if (conn_state->content_protection == @@ -4050,7 +3801,8 @@ static void intel_enable_ddi(struct intel_encoder *encoder, (u8)conn_state->hdcp_content_type); } -static void intel_disable_ddi_dp(struct intel_encoder *encoder, +static void intel_disable_ddi_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -4070,10 +3822,12 @@ static void intel_disable_ddi_dp(struct intel_encoder *encoder, false); } -static void intel_disable_ddi_hdmi(struct intel_encoder *encoder, +static void intel_disable_ddi_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct drm_connector *connector = old_conn_state->connector; if (old_crtc_state->has_audio) @@ -4082,23 +3836,28 @@ static void intel_disable_ddi_hdmi(struct intel_encoder *encoder, if (!intel_hdmi_handle_sink_scrambling(encoder, connector, false, false)) - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] Failed to reset sink scrambling/TMDS bit clock ratio\n", - connector->base.id, connector->name); + drm_dbg_kms(&i915->drm, + "[CONNECTOR:%d:%s] Failed to reset sink scrambling/TMDS bit clock ratio\n", + connector->base.id, connector->name); } -static void intel_disable_ddi(struct intel_encoder *encoder, +static void intel_disable_ddi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { intel_hdcp_disable(to_intel_connector(old_conn_state->connector)); if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI)) - intel_disable_ddi_hdmi(encoder, old_crtc_state, old_conn_state); + intel_disable_ddi_hdmi(state, encoder, old_crtc_state, + old_conn_state); else - intel_disable_ddi_dp(encoder, old_crtc_state, old_conn_state); + intel_disable_ddi_dp(state, encoder, old_crtc_state, + old_conn_state); } -static void intel_ddi_update_pipe_dp(struct intel_encoder *encoder, +static void intel_ddi_update_pipe_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -4106,21 +3865,24 @@ static void intel_ddi_update_pipe_dp(struct intel_encoder *encoder, intel_ddi_set_dp_msa(crtc_state, conn_state); - intel_psr_update(intel_dp, crtc_state); + intel_psr_update(intel_dp, crtc_state, conn_state); + intel_dp_set_infoframes(encoder, true, crtc_state, conn_state); intel_edp_drrs_enable(intel_dp, crtc_state); - intel_panel_update_backlight(encoder, crtc_state, conn_state); + intel_panel_update_backlight(state, encoder, crtc_state, conn_state); } -static void intel_ddi_update_pipe(struct intel_encoder *encoder, +static void intel_ddi_update_pipe(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) - intel_ddi_update_pipe_dp(encoder, crtc_state, conn_state); + intel_ddi_update_pipe_dp(state, encoder, crtc_state, + conn_state); - intel_hdcp_update_pipe(encoder, crtc_state, conn_state); + intel_hdcp_update_pipe(state, encoder, crtc_state, conn_state); } static void @@ -4132,7 +3894,7 @@ intel_ddi_update_prepare(struct intel_atomic_state *state, crtc ? intel_atomic_get_new_crtc_state(state, crtc) : NULL; int required_lanes = crtc_state ? crtc_state->lane_count : 1; - WARN_ON(crtc && crtc->active); + drm_WARN_ON(state->base.dev, crtc && crtc->active); intel_tc_port_get_link(enc_to_dig_port(encoder), required_lanes); @@ -4149,7 +3911,8 @@ intel_ddi_update_complete(struct intel_atomic_state *state, } static void -intel_ddi_pre_pll_enable(struct intel_encoder *encoder, +intel_ddi_pre_pll_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -4223,6 +3986,74 @@ static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp) udelay(600); } +static void intel_ddi_set_link_train(struct intel_dp *intel_dp, + u8 dp_train_pat) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + u8 train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); + enum port port = dp_to_dig_port(intel_dp)->base.port; + u32 temp; + + temp = intel_de_read(dev_priv, intel_dp->regs.dp_tp_ctl); + + if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) + temp |= DP_TP_CTL_SCRAMBLE_DISABLE; + else + temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; + + temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; + switch (dp_train_pat & train_pat_mask) { + case DP_TRAINING_PATTERN_DISABLE: + temp |= DP_TP_CTL_LINK_TRAIN_NORMAL; + break; + case DP_TRAINING_PATTERN_1: + temp |= DP_TP_CTL_LINK_TRAIN_PAT1; + break; + case DP_TRAINING_PATTERN_2: + temp |= DP_TP_CTL_LINK_TRAIN_PAT2; + break; + case DP_TRAINING_PATTERN_3: + temp |= DP_TP_CTL_LINK_TRAIN_PAT3; + break; + case DP_TRAINING_PATTERN_4: + temp |= DP_TP_CTL_LINK_TRAIN_PAT4; + break; + } + + intel_de_write(dev_priv, intel_dp->regs.dp_tp_ctl, temp); + + intel_de_write(dev_priv, DDI_BUF_CTL(port), intel_dp->DP); + intel_de_posting_read(dev_priv, DDI_BUF_CTL(port)); +} + +static void intel_ddi_set_idle_link_train(struct intel_dp *intel_dp) +{ + struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + enum port port = encoder->port; + u32 val; + + val = intel_de_read(dev_priv, intel_dp->regs.dp_tp_ctl); + val &= ~DP_TP_CTL_LINK_TRAIN_MASK; + val |= DP_TP_CTL_LINK_TRAIN_IDLE; + intel_de_write(dev_priv, intel_dp->regs.dp_tp_ctl, val); + + /* + * Until TGL on PORT_A we can have only eDP in SST mode. There the only + * reason we need to set idle transmission mode is to work around a HW + * issue where we enable the pipe while not in idle link-training mode. + * In this case there is requirement to wait for a minimum number of + * idle patterns to be sent. + */ + if (port == PORT_A && INTEL_GEN(dev_priv) < 12) + return; + + if (intel_de_wait_for_set(dev_priv, intel_dp->regs.dp_tp_status, + DP_TP_STATUS_IDLE_DONE, 1)) + drm_err(&dev_priv->drm, + "Timed out waiting for DP idle patterns\n"); +} + static bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv, enum transcoder cpu_transcoder) { @@ -4249,18 +4080,84 @@ void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv, crtc_state->min_voltage_level = 2; } +static enum transcoder bdw_transcoder_master_readout(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder) +{ + u32 master_select; + + if (INTEL_GEN(dev_priv) >= 11) { + u32 ctl2 = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL2(cpu_transcoder)); + + if ((ctl2 & PORT_SYNC_MODE_ENABLE) == 0) + return INVALID_TRANSCODER; + + master_select = REG_FIELD_GET(PORT_SYNC_MODE_MASTER_SELECT_MASK, ctl2); + } else { + u32 ctl = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); + + if ((ctl & TRANS_DDI_PORT_SYNC_ENABLE) == 0) + return INVALID_TRANSCODER; + + master_select = REG_FIELD_GET(TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK, ctl); + } + + if (master_select == 0) + return TRANSCODER_EDP; + else + return master_select - 1; +} + +static void bdw_get_trans_port_sync_config(struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + u32 transcoders = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | + BIT(TRANSCODER_C) | BIT(TRANSCODER_D); + enum transcoder cpu_transcoder; + + crtc_state->master_transcoder = + bdw_transcoder_master_readout(dev_priv, crtc_state->cpu_transcoder); + + for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) { + enum intel_display_power_domain power_domain; + intel_wakeref_t trans_wakeref; + + power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); + trans_wakeref = intel_display_power_get_if_enabled(dev_priv, + power_domain); + + if (!trans_wakeref) + continue; + + if (bdw_transcoder_master_readout(dev_priv, cpu_transcoder) == + crtc_state->cpu_transcoder) + crtc_state->sync_mode_slaves_mask |= BIT(cpu_transcoder); + + intel_display_power_put(dev_priv, power_domain, trans_wakeref); + } + + drm_WARN_ON(&dev_priv->drm, + crtc_state->master_transcoder != INVALID_TRANSCODER && + crtc_state->sync_mode_slaves_mask); +} + void intel_ddi_get_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->uapi.crtc); enum transcoder cpu_transcoder = pipe_config->cpu_transcoder; + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); u32 temp, flags = 0; /* XXX: DSI transcoder paranoia */ - if (WARN_ON(transcoder_is_dsi(cpu_transcoder))) + if (drm_WARN_ON(&dev_priv->drm, transcoder_is_dsi(cpu_transcoder))) return; + if (INTEL_GEN(dev_priv) >= 12) { + intel_dp->regs.dp_tp_ctl = TGL_DP_TP_CTL(cpu_transcoder); + intel_dp->regs.dp_tp_status = TGL_DP_TP_STATUS(cpu_transcoder); + } + intel_dsc_get_config(encoder, pipe_config); temp = intel_de_read(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder)); @@ -4334,11 +4231,15 @@ void intel_ddi_get_config(struct intel_encoder *encoder, pipe_config->fec_enable = intel_de_read(dev_priv, dp_tp_ctl) & DP_TP_CTL_FEC_ENABLE; - DRM_DEBUG_KMS("[ENCODER:%d:%s] Fec status: %u\n", - encoder->base.base.id, encoder->base.name, - pipe_config->fec_enable); + drm_dbg_kms(&dev_priv->drm, + "[ENCODER:%d:%s] Fec status: %u\n", + encoder->base.base.id, encoder->base.name, + pipe_config->fec_enable); } + pipe_config->infoframes.enable |= + intel_hdmi_infoframes_enabled(encoder, pipe_config); + break; case TRANS_DDI_MODE_SELECT_DP_MST: pipe_config->output_types |= BIT(INTEL_OUTPUT_DP_MST); @@ -4350,6 +4251,9 @@ void intel_ddi_get_config(struct intel_encoder *encoder, REG_FIELD_GET(TRANS_DDI_MST_TRANSPORT_SELECT_MASK, temp); intel_dp_get_m_n(intel_crtc, pipe_config); + + pipe_config->infoframes.enable |= + intel_hdmi_infoframes_enabled(encoder, pipe_config); break; default: break; @@ -4373,8 +4277,9 @@ void intel_ddi_get_config(struct intel_encoder *encoder, * up by the BIOS, and thus we can't get the mode at module * load. */ - DRM_DEBUG_KMS("pipe has %d bpp for eDP panel, overriding BIOS-provided max %d bpp\n", - pipe_config->pipe_bpp, dev_priv->vbt.edp.bpp); + drm_dbg_kms(&dev_priv->drm, + "pipe has %d bpp for eDP panel, overriding BIOS-provided max %d bpp\n", + pipe_config->pipe_bpp, dev_priv->vbt.edp.bpp); dev_priv->vbt.edp.bpp = pipe_config->pipe_bpp; } @@ -4400,6 +4305,12 @@ void intel_ddi_get_config(struct intel_encoder *encoder, intel_read_infoframe(encoder, pipe_config, HDMI_INFOFRAME_TYPE_DRM, &pipe_config->infoframes.drm); + + if (INTEL_GEN(dev_priv) >= 8) + bdw_get_trans_port_sync_config(pipe_config); + + intel_read_dp_sdp(encoder, pipe_config, HDMI_PACKET_TYPE_GAMUT_METADATA); + intel_read_dp_sdp(encoder, pipe_config, DP_SDP_VSC); } static enum intel_output_type @@ -4429,7 +4340,7 @@ static int intel_ddi_compute_config(struct intel_encoder *encoder, enum port port = encoder->port; int ret; - if (HAS_TRANSCODER_EDP(dev_priv) && port == PORT_A) + if (HAS_TRANSCODER(dev_priv, TRANSCODER_EDP) && port == PORT_A) pipe_config->cpu_transcoder = TRANSCODER_EDP; if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI)) { @@ -4501,7 +4412,11 @@ intel_ddi_port_sync_transcoders(const struct intel_crtc_state *ref_crtc_state, u8 transcoders = 0; int i; - if (INTEL_GEN(dev_priv) < 11) + /* + * We don't enable port sync on BDW due to missing w/as and + * due to not having adjusted the modeset sequence appropriately. + */ + if (INTEL_GEN(dev_priv) < 9) return 0; if (!intel_crtc_has_type(ref_crtc_state, INTEL_OUTPUT_DP)) @@ -4533,12 +4448,13 @@ static int intel_ddi_compute_config_late(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state, struct drm_connector_state *conn_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct drm_connector *connector = conn_state->connector; u8 port_sync_transcoders = 0; - DRM_DEBUG_KMS("[ENCODER:%d:%s] [CRTC:%d:%s]", - encoder->base.base.id, encoder->base.name, - crtc_state->uapi.crtc->base.id, crtc_state->uapi.crtc->name); + drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] [CRTC:%d:%s]", + encoder->base.base.id, encoder->base.name, + crtc_state->uapi.crtc->base.id, crtc_state->uapi.crtc->name); if (connector->has_tile) port_sync_transcoders = intel_ddi_port_sync_transcoders(crtc_state, @@ -4580,6 +4496,7 @@ static const struct drm_encoder_funcs intel_ddi_funcs = { static struct intel_connector * intel_ddi_init_dp_connector(struct intel_digital_port *intel_dig_port) { + struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev); struct intel_connector *connector; enum port port = intel_dig_port->base.port; @@ -4590,6 +4507,24 @@ intel_ddi_init_dp_connector(struct intel_digital_port *intel_dig_port) intel_dig_port->dp.output_reg = DDI_BUF_CTL(port); intel_dig_port->dp.prepare_link_retrain = intel_ddi_prepare_link_retrain; + intel_dig_port->dp.set_link_train = intel_ddi_set_link_train; + intel_dig_port->dp.set_idle_link_train = intel_ddi_set_idle_link_train; + + if (INTEL_GEN(dev_priv) >= 12) + intel_dig_port->dp.set_signal_levels = tgl_set_signal_levels; + else if (INTEL_GEN(dev_priv) >= 11) + intel_dig_port->dp.set_signal_levels = icl_set_signal_levels; + else if (IS_CANNONLAKE(dev_priv)) + intel_dig_port->dp.set_signal_levels = cnl_set_signal_levels; + else if (IS_GEN9_LP(dev_priv)) + intel_dig_port->dp.set_signal_levels = bxt_set_signal_levels; + else + intel_dig_port->dp.set_signal_levels = hsw_set_signal_levels; + + if (INTEL_GEN(dev_priv) < 12) { + intel_dig_port->dp.regs.dp_tp_ctl = DP_TP_CTL(port); + intel_dig_port->dp.regs.dp_tp_status = DP_TP_STATUS(port); + } if (!intel_dp_init_connector(intel_dig_port, connector)) { kfree(connector); @@ -4661,7 +4596,8 @@ static int intel_hdmi_reset_link(struct intel_encoder *encoder, crtc_state = to_intel_crtc_state(crtc->base.state); - WARN_ON(!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)); + drm_WARN_ON(&dev_priv->drm, + !intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)); if (!crtc_state->hw.active) return 0; @@ -4676,7 +4612,8 @@ static int intel_hdmi_reset_link(struct intel_encoder *encoder, ret = drm_scdc_readb(adapter, SCDC_TMDS_CONFIG, &config); if (ret < 0) { - DRM_ERROR("Failed to read TMDS config: %d\n", ret); + drm_err(&dev_priv->drm, "Failed to read TMDS config: %d\n", + ret); return 0; } @@ -4700,15 +4637,17 @@ static int intel_hdmi_reset_link(struct intel_encoder *encoder, static enum intel_hotplug_state intel_ddi_hotplug(struct intel_encoder *encoder, - struct intel_connector *connector, - bool irq_received) + struct intel_connector *connector) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_digital_port *dig_port = enc_to_dig_port(encoder); + enum phy phy = intel_port_to_phy(i915, encoder->port); + bool is_tc = intel_phy_is_tc(i915, phy); struct drm_modeset_acquire_ctx ctx; enum intel_hotplug_state state; int ret; - state = intel_encoder_hotplug(encoder, connector, irq_received); + state = intel_encoder_hotplug(encoder, connector); drm_modeset_acquire_init(&ctx, 0); @@ -4746,14 +4685,45 @@ intel_ddi_hotplug(struct intel_encoder *encoder, * valid EDID. To solve this schedule another detection cycle if this * time around we didn't detect any change in the sink's connection * status. + * + * Type-c connectors which get their HPD signal deasserted then + * reasserted, without unplugging/replugging the sink from the + * connector, introduce a delay until the AUX channel communication + * becomes functional. Retry the detection for 5 seconds on type-c + * connectors to account for this delay. */ - if (state == INTEL_HOTPLUG_UNCHANGED && irq_received && + if (state == INTEL_HOTPLUG_UNCHANGED && + connector->hotplug_retries < (is_tc ? 5 : 1) && !dig_port->dp.is_mst) state = INTEL_HOTPLUG_RETRY; return state; } +static bool lpt_digital_port_connected(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + u32 bit = dev_priv->hotplug.pch_hpd[encoder->hpd_pin]; + + return intel_de_read(dev_priv, SDEISR) & bit; +} + +static bool hsw_digital_port_connected(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin]; + + return intel_de_read(dev_priv, DEISR) & bit; +} + +static bool bdw_digital_port_connected(struct intel_encoder *encoder) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin]; + + return intel_de_read(dev_priv, GEN8_DE_PORT_ISR) & bit; +} + static struct intel_connector * intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port) { @@ -4822,7 +4792,8 @@ intel_ddi_max_lanes(struct intel_digital_port *intel_dport) * so we use the proper lane count for our calculations. */ if (intel_ddi_a_force_4_lanes(intel_dport)) { - DRM_DEBUG_KMS("Forcing DDI_A_4_LANES for port A\n"); + drm_dbg_kms(&dev_priv->drm, + "Forcing DDI_A_4_LANES for port A\n"); intel_dport->saved_port_bits |= DDI_A_4_LANES; max_lanes = 4; } @@ -4850,12 +4821,14 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) init_dp = true; init_lspcon = true; init_hdmi = false; - DRM_DEBUG_KMS("VBT says port %c has lspcon\n", port_name(port)); + drm_dbg_kms(&dev_priv->drm, "VBT says port %c has lspcon\n", + port_name(port)); } if (!init_dp && !init_hdmi) { - DRM_DEBUG_KMS("VBT says port %c is not DVI/HDMI/DP compatible, respect it\n", - port_name(port)); + drm_dbg_kms(&dev_priv->drm, + "VBT says port %c is not DVI/HDMI/DP compatible, respect it\n", + port_name(port)); return; } @@ -4913,7 +4886,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) encoder->update_complete = intel_ddi_update_complete; } - WARN_ON(port > PORT_I); + drm_WARN_ON(&dev_priv->drm, port > PORT_I); intel_dig_port->ddi_io_power_domain = POWER_DOMAIN_PORT_DDI_A_IO + port - PORT_A; @@ -4934,17 +4907,36 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port) if (init_lspcon) { if (lspcon_init(intel_dig_port)) /* TODO: handle hdmi info frame part */ - DRM_DEBUG_KMS("LSPCON init success on port %c\n", - port_name(port)); + drm_dbg_kms(&dev_priv->drm, + "LSPCON init success on port %c\n", + port_name(port)); else /* * LSPCON init faied, but DP init was success, so * lets try to drive as DP++ port. */ - DRM_ERROR("LSPCON init failed on port %c\n", + drm_err(&dev_priv->drm, + "LSPCON init failed on port %c\n", port_name(port)); } + if (INTEL_GEN(dev_priv) >= 11) { + if (intel_phy_is_tc(dev_priv, phy)) + intel_dig_port->connected = intel_tc_port_connected; + else + intel_dig_port->connected = lpt_digital_port_connected; + } else if (INTEL_GEN(dev_priv) >= 8) { + if (port == PORT_A || IS_GEN9_LP(dev_priv)) + intel_dig_port->connected = bdw_digital_port_connected; + else + intel_dig_port->connected = lpt_digital_port_connected; + } else { + if (port == PORT_A) + intel_dig_port->connected = hsw_digital_port_connected; + else + intel_dig_port->connected = lpt_digital_port_connected; + } + intel_infoframe_init(intel_dig_port); return; diff --git a/drivers/gpu/drm/i915/display/intel_ddi.h b/drivers/gpu/drm/i915/display/intel_ddi.h index 167c6579d972..fbdf8ddde486 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.h +++ b/drivers/gpu/drm/i915/display/intel_ddi.h @@ -6,8 +6,6 @@ #ifndef __INTEL_DDI_H__ #define __INTEL_DDI_H__ -#include <drm/i915_drm.h> - #include "intel_display.h" struct drm_connector_state; @@ -19,16 +17,19 @@ struct intel_dp; struct intel_dpll_hw_state; struct intel_encoder; -void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder, +void intel_ddi_fdi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *intel_encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state); void hsw_fdi_link_train(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state); void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port); bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe); -void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state); +void intel_ddi_enable_transcoder_func(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state); -void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state); +void intel_ddi_enable_pipe_clock(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state); void intel_ddi_disable_pipe_clock(const struct intel_crtc_state *crtc_state); void intel_ddi_set_dp_msa(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); @@ -47,7 +48,5 @@ u8 intel_ddi_dp_pre_emphasis_max(struct intel_encoder *encoder, int intel_ddi_toggle_hdcp_signalling(struct intel_encoder *intel_encoder, bool enable); void icl_sanitize_encoder_pll_mapping(struct intel_encoder *encoder); -int cnl_calc_wrpll_link(struct drm_i915_private *dev_priv, - struct intel_dpll_hw_state *state); #endif /* __INTEL_DDI_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 86ca35d160b8..4747d96ec66d 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -41,7 +41,6 @@ #include <drm/drm_plane_helper.h> #include <drm/drm_probe_helper.h> #include <drm/drm_rect.h> -#include <drm/i915_drm.h> #include "display/intel_crt.h" #include "display/intel_ddi.h" @@ -203,9 +202,9 @@ int vlv_get_cck_clock(struct drm_i915_private *dev_priv, val = vlv_cck_read(dev_priv, reg); divider = val & CCK_FREQUENCY_VALUES; - WARN((val & CCK_FREQUENCY_STATUS) != - (divider << CCK_FREQUENCY_STATUS_SHIFT), - "%s change in progress\n", name); + drm_WARN(&dev_priv->drm, (val & CCK_FREQUENCY_STATUS) != + (divider << CCK_FREQUENCY_STATUS_SHIFT), + "%s change in progress\n", name); return DIV_ROUND_CLOSEST(ref_freq << 1, divider + 1); } @@ -239,9 +238,9 @@ static void intel_update_czclk(struct drm_i915_private *dev_priv) dev_priv->czclk_freq); } -static inline u32 /* units of 100MHz */ -intel_fdi_link_freq(struct drm_i915_private *dev_priv, - const struct intel_crtc_state *pipe_config) +/* units of 100MHz */ +static u32 intel_fdi_link_freq(struct drm_i915_private *dev_priv, + const struct intel_crtc_state *pipe_config) { if (HAS_DDI(dev_priv)) return pipe_config->port_clock; /* SPLL */ @@ -526,7 +525,7 @@ skl_wa_827(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable) intel_de_read(dev_priv, CLKGATE_DIS_PSL(pipe)) & ~(DUPS1_GATING_DIS | DUPS2_GATING_DIS)); } -/* Wa_2006604312:icl */ +/* Wa_2006604312:icl,ehl */ static void icl_wa_scalerclkgating(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable) @@ -545,17 +544,23 @@ needs_modeset(const struct intel_crtc_state *state) return drm_atomic_crtc_needs_modeset(&state->uapi); } -bool -is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state) +static bool +is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state) { - return (crtc_state->master_transcoder != INVALID_TRANSCODER || - crtc_state->sync_mode_slaves_mask); + return crtc_state->master_transcoder != INVALID_TRANSCODER; } static bool -is_trans_port_sync_slave(const struct intel_crtc_state *crtc_state) +is_trans_port_sync_master(const struct intel_crtc_state *crtc_state) { - return crtc_state->master_transcoder != INVALID_TRANSCODER; + return crtc_state->sync_mode_slaves_mask != 0; +} + +bool +is_trans_port_sync_mode(const struct intel_crtc_state *crtc_state) +{ + return is_trans_port_sync_master(crtc_state) || + is_trans_port_sync_slave(crtc_state); } /* @@ -621,45 +626,43 @@ int chv_calc_dpll_params(int refclk, struct dpll *clock) return clock->dot / 5; } -#define INTELPllInvalid(s) do { /* DRM_DEBUG(s); */ return false; } while (0) - /* * Returns whether the given set of divisors are valid for a given refclk with * the given connectors. */ -static bool intel_PLL_is_valid(struct drm_i915_private *dev_priv, +static bool intel_pll_is_valid(struct drm_i915_private *dev_priv, const struct intel_limit *limit, const struct dpll *clock) { - if (clock->n < limit->n.min || limit->n.max < clock->n) - INTELPllInvalid("n out of range\n"); - if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1) - INTELPllInvalid("p1 out of range\n"); - if (clock->m2 < limit->m2.min || limit->m2.max < clock->m2) - INTELPllInvalid("m2 out of range\n"); - if (clock->m1 < limit->m1.min || limit->m1.max < clock->m1) - INTELPllInvalid("m1 out of range\n"); + if (clock->n < limit->n.min || limit->n.max < clock->n) + return false; + if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1) + return false; + if (clock->m2 < limit->m2.min || limit->m2.max < clock->m2) + return false; + if (clock->m1 < limit->m1.min || limit->m1.max < clock->m1) + return false; if (!IS_PINEVIEW(dev_priv) && !IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv)) if (clock->m1 <= clock->m2) - INTELPllInvalid("m1 <= m2\n"); + return false; if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && !IS_GEN9_LP(dev_priv)) { if (clock->p < limit->p.min || limit->p.max < clock->p) - INTELPllInvalid("p out of range\n"); + return false; if (clock->m < limit->m.min || limit->m.max < clock->m) - INTELPllInvalid("m out of range\n"); + return false; } if (clock->vco < limit->vco.min || limit->vco.max < clock->vco) - INTELPllInvalid("vco out of range\n"); + return false; /* XXX: We may need to be checking "Dot clock" depending on the multiplier, * connector, etc., rather than just a single range. */ if (clock->dot < limit->dot.min || limit->dot.max < clock->dot) - INTELPllInvalid("dot out of range\n"); + return false; return true; } @@ -726,7 +729,7 @@ i9xx_find_best_dpll(const struct intel_limit *limit, int this_err; i9xx_calc_dpll_params(refclk, &clock); - if (!intel_PLL_is_valid(to_i915(dev), + if (!intel_pll_is_valid(to_i915(dev), limit, &clock)) continue; @@ -782,7 +785,7 @@ pnv_find_best_dpll(const struct intel_limit *limit, int this_err; pnv_calc_dpll_params(refclk, &clock); - if (!intel_PLL_is_valid(to_i915(dev), + if (!intel_pll_is_valid(to_i915(dev), limit, &clock)) continue; @@ -843,7 +846,7 @@ g4x_find_best_dpll(const struct intel_limit *limit, int this_err; i9xx_calc_dpll_params(refclk, &clock); - if (!intel_PLL_is_valid(to_i915(dev), + if (!intel_pll_is_valid(to_i915(dev), limit, &clock)) continue; @@ -882,7 +885,7 @@ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq, return calculated_clock->p > best_clock->p; } - if (WARN_ON_ONCE(!target_freq)) + if (drm_WARN_ON_ONCE(dev, !target_freq)) return false; *error_ppm = div_u64(1000000ULL * @@ -940,7 +943,7 @@ vlv_find_best_dpll(const struct intel_limit *limit, vlv_calc_dpll_params(refclk, &clock); - if (!intel_PLL_is_valid(to_i915(dev), + if (!intel_pll_is_valid(to_i915(dev), limit, &clock)) continue; @@ -1009,7 +1012,7 @@ chv_find_best_dpll(const struct intel_limit *limit, chv_calc_dpll_params(refclk, &clock); - if (!intel_PLL_is_valid(to_i915(dev), limit, &clock)) + if (!intel_pll_is_valid(to_i915(dev), limit, &clock)) continue; if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock, @@ -1090,7 +1093,8 @@ intel_wait_for_pipe_off(const struct intel_crtc_state *old_crtc_state) /* Wait for the Pipe State to go off */ if (intel_de_wait_for_clear(dev_priv, reg, I965_PIPECONF_ACTIVE, 100)) - WARN(1, "pipe_off wait timed out\n"); + drm_WARN(&dev_priv->drm, 1, + "pipe_off wait timed out\n"); } else { intel_wait_for_pipe_scanline_stopped(crtc); } @@ -1205,7 +1209,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) enum pipe panel_pipe = INVALID_PIPE; bool locked = true; - if (WARN_ON(HAS_DDI(dev_priv))) + if (drm_WARN_ON(&dev_priv->drm, HAS_DDI(dev_priv))) return; if (HAS_PCH_SPLIT(dev_priv)) { @@ -1241,7 +1245,8 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe) pp_reg = PP_CONTROL(0); port_sel = intel_de_read(dev_priv, PP_ON_DELAYS(0)) & PANEL_PORT_SELECT_MASK; - WARN_ON(port_sel != PANEL_PORT_SELECT_LVDS); + drm_WARN_ON(&dev_priv->drm, + port_sel != PANEL_PORT_SELECT_LVDS); intel_lvds_port_enabled(dev_priv, LVDS, &panel_pipe); } @@ -1482,7 +1487,9 @@ static void chv_enable_pll(struct intel_crtc *crtc, * DPLLB VGA mode also seems to cause problems. * We should always have it disabled. */ - WARN_ON((intel_de_read(dev_priv, DPLL(PIPE_B)) & DPLL_VGA_MODE_DIS) == 0); + drm_WARN_ON(&dev_priv->drm, + (intel_de_read(dev_priv, DPLL(PIPE_B)) & + DPLL_VGA_MODE_DIS) == 0); } else { intel_de_write(dev_priv, DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md); @@ -1630,10 +1637,11 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv, if (intel_de_wait_for_register(dev_priv, dpll_reg, port_mask, expected_mask, 1000)) - WARN(1, "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n", - dport->base.base.base.id, dport->base.base.name, - intel_de_read(dev_priv, dpll_reg) & port_mask, - expected_mask); + drm_WARN(&dev_priv->drm, 1, + "timed out waiting for [ENCODER:%d:%s] port ready: got 0x%x, expected 0x%x\n", + dport->base.base.base.id, dport->base.base.name, + intel_de_read(dev_priv, dpll_reg) & port_mask, + expected_mask); } static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state) @@ -1872,7 +1880,7 @@ void intel_enable_pipe(const struct intel_crtc_state *new_crtc_state) val = intel_de_read(dev_priv, reg); if (val & PIPECONF_ENABLE) { /* we keep both pipes enabled on 830 */ - WARN_ON(!IS_I830(dev_priv)); + drm_WARN_ON(&dev_priv->drm, !IS_I830(dev_priv)); return; } @@ -1965,16 +1973,16 @@ static bool is_aux_plane(const struct drm_framebuffer *fb, int plane) static int main_to_ccs_plane(const struct drm_framebuffer *fb, int main_plane) { - WARN_ON(!is_ccs_modifier(fb->modifier) || - (main_plane && main_plane >= fb->format->num_planes / 2)); + drm_WARN_ON(fb->dev, !is_ccs_modifier(fb->modifier) || + (main_plane && main_plane >= fb->format->num_planes / 2)); return fb->format->num_planes / 2 + main_plane; } static int ccs_to_main_plane(const struct drm_framebuffer *fb, int ccs_plane) { - WARN_ON(!is_ccs_modifier(fb->modifier) || - ccs_plane < fb->format->num_planes / 2); + drm_WARN_ON(fb->dev, !is_ccs_modifier(fb->modifier) || + ccs_plane < fb->format->num_planes / 2); return ccs_plane - fb->format->num_planes / 2; } @@ -2218,11 +2226,11 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, unsigned int pinctl; u32 alignment; - if (WARN_ON(!i915_gem_object_is_framebuffer(obj))) + if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj))) return ERR_PTR(-EINVAL); alignment = intel_surf_alignment(fb, 0); - if (WARN_ON(alignment && !is_power_of_2(alignment))) + if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment))) return ERR_PTR(-EINVAL); /* Note that the w/a also requires 64 PTE of padding following the @@ -2393,7 +2401,7 @@ static u32 intel_adjust_aligned_offset(int *x, int *y, struct drm_i915_private *dev_priv = to_i915(fb->dev); unsigned int cpp = fb->format->cpp[color_plane]; - WARN_ON(new_offset > old_offset); + drm_WARN_ON(&dev_priv->drm, new_offset > old_offset); if (!is_surface_linear(fb, color_plane)) { unsigned int tile_size, tile_width, tile_height; @@ -2715,9 +2723,10 @@ u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv, /* * We assume the primary plane for pipe A has - * the highest stride limits of them all. + * the highest stride limits of them all, + * if in case pipe A is disabled, use the first pipe from pipe_mask. */ - crtc = intel_get_crtc_for_pipe(dev_priv, PIPE_A); + crtc = intel_get_first_crtc(dev_priv); if (!crtc) return 0; @@ -2905,6 +2914,7 @@ intel_fb_plane_get_subsampling(int *hsub, int *vsub, static int intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y) { + struct drm_i915_private *i915 = to_i915(fb->dev); struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); int main_plane; int hsub, vsub; @@ -2933,7 +2943,8 @@ intel_fb_check_ccs_xy(struct drm_framebuffer *fb, int ccs_plane, int x, int y) * x/y offsets must match between CCS and the main surface. */ if (main_x != ccs_x || main_y != ccs_y) { - DRM_DEBUG_KMS("Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n", + drm_dbg_kms(&i915->drm, + "Bad CCS x/y (main %d,%d ccs %d,%d) full (main %d,%d ccs %d,%d)\n", main_x, main_y, ccs_x, ccs_y, intel_fb->normal[main_plane].x, @@ -2981,7 +2992,7 @@ setup_fb_rotation(int plane, const struct intel_remapped_plane_info *plane_info, fb->modifier != I915_FORMAT_MOD_Yf_TILED) return 0; - if (WARN_ON(plane >= ARRAY_SIZE(rot_info->plane))) + if (drm_WARN_ON(fb->dev, plane >= ARRAY_SIZE(rot_info->plane))) return 0; rot_info->plane[plane] = *plane_info; @@ -3155,7 +3166,7 @@ intel_plane_remap_gtt(struct intel_plane_state *plane_state) src_w = drm_rect_width(&plane_state->uapi.src) >> 16; src_h = drm_rect_height(&plane_state->uapi.src) >> 16; - WARN_ON(is_ccs_modifier(fb->modifier)); + drm_WARN_ON(&dev_priv->drm, is_ccs_modifier(fb->modifier)); /* Make src coordinates relative to the viewport */ drm_rect_translate(&plane_state->uapi.src, @@ -3196,7 +3207,7 @@ intel_plane_remap_gtt(struct intel_plane_state *plane_state) DRM_MODE_ROTATE_0, tile_size); offset /= tile_size; - WARN_ON(i >= ARRAY_SIZE(info->plane)); + drm_WARN_ON(&dev_priv->drm, i >= ARRAY_SIZE(info->plane)); info->plane[i].offset = offset; info->plane[i].stride = DIV_ROUND_UP(fb->pitches[i], tile_width * cpp); @@ -3331,6 +3342,8 @@ int skl_format_to_fourcc(int format, bool rgb_order, bool alpha) return DRM_FORMAT_RGB565; case PLANE_CTL_FORMAT_NV12: return DRM_FORMAT_NV12; + case PLANE_CTL_FORMAT_XYUV: + return DRM_FORMAT_XYUV8888; case PLANE_CTL_FORMAT_P010: return DRM_FORMAT_P010; case PLANE_CTL_FORMAT_P012: @@ -3847,7 +3860,7 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state) intel_add_fb_offsets(&x, &y, plane_state, 0); offset = intel_plane_compute_aligned_offset(&x, &y, plane_state, 0); alignment = intel_surf_alignment(fb, 0); - if (WARN_ON(alignment && !is_power_of_2(alignment))) + if (drm_WARN_ON(&dev_priv->drm, alignment && !is_power_of_2(alignment))) return -EINVAL; /* @@ -4494,10 +4507,15 @@ static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id) { struct drm_device *dev = intel_crtc->base.dev; struct drm_i915_private *dev_priv = to_i915(dev); + unsigned long irqflags; + + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + + intel_de_write_fw(dev_priv, SKL_PS_CTRL(intel_crtc->pipe, id), 0); + intel_de_write_fw(dev_priv, SKL_PS_WIN_POS(intel_crtc->pipe, id), 0); + intel_de_write_fw(dev_priv, SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0); - intel_de_write(dev_priv, SKL_PS_CTRL(intel_crtc->pipe, id), 0); - intel_de_write(dev_priv, SKL_PS_WIN_POS(intel_crtc->pipe, id), 0); - intel_de_write(dev_priv, SKL_PS_WIN_SZ(intel_crtc->pipe, id), 0); + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } /* @@ -4570,6 +4588,8 @@ static u32 skl_plane_ctl_format(u32 pixel_format) case DRM_FORMAT_XRGB16161616F: case DRM_FORMAT_ARGB16161616F: return PLANE_CTL_FORMAT_XRGB_16161616F; + case DRM_FORMAT_XYUV8888: + return PLANE_CTL_FORMAT_XYUV; case DRM_FORMAT_YUYV: return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV; case DRM_FORMAT_YVYU: @@ -4841,7 +4861,7 @@ __intel_display_resume(struct drm_device *dev, ret = drm_atomic_helper_commit_duplicated_state(state, ctx); - WARN_ON(ret == -EDEADLK); + drm_WARN_ON(dev, ret == -EDEADLK); return ret; } @@ -4988,37 +5008,6 @@ static void icl_set_pipe_chicken(struct intel_crtc *crtc) intel_de_write(dev_priv, PIPE_CHICKEN(pipe), tmp); } -static void icl_enable_trans_port_sync(const struct intel_crtc_state *crtc_state) -{ - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - u32 trans_ddi_func_ctl2_val; - u8 master_select; - - /* - * Configure the master select and enable Transcoder Port Sync for - * Slave CRTCs transcoder. - */ - if (crtc_state->master_transcoder == INVALID_TRANSCODER) - return; - - if (crtc_state->master_transcoder == TRANSCODER_EDP) - master_select = 0; - else - master_select = crtc_state->master_transcoder + 1; - - /* Set the master select bits for Tranascoder Port Sync */ - trans_ddi_func_ctl2_val = (PORT_SYNC_MODE_MASTER_SELECT(master_select) & - PORT_SYNC_MODE_MASTER_SELECT_MASK) << - PORT_SYNC_MODE_MASTER_SELECT_SHIFT; - /* Enable Transcoder Port Sync */ - trans_ddi_func_ctl2_val |= PORT_SYNC_MODE_ENABLE; - - intel_de_write(dev_priv, - TRANS_DDI_FUNC_CTL2(crtc_state->cpu_transcoder), - trans_ddi_func_ctl2_val); -} - static void intel_fdi_normal_train(struct intel_crtc *crtc) { struct drm_device *dev = crtc->base.dev; @@ -5621,10 +5610,10 @@ static void lpt_program_iclkip(const struct intel_crtc_state *crtc_state) } /* This should not happen with any sane values */ - WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) & - ~SBI_SSCDIVINTPHASE_DIVSEL_MASK); - WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) & - ~SBI_SSCDIVINTPHASE_INCVAL_MASK); + drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(divsel) & + ~SBI_SSCDIVINTPHASE_DIVSEL_MASK); + drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIR(phasedir) & + ~SBI_SSCDIVINTPHASE_INCVAL_MASK); drm_dbg_kms(&dev_priv->drm, "iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n", @@ -5730,8 +5719,12 @@ static void cpt_set_fdi_bc_bifurcation(struct drm_i915_private *dev_priv, bool e if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable) return; - WARN_ON(intel_de_read(dev_priv, FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE); - WARN_ON(intel_de_read(dev_priv, FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, FDI_RX_CTL(PIPE_B)) & + FDI_RX_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, FDI_RX_CTL(PIPE_C)) & + FDI_RX_ENABLE); temp &= ~FDI_BC_BIFURCATION_SELECT; if (enable) @@ -5880,7 +5873,7 @@ static void ilk_pch_enable(const struct intel_atomic_state *state, temp |= TRANS_DP_VSYNC_ACTIVE_HIGH; port = intel_get_crtc_new_encoder(state, crtc_state)->port; - WARN_ON(port < PORT_B || port > PORT_D); + drm_WARN_ON(dev, port < PORT_B || port > PORT_D); temp |= TRANS_DP_PORT_SEL(port); intel_de_write(dev_priv, reg, temp); @@ -6096,30 +6089,26 @@ skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach, return 0; } -/** - * skl_update_scaler_crtc - Stages update to scaler state for a given crtc. - * - * @state: crtc's scaler state - * - * Return - * 0 - scaler_usage updated successfully - * error - requested scaling cannot be supported or other error condition - */ -int skl_update_scaler_crtc(struct intel_crtc_state *state) +static int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state) { - const struct drm_display_mode *adjusted_mode = &state->hw.adjusted_mode; - bool need_scaler = false; + const struct drm_display_mode *adjusted_mode = + &crtc_state->hw.adjusted_mode; + int width, height; - if (state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 || - state->pch_pfit.enabled) - need_scaler = true; + if (crtc_state->pch_pfit.enabled) { + width = drm_rect_width(&crtc_state->pch_pfit.dst); + height = drm_rect_height(&crtc_state->pch_pfit.dst); + } else { + width = adjusted_mode->crtc_hdisplay; + height = adjusted_mode->crtc_vdisplay; + } - return skl_update_scaler(state, !state->hw.active, SKL_CRTC_INDEX, - &state->scaler_state.scaler_id, - state->pipe_src_w, state->pipe_src_h, - adjusted_mode->crtc_hdisplay, - adjusted_mode->crtc_vdisplay, NULL, 0, - need_scaler); + return skl_update_scaler(crtc_state, !crtc_state->hw.active, + SKL_CRTC_INDEX, + &crtc_state->scaler_state.scaler_id, + crtc_state->pipe_src_w, crtc_state->pipe_src_h, + width, height, NULL, 0, + crtc_state->pch_pfit.enabled); } /** @@ -6186,6 +6175,7 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state, case DRM_FORMAT_UYVY: case DRM_FORMAT_VYUY: case DRM_FORMAT_NV12: + case DRM_FORMAT_XYUV8888: case DRM_FORMAT_P010: case DRM_FORMAT_P012: case DRM_FORMAT_P016: @@ -6227,63 +6217,80 @@ static void skl_pfit_enable(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - enum pipe pipe = crtc->pipe; const struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state; + struct drm_rect src = { + .x2 = crtc_state->pipe_src_w << 16, + .y2 = crtc_state->pipe_src_h << 16, + }; + const struct drm_rect *dst = &crtc_state->pch_pfit.dst; + u16 uv_rgb_hphase, uv_rgb_vphase; + enum pipe pipe = crtc->pipe; + int width = drm_rect_width(dst); + int height = drm_rect_height(dst); + int x = dst->x1; + int y = dst->y1; + int hscale, vscale; + unsigned long irqflags; + int id; - if (crtc_state->pch_pfit.enabled) { - u16 uv_rgb_hphase, uv_rgb_vphase; - int pfit_w, pfit_h, hscale, vscale; - int id; + if (!crtc_state->pch_pfit.enabled) + return; - if (WARN_ON(crtc_state->scaler_state.scaler_id < 0)) - return; + if (drm_WARN_ON(&dev_priv->drm, + crtc_state->scaler_state.scaler_id < 0)) + return; - pfit_w = (crtc_state->pch_pfit.size >> 16) & 0xFFFF; - pfit_h = crtc_state->pch_pfit.size & 0xFFFF; + hscale = drm_rect_calc_hscale(&src, dst, 0, INT_MAX); + vscale = drm_rect_calc_vscale(&src, dst, 0, INT_MAX); - hscale = (crtc_state->pipe_src_w << 16) / pfit_w; - vscale = (crtc_state->pipe_src_h << 16) / pfit_h; + uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false); + uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false); - uv_rgb_hphase = skl_scaler_calc_phase(1, hscale, false); - uv_rgb_vphase = skl_scaler_calc_phase(1, vscale, false); + id = scaler_state->scaler_id; - id = scaler_state->scaler_id; - intel_de_write(dev_priv, SKL_PS_CTRL(pipe, id), - PS_SCALER_EN | PS_FILTER_MEDIUM | scaler_state->scalers[id].mode); - intel_de_write_fw(dev_priv, SKL_PS_VPHASE(pipe, id), - PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase)); - intel_de_write_fw(dev_priv, SKL_PS_HPHASE(pipe, id), - PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase)); - intel_de_write(dev_priv, SKL_PS_WIN_POS(pipe, id), - crtc_state->pch_pfit.pos); - intel_de_write(dev_priv, SKL_PS_WIN_SZ(pipe, id), - crtc_state->pch_pfit.size); - } + spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); + + intel_de_write_fw(dev_priv, SKL_PS_CTRL(pipe, id), PS_SCALER_EN | + PS_FILTER_MEDIUM | scaler_state->scalers[id].mode); + intel_de_write_fw(dev_priv, SKL_PS_VPHASE(pipe, id), + PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_vphase)); + intel_de_write_fw(dev_priv, SKL_PS_HPHASE(pipe, id), + PS_Y_PHASE(0) | PS_UV_RGB_PHASE(uv_rgb_hphase)); + intel_de_write_fw(dev_priv, SKL_PS_WIN_POS(pipe, id), + x << 16 | y); + intel_de_write_fw(dev_priv, SKL_PS_WIN_SZ(pipe, id), + width << 16 | height); + + spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); } static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + const struct drm_rect *dst = &crtc_state->pch_pfit.dst; enum pipe pipe = crtc->pipe; + int width = drm_rect_width(dst); + int height = drm_rect_height(dst); + int x = dst->x1; + int y = dst->y1; - if (crtc_state->pch_pfit.enabled) { - /* Force use of hard-coded filter coefficients - * as some pre-programmed values are broken, - * e.g. x201. - */ - if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) - intel_de_write(dev_priv, PF_CTL(pipe), - PF_ENABLE | PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe)); - else - intel_de_write(dev_priv, PF_CTL(pipe), - PF_ENABLE | PF_FILTER_MED_3x3); - intel_de_write(dev_priv, PF_WIN_POS(pipe), - crtc_state->pch_pfit.pos); - intel_de_write(dev_priv, PF_WIN_SZ(pipe), - crtc_state->pch_pfit.size); - } + if (!crtc_state->pch_pfit.enabled) + return; + + /* Force use of hard-coded filter coefficients + * as some pre-programmed values are broken, + * e.g. x201. + */ + if (IS_IVYBRIDGE(dev_priv) || IS_HASWELL(dev_priv)) + intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE | + PF_FILTER_MED_3x3 | PF_PIPE_SEL_IVB(pipe)); + else + intel_de_write(dev_priv, PF_CTL(pipe), PF_ENABLE | + PF_FILTER_MED_3x3); + intel_de_write(dev_priv, PF_WIN_POS(pipe), x << 16 | y); + intel_de_write(dev_priv, PF_WIN_SZ(pipe), width << 16 | height); } void hsw_enable_ips(const struct intel_crtc_state *crtc_state) @@ -6300,11 +6307,11 @@ void hsw_enable_ips(const struct intel_crtc_state *crtc_state) * This function is called from post_plane_update, which is run after * a vblank wait. */ - WARN_ON(!(crtc_state->active_planes & ~BIT(PLANE_CURSOR))); + drm_WARN_ON(dev, !(crtc_state->active_planes & ~BIT(PLANE_CURSOR))); if (IS_BROADWELL(dev_priv)) { - WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, - IPS_ENABLE | IPS_PCODE_CONTROL)); + drm_WARN_ON(dev, sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, + IPS_ENABLE | IPS_PCODE_CONTROL)); /* Quoting Art Runyan: "its not safe to expect any particular * value in IPS_CTL bit 31 after enabling IPS through the * mailbox." Moreover, the mailbox may return a bogus state, @@ -6333,7 +6340,8 @@ void hsw_disable_ips(const struct intel_crtc_state *crtc_state) return; if (IS_BROADWELL(dev_priv)) { - WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0)); + drm_WARN_ON(dev, + sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0)); /* * Wait for PCODE to finish disabling IPS. The BSpec specified * 42ms timeout value leads to occasional timeouts so use 100ms @@ -6441,8 +6449,8 @@ static bool needs_scalerclk_wa(const struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - /* Wa_2006604312:icl */ - if (crtc_state->scaler_state.scaler_users > 0 && IS_ICELAKE(dev_priv)) + /* Wa_2006604312:icl,ehl */ + if (crtc_state->scaler_state.scaler_users > 0 && IS_GEN(dev_priv, 11)) return true; return false; @@ -6512,7 +6520,7 @@ static void intel_pre_plane_update(struct intel_atomic_state *state, needs_nv12_wa(new_crtc_state)) skl_wa_827(dev_priv, pipe, true); - /* Wa_2006604312:icl */ + /* Wa_2006604312:icl,ehl */ if (!needs_scalerclk_wa(old_crtc_state) && needs_scalerclk_wa(new_crtc_state)) icl_wa_scalerclkgating(dev_priv, pipe, true); @@ -6624,7 +6632,7 @@ intel_connector_primary_encoder(struct intel_connector *connector) return &dp_to_dig_port(connector->mst_port)->base; encoder = intel_attached_encoder(connector); - WARN_ON(!encoder); + drm_WARN_ON(connector->base.dev, !encoder); return encoder; } @@ -6698,7 +6706,8 @@ static void intel_encoders_pre_pll_enable(struct intel_atomic_state *state, continue; if (encoder->pre_pll_enable) - encoder->pre_pll_enable(encoder, crtc_state, conn_state); + encoder->pre_pll_enable(state, encoder, + crtc_state, conn_state); } } @@ -6719,7 +6728,8 @@ static void intel_encoders_pre_enable(struct intel_atomic_state *state, continue; if (encoder->pre_enable) - encoder->pre_enable(encoder, crtc_state, conn_state); + encoder->pre_enable(state, encoder, + crtc_state, conn_state); } } @@ -6740,7 +6750,8 @@ static void intel_encoders_enable(struct intel_atomic_state *state, continue; if (encoder->enable) - encoder->enable(encoder, crtc_state, conn_state); + encoder->enable(state, encoder, + crtc_state, conn_state); intel_opregion_notify_encoder(encoder, true); } } @@ -6763,7 +6774,8 @@ static void intel_encoders_disable(struct intel_atomic_state *state, intel_opregion_notify_encoder(encoder, false); if (encoder->disable) - encoder->disable(encoder, old_crtc_state, old_conn_state); + encoder->disable(state, encoder, + old_crtc_state, old_conn_state); } } @@ -6784,7 +6796,8 @@ static void intel_encoders_post_disable(struct intel_atomic_state *state, continue; if (encoder->post_disable) - encoder->post_disable(encoder, old_crtc_state, old_conn_state); + encoder->post_disable(state, encoder, + old_crtc_state, old_conn_state); } } @@ -6805,7 +6818,8 @@ static void intel_encoders_post_pll_disable(struct intel_atomic_state *state, continue; if (encoder->post_pll_disable) - encoder->post_pll_disable(encoder, old_crtc_state, old_conn_state); + encoder->post_pll_disable(state, encoder, + old_crtc_state, old_conn_state); } } @@ -6826,7 +6840,8 @@ static void intel_encoders_update_pipe(struct intel_atomic_state *state, continue; if (encoder->update_pipe) - encoder->update_pipe(encoder, crtc_state, conn_state); + encoder->update_pipe(state, encoder, + crtc_state, conn_state); } } @@ -6846,7 +6861,7 @@ static void ilk_crtc_enable(struct intel_atomic_state *state, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (WARN_ON(crtc->active)) + if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; /* @@ -7002,7 +7017,7 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, enum transcoder cpu_transcoder = new_crtc_state->cpu_transcoder; bool psl_clkgate_wa; - if (WARN_ON(crtc->active)) + if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; intel_encoders_pre_pll_enable(state, crtc); @@ -7015,9 +7030,6 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, if (!transcoder_is_dsi(cpu_transcoder)) intel_set_pipe_timings(new_crtc_state); - if (INTEL_GEN(dev_priv) >= 11) - icl_enable_trans_port_sync(new_crtc_state); - intel_set_pipe_src_size(new_crtc_state); if (cpu_transcoder != TRANSCODER_EDP && @@ -7065,9 +7077,6 @@ static void hsw_crtc_enable(struct intel_atomic_state *state, if (INTEL_GEN(dev_priv) >= 11) icl_set_pipe_chicken(crtc); - if (!transcoder_is_dsi(cpu_transcoder)) - intel_ddi_enable_transcoder_func(new_crtc_state); - if (dev_priv->display.initial_watermarks) dev_priv->display.initial_watermarks(state, crtc); @@ -7098,11 +7107,12 @@ void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state) /* To avoid upsetting the power well on haswell only disable the pfit if * it's in use. The hw state code will make sure we get this right. */ - if (old_crtc_state->pch_pfit.enabled) { - intel_de_write(dev_priv, PF_CTL(pipe), 0); - intel_de_write(dev_priv, PF_WIN_POS(pipe), 0); - intel_de_write(dev_priv, PF_WIN_SZ(pipe), 0); - } + if (!old_crtc_state->pch_pfit.enabled) + return; + + intel_de_write(dev_priv, PF_CTL(pipe), 0); + intel_de_write(dev_priv, PF_WIN_POS(pipe), 0); + intel_de_write(dev_priv, PF_WIN_SZ(pipe), 0); } static void ilk_crtc_disable(struct intel_atomic_state *state, @@ -7185,7 +7195,8 @@ static void i9xx_pfit_enable(const struct intel_crtc_state *crtc_state) * The panel fitter should only be adjusted whilst the pipe is disabled, * according to register description and PRM. */ - WARN_ON(intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, PFIT_CONTROL) & PFIT_ENABLE); assert_pipe_disabled(dev_priv, crtc_state->cpu_transcoder); intel_de_write(dev_priv, PFIT_PGM_RATIOS, @@ -7289,7 +7300,17 @@ intel_aux_power_domain(struct intel_digital_port *dig_port) } } - switch (dig_port->aux_ch) { + return intel_legacy_aux_to_power_domain(dig_port->aux_ch); +} + +/* + * Converts aux_ch to power_domain without caring about TBT ports for that use + * intel_aux_power_domain() + */ +enum intel_display_power_domain +intel_legacy_aux_to_power_domain(enum aux_ch aux_ch) +{ + switch (aux_ch) { case AUX_CH_A: return POWER_DOMAIN_AUX_A; case AUX_CH_B: @@ -7305,7 +7326,7 @@ intel_aux_power_domain(struct intel_digital_port *dig_port) case AUX_CH_G: return POWER_DOMAIN_AUX_G; default: - MISSING_CASE(dig_port->aux_ch); + MISSING_CASE(aux_ch); return POWER_DOMAIN_AUX_A; } } @@ -7381,7 +7402,7 @@ static void valleyview_crtc_enable(struct intel_atomic_state *state, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (WARN_ON(crtc->active)) + if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; if (intel_crtc_has_dp_encoder(new_crtc_state)) @@ -7447,7 +7468,7 @@ static void i9xx_crtc_enable(struct intel_atomic_state *state, struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - if (WARN_ON(crtc->active)) + if (drm_WARN_ON(&dev_priv->drm, crtc->active)) return; i9xx_set_pll_dividers(new_crtc_state); @@ -7593,7 +7614,7 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, temp_crtc_state = intel_atomic_get_crtc_state(state, crtc); ret = drm_atomic_add_affected_connectors(state, &crtc->base); - WARN_ON(IS_ERR(temp_crtc_state) || ret); + drm_WARN_ON(&dev_priv->drm, IS_ERR(temp_crtc_state) || ret); dev_priv->display.crtc_disable(to_intel_atomic_state(state), crtc); @@ -7606,7 +7627,8 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc, crtc->active = false; crtc->base.enabled = false; - WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0); + drm_WARN_ON(&dev_priv->drm, + drm_atomic_set_mode_for_crtc(&crtc_state->uapi, NULL) < 0); crtc_state->uapi.active = false; crtc_state->uapi.connector_mask = 0; crtc_state->uapi.encoder_mask = 0; @@ -7918,39 +7940,36 @@ static bool intel_crtc_supports_double_wide(const struct intel_crtc *crtc) (crtc->pipe == PIPE_A || IS_I915G(dev_priv)); } -static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config) +static u32 ilk_pipe_pixel_rate(const struct intel_crtc_state *crtc_state) { - u32 pixel_rate; - - pixel_rate = pipe_config->hw.adjusted_mode.crtc_clock; + u32 pixel_rate = crtc_state->hw.adjusted_mode.crtc_clock; + unsigned int pipe_w, pipe_h, pfit_w, pfit_h; /* * We only use IF-ID interlacing. If we ever use * PF-ID we'll need to adjust the pixel_rate here. */ - if (pipe_config->pch_pfit.enabled) { - u64 pipe_w, pipe_h, pfit_w, pfit_h; - u32 pfit_size = pipe_config->pch_pfit.size; + if (!crtc_state->pch_pfit.enabled) + return pixel_rate; - pipe_w = pipe_config->pipe_src_w; - pipe_h = pipe_config->pipe_src_h; + pipe_w = crtc_state->pipe_src_w; + pipe_h = crtc_state->pipe_src_h; - pfit_w = (pfit_size >> 16) & 0xFFFF; - pfit_h = pfit_size & 0xFFFF; - if (pipe_w < pfit_w) - pipe_w = pfit_w; - if (pipe_h < pfit_h) - pipe_h = pfit_h; + pfit_w = drm_rect_width(&crtc_state->pch_pfit.dst); + pfit_h = drm_rect_height(&crtc_state->pch_pfit.dst); - if (WARN_ON(!pfit_w || !pfit_h)) - return pixel_rate; + if (pipe_w < pfit_w) + pipe_w = pfit_w; + if (pipe_h < pfit_h) + pipe_h = pfit_h; - pixel_rate = div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h), - pfit_w * pfit_h); - } + if (drm_WARN_ON(crtc_state->uapi.crtc->dev, + !pfit_w || !pfit_h)) + return pixel_rate; - return pixel_rate; + return div_u64(mul_u32_u32(pixel_rate, pipe_w * pipe_h), + pfit_w * pfit_h); } static void intel_crtc_compute_pixel_rate(struct intel_crtc_state *crtc_state) @@ -8119,7 +8138,7 @@ static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv) } } -static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) +static bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) { if (i915_modparams.panel_use_ssc >= 0) return i915_modparams.panel_use_ssc != 0; @@ -9144,9 +9163,9 @@ static bool i9xx_has_pfit(struct drm_i915_private *dev_priv) IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv); } -static void i9xx_get_pfit_config(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +static void i9xx_get_pfit_config(struct intel_crtc_state *crtc_state) { + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 tmp; @@ -9166,9 +9185,9 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc, return; } - pipe_config->gmch_pfit.control = tmp; - pipe_config->gmch_pfit.pgm_ratios = intel_de_read(dev_priv, - PFIT_PGM_RATIOS); + crtc_state->gmch_pfit.control = tmp; + crtc_state->gmch_pfit.pgm_ratios = + intel_de_read(dev_priv, PFIT_PGM_RATIOS); } static void vlv_crtc_clock_get(struct intel_crtc *crtc, @@ -9216,7 +9235,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc, if (!plane->get_hw_state(plane, &pipe)) return; - WARN_ON(pipe != crtc->pipe); + drm_WARN_ON(dev, pipe != crtc->pipe); intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); if (!intel_fb) { @@ -9328,7 +9347,8 @@ bdw_get_pipemisc_output_format(struct intel_crtc *crtc) if (tmp & PIPEMISC_YUV420_ENABLE) { /* We support 4:2:0 in full blend mode only */ - WARN_ON((tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0); + drm_WARN_ON(&dev_priv->drm, + (tmp & PIPEMISC_YUV420_MODE_FULL_BLEND) == 0); return INTEL_OUTPUT_FORMAT_YCBCR420; } else if (tmp & PIPEMISC_OUTPUT_COLORSPACE_YUV) { @@ -9373,7 +9393,6 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB; pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; pipe_config->shared_dpll = NULL; - pipe_config->master_transcoder = INVALID_TRANSCODER; ret = false; @@ -9418,7 +9437,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc, intel_get_pipe_timings(crtc, pipe_config); intel_get_pipe_src_size(crtc, pipe_config); - i9xx_get_pfit_config(crtc, pipe_config); + i9xx_get_pfit_config(pipe_config); if (INTEL_GEN(dev_priv) >= 4) { /* No way to read it out on pipes B and C */ @@ -9517,7 +9536,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv) } /* Check if any DPLLs are using the SSC source */ - for (i = 0; i < dev_priv->num_shared_dpll; i++) { + for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { u32 temp = intel_de_read(dev_priv, PCH_DPLL(i)); if (!(temp & DPLL_VCO_ENABLE)) @@ -9755,10 +9774,11 @@ static void lpt_enable_clkout_dp(struct drm_i915_private *dev_priv, { u32 reg, tmp; - if (WARN(with_fdi && !with_spread, "FDI requires downspread\n")) + if (drm_WARN(&dev_priv->drm, with_fdi && !with_spread, + "FDI requires downspread\n")) with_spread = true; - if (WARN(HAS_PCH_LPT_LP(dev_priv) && - with_fdi, "LP PCH doesn't have FDI\n")) + if (drm_WARN(&dev_priv->drm, HAS_PCH_LPT_LP(dev_priv) && + with_fdi, "LP PCH doesn't have FDI\n")) with_fdi = false; mutex_lock(&dev_priv->sb_lock); @@ -9852,10 +9872,10 @@ static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps) u32 tmp; int idx = BEND_IDX(steps); - if (WARN_ON(steps % 5 != 0)) + if (drm_WARN_ON(&dev_priv->drm, steps % 5 != 0)) return; - if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase))) + if (drm_WARN_ON(&dev_priv->drm, idx >= ARRAY_SIZE(sscdivintphase))) return; mutex_lock(&dev_priv->sb_lock); @@ -10023,8 +10043,8 @@ static void ilk_set_pipeconf(const struct intel_crtc_state *crtc_state) * This would end up with an odd purple hue over * the entire display. Make sure we don't do it. */ - WARN_ON(crtc_state->limited_color_range && - crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB); + drm_WARN_ON(&dev_priv->drm, crtc_state->limited_color_range && + crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB); if (crtc_state->limited_color_range) val |= PIPECONF_COLOR_RANGE_SELECT; @@ -10103,6 +10123,9 @@ static void bdw_set_pipemisc(const struct intel_crtc_state *crtc_state) BIT(PLANE_CURSOR))) == 0) val |= PIPEMISC_HDR_MODE_PRECISION; + if (INTEL_GEN(dev_priv) >= 12) + val |= PIPEMISC_PIXEL_ROUNDING_TRUNC; + intel_de_write(dev_priv, PIPEMISC(crtc->pipe), val); } @@ -10384,37 +10407,47 @@ static void ilk_get_fdi_m_n_config(struct intel_crtc *crtc, &pipe_config->fdi_m_n, NULL); } -static void skl_get_pfit_config(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +static void ilk_get_pfit_pos_size(struct intel_crtc_state *crtc_state, + u32 pos, u32 size) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state; - u32 ps_ctrl = 0; + drm_rect_init(&crtc_state->pch_pfit.dst, + pos >> 16, pos & 0xffff, + size >> 16, size & 0xffff); +} + +static void skl_get_pfit_config(struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state; int id = -1; int i; /* find scaler attached to this pipe */ for (i = 0; i < crtc->num_scalers; i++) { - ps_ctrl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i)); - if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) { - id = i; - pipe_config->pch_pfit.enabled = true; - pipe_config->pch_pfit.pos = intel_de_read(dev_priv, - SKL_PS_WIN_POS(crtc->pipe, i)); - pipe_config->pch_pfit.size = intel_de_read(dev_priv, - SKL_PS_WIN_SZ(crtc->pipe, i)); - scaler_state->scalers[i].in_use = true; - break; - } + u32 ctl, pos, size; + + ctl = intel_de_read(dev_priv, SKL_PS_CTRL(crtc->pipe, i)); + if ((ctl & (PS_SCALER_EN | PS_PLANE_SEL_MASK)) != PS_SCALER_EN) + continue; + + id = i; + crtc_state->pch_pfit.enabled = true; + + pos = intel_de_read(dev_priv, SKL_PS_WIN_POS(crtc->pipe, i)); + size = intel_de_read(dev_priv, SKL_PS_WIN_SZ(crtc->pipe, i)); + + ilk_get_pfit_pos_size(crtc_state, pos, size); + + scaler_state->scalers[i].in_use = true; + break; } scaler_state->scaler_id = id; - if (id >= 0) { + if (id >= 0) scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX); - } else { + else scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX); - } } static void @@ -10435,7 +10468,7 @@ skl_get_initial_plane_config(struct intel_crtc *crtc, if (!plane->get_hw_state(plane, &pipe)) return; - WARN_ON(pipe != crtc->pipe); + drm_WARN_ON(dev, pipe != crtc->pipe); intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL); if (!intel_fb) { @@ -10550,30 +10583,30 @@ error: kfree(intel_fb); } -static void ilk_get_pfit_config(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config) +static void ilk_get_pfit_config(struct intel_crtc_state *crtc_state) { - struct drm_device *dev = crtc->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - u32 tmp; + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + u32 ctl, pos, size; - tmp = intel_de_read(dev_priv, PF_CTL(crtc->pipe)); - - if (tmp & PF_ENABLE) { - pipe_config->pch_pfit.enabled = true; - pipe_config->pch_pfit.pos = intel_de_read(dev_priv, - PF_WIN_POS(crtc->pipe)); - pipe_config->pch_pfit.size = intel_de_read(dev_priv, - PF_WIN_SZ(crtc->pipe)); - - /* We currently do not free assignements of panel fitters on - * ivb/hsw (since we don't use the higher upscaling modes which - * differentiates them) so just WARN about this case for now. */ - if (IS_GEN(dev_priv, 7)) { - WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) != - PF_PIPE_SEL_IVB(crtc->pipe)); - } - } + ctl = intel_de_read(dev_priv, PF_CTL(crtc->pipe)); + if ((ctl & PF_ENABLE) == 0) + return; + + crtc_state->pch_pfit.enabled = true; + + pos = intel_de_read(dev_priv, PF_WIN_POS(crtc->pipe)); + size = intel_de_read(dev_priv, PF_WIN_SZ(crtc->pipe)); + + ilk_get_pfit_pos_size(crtc_state, pos, size); + + /* + * We currently do not free assignements of panel fitters on + * ivb/hsw (since we don't use the higher upscaling modes which + * differentiates them) so just WARN about this case for now. + */ + drm_WARN_ON(&dev_priv->drm, IS_GEN(dev_priv, 7) && + (ctl & PF_PIPE_SEL_MASK_IVB) != PF_PIPE_SEL_IVB(crtc->pipe)); } static bool ilk_get_pipe_config(struct intel_crtc *crtc, @@ -10593,7 +10626,6 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe; pipe_config->shared_dpll = NULL; - pipe_config->master_transcoder = INVALID_TRANSCODER; ret = false; tmp = intel_de_read(dev_priv, PIPECONF(crtc->pipe)); @@ -10669,8 +10701,8 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, intel_get_shared_dpll_by_id(dev_priv, pll_id); pll = pipe_config->shared_dpll; - WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll, - &pipe_config->dpll_hw_state)); + drm_WARN_ON(dev, !pll->info->funcs->get_hw_state(dev_priv, pll, + &pipe_config->dpll_hw_state)); tmp = pipe_config->dpll_hw_state.dpll; pipe_config->pixel_multiplier = @@ -10685,7 +10717,7 @@ static bool ilk_get_pipe_config(struct intel_crtc *crtc, intel_get_pipe_timings(crtc, pipe_config); intel_get_pipe_src_size(crtc, pipe_config); - ilk_get_pfit_config(crtc, pipe_config); + ilk_get_pfit_config(pipe_config); ret = true; @@ -10727,7 +10759,7 @@ static void cnl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, temp = intel_de_read(dev_priv, DPCLKA_CFGCR0) & DPCLKA_CFGCR0_DDI_CLK_SEL_MASK(port); id = temp >> DPCLKA_CFGCR0_DDI_CLK_SEL_SHIFT(port); - if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL2)) + if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL2)) return; pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); @@ -10754,12 +10786,13 @@ static void icl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, port)); port_dpll_id = ICL_PORT_DPLL_MG_PHY; } else { - WARN_ON(clk_sel < DDI_CLK_SEL_TBT_162); + drm_WARN_ON(&dev_priv->drm, + clk_sel < DDI_CLK_SEL_TBT_162); id = DPLL_ID_ICL_TBTPLL; port_dpll_id = ICL_PORT_DPLL_DEFAULT; } } else { - WARN(1, "Invalid port %x\n", port); + drm_WARN(&dev_priv->drm, 1, "Invalid port %x\n", port); return; } @@ -10802,7 +10835,7 @@ static void skl_get_ddi_pll(struct drm_i915_private *dev_priv, enum port port, temp = intel_de_read(dev_priv, DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port); id = temp >> (port * 3 + 1); - if (WARN_ON(id < SKL_DPLL0 || id > SKL_DPLL3)) + if (drm_WARN_ON(&dev_priv->drm, id < SKL_DPLL0 || id > SKL_DPLL3)) return; pipe_config->shared_dpll = intel_get_shared_dpll_by_id(dev_priv, id); @@ -10861,7 +10894,7 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, panel_transcoder_mask |= BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1); - if (HAS_TRANSCODER_EDP(dev_priv)) + if (HAS_TRANSCODER(dev_priv, TRANSCODER_EDP)) panel_transcoder_mask |= BIT(TRANSCODER_EDP); /* @@ -10896,8 +10929,9 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, switch (tmp & TRANS_DDI_EDP_INPUT_MASK) { default: - WARN(1, "unknown pipe linked to transcoder %s\n", - transcoder_name(panel_transcoder)); + drm_WARN(dev, 1, + "unknown pipe linked to transcoder %s\n", + transcoder_name(panel_transcoder)); /* fall through */ case TRANS_DDI_EDP_INPUT_A_ONOFF: force_thru = true; @@ -10925,11 +10959,11 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc, /* * Valid combos: none, eDP, DSI0, DSI1, DSI0+DSI1 */ - WARN_ON((enabled_panel_transcoders & BIT(TRANSCODER_EDP)) && - enabled_panel_transcoders != BIT(TRANSCODER_EDP)); + drm_WARN_ON(dev, (enabled_panel_transcoders & BIT(TRANSCODER_EDP)) && + enabled_panel_transcoders != BIT(TRANSCODER_EDP)); power_domain = POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder); - WARN_ON(*power_domain_mask & BIT_ULL(power_domain)); + drm_WARN_ON(dev, *power_domain_mask & BIT_ULL(power_domain)); wf = intel_display_power_get_if_enabled(dev_priv, power_domain); if (!wf) @@ -10963,7 +10997,7 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc, cpu_transcoder = TRANSCODER_DSI_C; power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); - WARN_ON(*power_domain_mask & BIT_ULL(power_domain)); + drm_WARN_ON(dev, *power_domain_mask & BIT_ULL(power_domain)); wf = intel_display_power_get_if_enabled(dev_priv, power_domain); if (!wf) @@ -11032,7 +11066,8 @@ static void hsw_get_ddi_port_state(struct intel_crtc *crtc, pll = pipe_config->shared_dpll; if (pll) { - WARN_ON(!pll->info->funcs->get_hw_state(dev_priv, pll, + drm_WARN_ON(&dev_priv->drm, + !pll->info->funcs->get_hw_state(dev_priv, pll, &pipe_config->dpll_hw_state)); } @@ -11053,60 +11088,6 @@ static void hsw_get_ddi_port_state(struct intel_crtc *crtc, } } -static enum transcoder transcoder_master_readout(struct drm_i915_private *dev_priv, - enum transcoder cpu_transcoder) -{ - u32 trans_port_sync, master_select; - - trans_port_sync = intel_de_read(dev_priv, - TRANS_DDI_FUNC_CTL2(cpu_transcoder)); - - if ((trans_port_sync & PORT_SYNC_MODE_ENABLE) == 0) - return INVALID_TRANSCODER; - - master_select = trans_port_sync & - PORT_SYNC_MODE_MASTER_SELECT_MASK; - if (master_select == 0) - return TRANSCODER_EDP; - else - return master_select - 1; -} - -static void icl_get_trans_port_sync_config(struct intel_crtc_state *crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - u32 transcoders; - enum transcoder cpu_transcoder; - - crtc_state->master_transcoder = transcoder_master_readout(dev_priv, - crtc_state->cpu_transcoder); - - transcoders = BIT(TRANSCODER_A) | - BIT(TRANSCODER_B) | - BIT(TRANSCODER_C) | - BIT(TRANSCODER_D); - for_each_cpu_transcoder_masked(dev_priv, cpu_transcoder, transcoders) { - enum intel_display_power_domain power_domain; - intel_wakeref_t trans_wakeref; - - power_domain = POWER_DOMAIN_TRANSCODER(cpu_transcoder); - trans_wakeref = intel_display_power_get_if_enabled(dev_priv, - power_domain); - - if (!trans_wakeref) - continue; - - if (transcoder_master_readout(dev_priv, cpu_transcoder) == - crtc_state->cpu_transcoder) - crtc_state->sync_mode_slaves_mask |= BIT(cpu_transcoder); - - intel_display_power_put(dev_priv, power_domain, trans_wakeref); - } - - WARN_ON(crtc_state->master_transcoder != INVALID_TRANSCODER && - crtc_state->sync_mode_slaves_mask); -} - static bool hsw_get_pipe_config(struct intel_crtc *crtc, struct intel_crtc_state *pipe_config) { @@ -11135,7 +11116,7 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, if (IS_GEN9_LP(dev_priv) && bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask, wakerefs)) { - WARN_ON(active); + drm_WARN_ON(&dev_priv->drm, active); active = true; } @@ -11202,7 +11183,7 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, REG_FIELD_GET(HSW_IPS_LINETIME_MASK, tmp); power_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe); - WARN_ON(power_domain_mask & BIT_ULL(power_domain)); + drm_WARN_ON(&dev_priv->drm, power_domain_mask & BIT_ULL(power_domain)); wf = intel_display_power_get_if_enabled(dev_priv, power_domain); if (wf) { @@ -11210,9 +11191,9 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, power_domain_mask |= BIT_ULL(power_domain); if (INTEL_GEN(dev_priv) >= 9) - skl_get_pfit_config(crtc, pipe_config); + skl_get_pfit_config(pipe_config); else - ilk_get_pfit_config(crtc, pipe_config); + ilk_get_pfit_config(pipe_config); } if (hsw_crtc_supports_ips(crtc)) { @@ -11238,10 +11219,6 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc, pipe_config->pixel_multiplier = 1; } - if (INTEL_GEN(dev_priv) >= 11 && - !transcoder_is_dsi(pipe_config->cpu_transcoder)) - icl_get_trans_port_sync_config(pipe_config); - out: for_each_power_domain(power_domain, power_domain_mask) intel_display_power_put(dev_priv, @@ -11449,8 +11426,8 @@ static int i845_check_cursor(struct intel_crtc_state *crtc_state, return -EINVAL; } - WARN_ON(plane_state->uapi.visible && - plane_state->color_plane[0].stride != fb->pitches[0]); + drm_WARN_ON(&i915->drm, plane_state->uapi.visible && + plane_state->color_plane[0].stride != fb->pitches[0]); switch (fb->pitches[0]) { case 256: @@ -11666,8 +11643,8 @@ static int i9xx_check_cursor(struct intel_crtc_state *crtc_state, return -EINVAL; } - WARN_ON(plane_state->uapi.visible && - plane_state->color_plane[0].stride != fb->pitches[0]); + drm_WARN_ON(&dev_priv->drm, plane_state->uapi.visible && + plane_state->color_plane[0].stride != fb->pitches[0]); if (fb->pitches[0] != drm_rect_width(&plane_state->uapi.dst) * fb->format->cpp[0]) { @@ -11886,7 +11863,7 @@ int intel_get_load_detect_pipe(struct drm_connector *connector, old->restore_state = NULL; - WARN_ON(!drm_modeset_is_locked(&config->connection_mutex)); + drm_WARN_ON(dev, !drm_modeset_is_locked(&config->connection_mutex)); /* * Algorithm gets a little messy: @@ -12330,7 +12307,7 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat was_visible = old_plane_state->uapi.visible; visible = plane_state->uapi.visible; - if (!was_crtc_enabled && WARN_ON(was_visible)) + if (!was_crtc_enabled && drm_WARN_ON(&dev_priv->drm, was_visible)) was_visible = false; /* @@ -12344,10 +12321,8 @@ int intel_plane_atomic_calc_changes(const struct intel_crtc_state *old_crtc_stat * only combine the results from all planes in the current place? */ if (!is_crtc_enabled) { - plane_state->uapi.visible = visible = false; - crtc_state->active_planes &= ~BIT(plane->id); - crtc_state->data_rate[plane->id] = 0; - crtc_state->min_cdclk[plane->id] = 0; + intel_plane_set_invisible(crtc_state, plane_state); + visible = false; } if (!was_visible && !visible) @@ -12477,8 +12452,10 @@ static int icl_add_linked_planes(struct intel_atomic_state *state) if (IS_ERR(linked_plane_state)) return PTR_ERR(linked_plane_state); - WARN_ON(linked_plane_state->planar_linked_plane != plane); - WARN_ON(linked_plane_state->planar_slave == plane_state->planar_slave); + drm_WARN_ON(state->base.dev, + linked_plane_state->planar_linked_plane != plane); + drm_WARN_ON(state->base.dev, + linked_plane_state->planar_slave == plane_state->planar_slave); } return 0; @@ -12676,7 +12653,7 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, if (mode_changed && crtc_state->hw.enable && dev_priv->display.crtc_compute_clock && - !WARN_ON(crtc_state->shared_dpll)) { + !drm_WARN_ON(&dev_priv->drm, crtc_state->shared_dpll)) { ret = dev_priv->display.crtc_compute_clock(crtc, crtc_state); if (ret) return ret; @@ -12706,7 +12683,8 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state, } if (dev_priv->display.compute_intermediate_wm) { - if (WARN_ON(!dev_priv->display.compute_pipe_wm)) + if (drm_WARN_ON(&dev_priv->drm, + !dev_priv->display.compute_pipe_wm)) return 0; /* @@ -12852,19 +12830,20 @@ compute_baseline_pipe_bpp(struct intel_crtc *crtc, return 0; } -static void intel_dump_crtc_timings(const struct drm_display_mode *mode) +static void intel_dump_crtc_timings(struct drm_i915_private *i915, + const struct drm_display_mode *mode) { - DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, " - "type: 0x%x flags: 0x%x\n", - mode->crtc_clock, - mode->crtc_hdisplay, mode->crtc_hsync_start, - mode->crtc_hsync_end, mode->crtc_htotal, - mode->crtc_vdisplay, mode->crtc_vsync_start, - mode->crtc_vsync_end, mode->crtc_vtotal, - mode->type, mode->flags); + drm_dbg_kms(&i915->drm, "crtc timings: %d %d %d %d %d %d %d %d %d, " + "type: 0x%x flags: 0x%x\n", + mode->crtc_clock, + mode->crtc_hdisplay, mode->crtc_hsync_start, + mode->crtc_hsync_end, mode->crtc_htotal, + mode->crtc_vdisplay, mode->crtc_vsync_start, + mode->crtc_vsync_end, mode->crtc_vtotal, + mode->type, mode->flags); } -static inline void +static void intel_dump_m_n_config(const struct intel_crtc_state *pipe_config, const char *id, unsigned int lane_count, const struct intel_link_m_n *m_n) @@ -12888,6 +12867,16 @@ intel_dump_infoframe(struct drm_i915_private *dev_priv, hdmi_infoframe_log(KERN_DEBUG, dev_priv->drm.dev, frame); } +static void +intel_dump_dp_vsc_sdp(struct drm_i915_private *dev_priv, + const struct drm_dp_vsc_sdp *vsc) +{ + if (!drm_debug_enabled(DRM_UT_KMS)) + return; + + drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, vsc); +} + #define OUTPUT_TYPE(x) [INTEL_OUTPUT_ ## x] = #x static const char * const output_type_str[] = { @@ -13008,6 +12997,11 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, transcoder_name(pipe_config->cpu_transcoder), pipe_config->pipe_bpp, pipe_config->dither); + drm_dbg_kms(&dev_priv->drm, + "port sync: master transcoder: %s, slave transcoder bitmask = 0x%x\n", + transcoder_name(pipe_config->master_transcoder), + pipe_config->sync_mode_slaves_mask); + if (pipe_config->has_pch_encoder) intel_dump_m_n_config(pipe_config, "fdi", pipe_config->fdi_lanes, @@ -13040,12 +13034,21 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, if (pipe_config->infoframes.enable & intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_VENDOR)) intel_dump_infoframe(dev_priv, &pipe_config->infoframes.hdmi); + if (pipe_config->infoframes.enable & + intel_hdmi_infoframe_enable(HDMI_INFOFRAME_TYPE_DRM)) + intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm); + if (pipe_config->infoframes.enable & + intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA)) + intel_dump_infoframe(dev_priv, &pipe_config->infoframes.drm); + if (pipe_config->infoframes.enable & + intel_hdmi_infoframe_enable(DP_SDP_VSC)) + intel_dump_dp_vsc_sdp(dev_priv, &pipe_config->infoframes.vsc); drm_dbg_kms(&dev_priv->drm, "requested mode:\n"); drm_mode_debug_printmodeline(&pipe_config->hw.mode); drm_dbg_kms(&dev_priv->drm, "adjusted mode:\n"); drm_mode_debug_printmodeline(&pipe_config->hw.adjusted_mode); - intel_dump_crtc_timings(&pipe_config->hw.adjusted_mode); + intel_dump_crtc_timings(dev_priv, &pipe_config->hw.adjusted_mode); drm_dbg_kms(&dev_priv->drm, "port clock: %d, pipe src size: %dx%d, pixel rate %d\n", pipe_config->port_clock, @@ -13070,9 +13073,8 @@ static void intel_dump_pipe_config(const struct intel_crtc_state *pipe_config, pipe_config->gmch_pfit.lvds_border_bits); else drm_dbg_kms(&dev_priv->drm, - "pch pfit: pos: 0x%08x, size: 0x%08x, %s, force thru: %s\n", - pipe_config->pch_pfit.pos, - pipe_config->pch_pfit.size, + "pch pfit: " DRM_RECT_FMT ", %s, force thru: %s\n", + DRM_RECT_ARG(&pipe_config->pch_pfit.dst), enableddisabled(pipe_config->pch_pfit.enabled), yesno(pipe_config->pch_pfit.force_thru)); @@ -13141,24 +13143,21 @@ static bool check_digital_port_conflicts(struct intel_atomic_state *state) encoder = to_intel_encoder(connector_state->best_encoder); - WARN_ON(!connector_state->crtc); + drm_WARN_ON(dev, !connector_state->crtc); switch (encoder->type) { - unsigned int port_mask; case INTEL_OUTPUT_DDI: - if (WARN_ON(!HAS_DDI(to_i915(dev)))) + if (drm_WARN_ON(dev, !HAS_DDI(to_i915(dev)))) break; /* else, fall through */ case INTEL_OUTPUT_DP: case INTEL_OUTPUT_HDMI: case INTEL_OUTPUT_EDP: - port_mask = 1 << encoder->port; - /* the same port mustn't appear more than once */ - if (used_ports & port_mask) + if (used_ports & BIT(encoder->port)) ret = false; - used_ports |= port_mask; + used_ports |= BIT(encoder->port); break; case INTEL_OUTPUT_DP_MST: used_mst_ports |= @@ -13197,7 +13196,8 @@ static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state { crtc_state->uapi.enable = crtc_state->hw.enable; crtc_state->uapi.active = crtc_state->hw.active; - WARN_ON(drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0); + drm_WARN_ON(crtc_state->uapi.crtc->dev, + drm_atomic_set_mode_for_crtc(&crtc_state->uapi, &crtc_state->hw.mode) < 0); crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode; @@ -13365,7 +13365,8 @@ encoder_retry: } if (ret == RETRY) { - if (WARN(!retry, "loop in pipe configuration computation\n")) + if (drm_WARN(&i915->drm, !retry, + "loop in pipe configuration computation\n")) return -EINVAL; drm_dbg_kms(&i915->drm, "CRTC bw constrained, retrying\n"); @@ -13489,6 +13490,13 @@ intel_compare_infoframe(const union hdmi_infoframe *a, return memcmp(a, b, sizeof(*a)) == 0; } +static bool +intel_compare_dp_vsc_sdp(const struct drm_dp_vsc_sdp *a, + const struct drm_dp_vsc_sdp *b) +{ + return memcmp(a, b, sizeof(*a)) == 0; +} + static void pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv, bool fastset, const char *name, @@ -13514,6 +13522,31 @@ pipe_config_infoframe_mismatch(struct drm_i915_private *dev_priv, } } +static void +pipe_config_dp_vsc_sdp_mismatch(struct drm_i915_private *dev_priv, + bool fastset, const char *name, + const struct drm_dp_vsc_sdp *a, + const struct drm_dp_vsc_sdp *b) +{ + if (fastset) { + if (!drm_debug_enabled(DRM_UT_KMS)) + return; + + drm_dbg_kms(&dev_priv->drm, + "fastset mismatch in %s dp sdp\n", name); + drm_dbg_kms(&dev_priv->drm, "expected:\n"); + drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, a); + drm_dbg_kms(&dev_priv->drm, "found:\n"); + drm_dp_vsc_sdp_log(KERN_DEBUG, dev_priv->drm.dev, b); + } else { + drm_err(&dev_priv->drm, "mismatch in %s dp sdp\n", name); + drm_err(&dev_priv->drm, "expected:\n"); + drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, a); + drm_err(&dev_priv->drm, "found:\n"); + drm_dp_vsc_sdp_log(KERN_ERR, dev_priv->drm.dev, b); + } +} + static void __printf(4, 5) pipe_config_mismatch(bool fastset, const struct intel_crtc *crtc, const char *name, const char *format, ...) @@ -13715,6 +13748,17 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, } \ } while (0) +#define PIPE_CONF_CHECK_DP_VSC_SDP(name) do { \ + if (!current_config->has_psr && !pipe_config->has_psr && \ + !intel_compare_dp_vsc_sdp(¤t_config->infoframes.name, \ + &pipe_config->infoframes.name)) { \ + pipe_config_dp_vsc_sdp_mismatch(dev_priv, fastset, __stringify(name), \ + ¤t_config->infoframes.name, \ + &pipe_config->infoframes.name); \ + ret = false; \ + } \ +} while (0) + #define PIPE_CONF_CHECK_COLOR_LUT(name1, name2, bit_precision) do { \ if (current_config->name1 != pipe_config->name1) { \ pipe_config_mismatch(fastset, crtc, __stringify(name1), \ @@ -13815,8 +13859,10 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_BOOL(pch_pfit.enabled); if (current_config->pch_pfit.enabled) { - PIPE_CONF_CHECK_X(pch_pfit.pos); - PIPE_CONF_CHECK_X(pch_pfit.size); + PIPE_CONF_CHECK_I(pch_pfit.dst.x1); + PIPE_CONF_CHECK_I(pch_pfit.dst.y1); + PIPE_CONF_CHECK_I(pch_pfit.dst.x2); + PIPE_CONF_CHECK_I(pch_pfit.dst.y2); } PIPE_CONF_CHECK_I(scaler_state.scaler_id); @@ -13890,6 +13936,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config, PIPE_CONF_CHECK_INFOFRAME(spd); PIPE_CONF_CHECK_INFOFRAME(hdmi); PIPE_CONF_CHECK_INFOFRAME(drm); + PIPE_CONF_CHECK_DP_VSC_SDP(vsc); PIPE_CONF_CHECK_X(sync_mode_slaves_mask); PIPE_CONF_CHECK_I(master_transcoder); @@ -13925,9 +13972,10 @@ static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv, * FDI already provided one idea for the dotclock. * Yell if the encoder disagrees. */ - WARN(!intel_fuzzy_clock_check(fdi_dotclock, dotclock), - "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n", - fdi_dotclock, dotclock); + drm_WARN(&dev_priv->drm, + !intel_fuzzy_clock_check(fdi_dotclock, dotclock), + "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n", + fdi_dotclock, dotclock); } } @@ -14269,11 +14317,11 @@ verify_single_dpll_state(struct drm_i915_private *dev_priv, if (new_crtc_state->hw.active) I915_STATE_WARN(!(pll->active_mask & crtc_mask), "pll active mismatch (expected pipe %c in active mask 0x%02x)\n", - pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask); + pipe_name(crtc->pipe), pll->active_mask); else I915_STATE_WARN(pll->active_mask & crtc_mask, "pll active mismatch (didn't expect pipe %c in active mask 0x%02x)\n", - pipe_name(drm_crtc_index(&crtc->base)), pll->active_mask); + pipe_name(crtc->pipe), pll->active_mask); I915_STATE_WARN(!(pll->state.crtc_mask & crtc_mask), "pll enabled crtcs mismatch (expected 0x%x in 0x%02x)\n", @@ -14302,10 +14350,10 @@ verify_shared_dpll_state(struct intel_crtc *crtc, I915_STATE_WARN(pll->active_mask & crtc_mask, "pll active mismatch (didn't expect pipe %c in active mask)\n", - pipe_name(drm_crtc_index(&crtc->base))); + pipe_name(crtc->pipe)); I915_STATE_WARN(pll->state.crtc_mask & crtc_mask, "pll enabled crtcs mismatch (found %x in enabled mask)\n", - pipe_name(drm_crtc_index(&crtc->base))); + pipe_name(crtc->pipe)); } } @@ -14329,8 +14377,10 @@ verify_disabled_dpll_state(struct drm_i915_private *dev_priv) { int i; - for (i = 0; i < dev_priv->num_shared_dpll; i++) - verify_single_dpll_state(dev_priv, &dev_priv->shared_dplls[i], NULL, NULL); + for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) + verify_single_dpll_state(dev_priv, + &dev_priv->dpll.shared_dplls[i], + NULL, NULL); } static void @@ -14713,8 +14763,8 @@ static int intel_atomic_check(struct drm_device *dev, /* Catch I915_MODE_FLAG_INHERITED */ for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { - if (new_crtc_state->hw.mode.private_flags != - old_crtc_state->hw.mode.private_flags) + if (new_crtc_state->uapi.mode.private_flags != + old_crtc_state->uapi.mode.private_flags) new_crtc_state->uapi.mode_changed = true; } @@ -14964,11 +15014,13 @@ static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state, } static void commit_pipe_config(struct intel_atomic_state *state, - struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state) + struct intel_crtc *crtc) { - struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); bool modeset = needs_modeset(new_crtc_state); /* @@ -14994,22 +15046,35 @@ static void commit_pipe_config(struct intel_atomic_state *state, dev_priv->display.atomic_update_watermarks(state, crtc); } -static void intel_update_crtc(struct intel_crtc *crtc, - struct intel_atomic_state *state, - struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state) +static void intel_enable_crtc(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - bool modeset = needs_modeset(new_crtc_state); + const struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (!needs_modeset(new_crtc_state)) + return; - if (modeset) { - intel_crtc_update_active_timings(new_crtc_state); + intel_crtc_update_active_timings(new_crtc_state); - dev_priv->display.crtc_enable(state, crtc); + dev_priv->display.crtc_enable(state, crtc); - /* vblanks work again, re-enable pipe CRC. */ - intel_crtc_enable_pipe_crc(crtc); - } else { + /* vblanks work again, re-enable pipe CRC. */ + intel_crtc_enable_pipe_crc(crtc); +} + +static void intel_update_crtc(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_crtc_state *old_crtc_state = + intel_atomic_get_old_crtc_state(state, crtc); + struct intel_crtc_state *new_crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + bool modeset = needs_modeset(new_crtc_state); + + if (!modeset) { if (new_crtc_state->preload_luts && (new_crtc_state->uapi.color_mgmt_changed || new_crtc_state->update_pipe)) @@ -15029,7 +15094,7 @@ static void intel_update_crtc(struct intel_crtc *crtc, /* Perform vblank evasion around commit operation */ intel_pipe_update_start(new_crtc_state); - commit_pipe_config(state, old_crtc_state, new_crtc_state); + commit_pipe_config(state, crtc); if (INTEL_GEN(dev_priv) >= 9) skl_update_planes_on_crtc(state, crtc); @@ -15049,17 +15114,6 @@ static void intel_update_crtc(struct intel_crtc *crtc, intel_crtc_arm_fifo_underrun(crtc, new_crtc_state); } -static struct intel_crtc *intel_get_slave_crtc(const struct intel_crtc_state *new_crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(new_crtc_state->uapi.crtc->dev); - enum transcoder slave_transcoder; - - WARN_ON(!is_power_of_2(new_crtc_state->sync_mode_slaves_mask)); - - slave_transcoder = ffs(new_crtc_state->sync_mode_slaves_mask) - 1; - return intel_get_crtc_for_pipe(dev_priv, - (enum pipe)slave_transcoder); -} static void intel_old_crtc_state_disables(struct intel_atomic_state *state, struct intel_crtc_state *old_crtc_state, @@ -15135,129 +15189,19 @@ static void intel_commit_modeset_disables(struct intel_atomic_state *state) static void intel_commit_modeset_enables(struct intel_atomic_state *state) { + struct intel_crtc_state *new_crtc_state; struct intel_crtc *crtc; - struct intel_crtc_state *old_crtc_state, *new_crtc_state; int i; - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) { + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { if (!new_crtc_state->hw.active) continue; - intel_update_crtc(crtc, state, old_crtc_state, - new_crtc_state); + intel_enable_crtc(state, crtc); + intel_update_crtc(state, crtc); } } -static void intel_crtc_enable_trans_port_sync(struct intel_crtc *crtc, - struct intel_atomic_state *state, - struct intel_crtc_state *new_crtc_state) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - - intel_crtc_update_active_timings(new_crtc_state); - dev_priv->display.crtc_enable(state, crtc); - intel_crtc_enable_pipe_crc(crtc); -} - -static void intel_set_dp_tp_ctl_normal(struct intel_crtc *crtc, - struct intel_atomic_state *state) -{ - struct drm_connector *uninitialized_var(conn); - struct drm_connector_state *conn_state; - struct intel_dp *intel_dp; - int i; - - for_each_new_connector_in_state(&state->base, conn, conn_state, i) { - if (conn_state->crtc == &crtc->base) - break; - } - intel_dp = intel_attached_dp(to_intel_connector(conn)); - intel_dp_stop_link_train(intel_dp); -} - -/* - * TODO: This is only called from port sync and it is identical to what will be - * executed again in intel_update_crtc() over port sync pipes - */ -static void intel_post_crtc_enable_updates(struct intel_crtc *crtc, - struct intel_atomic_state *state) -{ - struct intel_crtc_state *new_crtc_state = - intel_atomic_get_new_crtc_state(state, crtc); - struct intel_crtc_state *old_crtc_state = - intel_atomic_get_old_crtc_state(state, crtc); - bool modeset = needs_modeset(new_crtc_state); - - if (new_crtc_state->update_pipe && !new_crtc_state->enable_fbc) - intel_fbc_disable(crtc); - else - intel_fbc_enable(state, crtc); - - /* Perform vblank evasion around commit operation */ - intel_pipe_update_start(new_crtc_state); - commit_pipe_config(state, old_crtc_state, new_crtc_state); - skl_update_planes_on_crtc(state, crtc); - intel_pipe_update_end(new_crtc_state); - - /* - * We usually enable FIFO underrun interrupts as part of the - * CRTC enable sequence during modesets. But when we inherit a - * valid pipe configuration from the BIOS we need to take care - * of enabling them on the CRTC's first fastset. - */ - if (new_crtc_state->update_pipe && !modeset && - old_crtc_state->hw.mode.private_flags & I915_MODE_FLAG_INHERITED) - intel_crtc_arm_fifo_underrun(crtc, new_crtc_state); -} - -static void intel_update_trans_port_sync_crtcs(struct intel_crtc *crtc, - struct intel_atomic_state *state, - struct intel_crtc_state *old_crtc_state, - struct intel_crtc_state *new_crtc_state) -{ - struct drm_i915_private *i915 = to_i915(crtc->base.dev); - struct intel_crtc *slave_crtc = intel_get_slave_crtc(new_crtc_state); - struct intel_crtc_state *new_slave_crtc_state = - intel_atomic_get_new_crtc_state(state, slave_crtc); - struct intel_crtc_state *old_slave_crtc_state = - intel_atomic_get_old_crtc_state(state, slave_crtc); - - WARN_ON(!slave_crtc || !new_slave_crtc_state || - !old_slave_crtc_state); - - drm_dbg_kms(&i915->drm, - "Updating Transcoder Port Sync Master CRTC = %d %s and Slave CRTC %d %s\n", - crtc->base.base.id, crtc->base.name, - slave_crtc->base.base.id, slave_crtc->base.name); - - /* Enable seq for slave with with DP_TP_CTL left Idle until the - * master is ready - */ - intel_crtc_enable_trans_port_sync(slave_crtc, - state, - new_slave_crtc_state); - - /* Enable seq for master with with DP_TP_CTL left Idle */ - intel_crtc_enable_trans_port_sync(crtc, - state, - new_crtc_state); - - /* Set Slave's DP_TP_CTL to Normal */ - intel_set_dp_tp_ctl_normal(slave_crtc, - state); - - /* Set Master's DP_TP_CTL To Normal */ - usleep_range(200, 400); - intel_set_dp_tp_ctl_normal(crtc, - state); - - /* Now do the post crtc enable for all master and slaves */ - intel_post_crtc_enable_updates(slave_crtc, - state); - intel_post_crtc_enable_updates(crtc, - state); -} - static void icl_dbuf_slice_pre_update(struct intel_atomic_state *state) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); @@ -15287,7 +15231,6 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state) struct intel_crtc *crtc; struct intel_crtc_state *old_crtc_state, *new_crtc_state; struct skl_ddb_entry entries[I915_MAX_PIPES] = {}; - const u8 num_pipes = INTEL_NUM_PIPES(dev_priv); u8 update_pipes = 0, modeset_pipes = 0; int i; @@ -15324,14 +15267,13 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state) continue; if (skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb, - entries, num_pipes, pipe)) + entries, I915_MAX_PIPES, pipe)) continue; entries[pipe] = new_crtc_state->wm.skl.ddb; update_pipes &= ~BIT(pipe); - intel_update_crtc(crtc, state, old_crtc_state, - new_crtc_state); + intel_update_crtc(state, crtc); /* * If this is an already active pipe, it's DDB changed, @@ -15346,67 +15288,62 @@ static void skl_commit_modeset_enables(struct intel_atomic_state *state) } } + update_pipes = modeset_pipes; + /* * Enable all pipes that needs a modeset and do not depends on other * pipes */ - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { enum pipe pipe = crtc->pipe; if ((modeset_pipes & BIT(pipe)) == 0) continue; if (intel_dp_mst_is_slave_trans(new_crtc_state) || - is_trans_port_sync_slave(new_crtc_state)) + is_trans_port_sync_master(new_crtc_state)) continue; - WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb, - entries, num_pipes, pipe)); - - entries[pipe] = new_crtc_state->wm.skl.ddb; modeset_pipes &= ~BIT(pipe); - if (is_trans_port_sync_mode(new_crtc_state)) { - struct intel_crtc *slave_crtc; + intel_enable_crtc(state, crtc); + } - intel_update_trans_port_sync_crtcs(crtc, state, - old_crtc_state, - new_crtc_state); + /* + * Then we enable all remaining pipes that depend on other + * pipes: MST slaves and port sync masters. + */ + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { + enum pipe pipe = crtc->pipe; + + if ((modeset_pipes & BIT(pipe)) == 0) + continue; - slave_crtc = intel_get_slave_crtc(new_crtc_state); - /* TODO: update entries[] of slave */ - modeset_pipes &= ~BIT(slave_crtc->pipe); + modeset_pipes &= ~BIT(pipe); - } else { - intel_update_crtc(crtc, state, old_crtc_state, - new_crtc_state); - } + intel_enable_crtc(state, crtc); } /* - * Finally enable all pipes that needs a modeset and depends on - * other pipes, right now it is only MST slaves as both port sync slave - * and master are enabled together + * Finally we do the plane updates/etc. for all pipes that got enabled. */ - for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, - new_crtc_state, i) { + for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { enum pipe pipe = crtc->pipe; - if ((modeset_pipes & BIT(pipe)) == 0) + if ((update_pipes & BIT(pipe)) == 0) continue; - WARN_ON(skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb, - entries, num_pipes, pipe)); + drm_WARN_ON(&dev_priv->drm, skl_ddb_allocation_overlaps(&new_crtc_state->wm.skl.ddb, + entries, I915_MAX_PIPES, pipe)); entries[pipe] = new_crtc_state->wm.skl.ddb; - modeset_pipes &= ~BIT(pipe); + update_pipes &= ~BIT(pipe); - intel_update_crtc(crtc, state, old_crtc_state, new_crtc_state); + intel_update_crtc(state, crtc); } - WARN_ON(modeset_pipes); - + drm_WARN_ON(&dev_priv->drm, modeset_pipes); + drm_WARN_ON(&dev_priv->drm, update_pipes); } static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv) @@ -15505,16 +15442,11 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) intel_set_cdclk_pre_plane_update(state); - /* - * SKL workaround: bspec recommends we disable the SAGV when we - * have more then one pipe enabled - */ - if (!intel_can_enable_sagv(state)) - intel_disable_sagv(dev_priv); - intel_modeset_verify_disabled(dev_priv, state); } + intel_sagv_pre_plane_update(state); + /* Complete the events for pipes that have now been disabled */ for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) { bool modeset = needs_modeset(new_crtc_state); @@ -15610,8 +15542,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state) if (state->modeset) intel_verify_planes(state); - if (state->modeset && intel_can_enable_sagv(state)) - intel_enable_sagv(dev_priv); + intel_sagv_post_plane_update(state); drm_atomic_helper_commit_hw_done(&state->base); @@ -15947,7 +15878,7 @@ intel_prepare_plane_fb(struct drm_plane *_plane, if (new_plane_state->uapi.fence) { /* explicit fencing */ ret = i915_sw_fence_await_dma_fence(&state->commit_ready, new_plane_state->uapi.fence, - I915_FENCE_TIMEOUT, + i915_fence_timeout(dev_priv), GFP_KERNEL); if (ret < 0) return ret; @@ -15974,7 +15905,8 @@ intel_prepare_plane_fb(struct drm_plane *_plane, ret = i915_sw_fence_await_reservation(&state->commit_ready, obj->base.resv, NULL, - false, I915_FENCE_TIMEOUT, + false, + i915_fence_timeout(dev_priv), GFP_KERNEL); if (ret < 0) goto unpin_fb; @@ -16289,7 +16221,6 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) struct intel_plane *plane; const struct drm_plane_funcs *plane_funcs; unsigned int supported_rotations; - unsigned int possible_crtcs; const u32 *formats; int num_formats; int ret, zpos; @@ -16370,18 +16301,16 @@ intel_primary_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) plane->get_hw_state = i9xx_plane_get_hw_state; plane->check_plane = i9xx_plane_check; - possible_crtcs = BIT(pipe); - if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, - possible_crtcs, plane_funcs, + 0, plane_funcs, formats, num_formats, i9xx_format_modifiers, DRM_PLANE_TYPE_PRIMARY, "primary %c", pipe_name(pipe)); else ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, - possible_crtcs, plane_funcs, + 0, plane_funcs, formats, num_formats, i9xx_format_modifiers, DRM_PLANE_TYPE_PRIMARY, @@ -16423,7 +16352,6 @@ static struct intel_plane * intel_cursor_plane_create(struct drm_i915_private *dev_priv, enum pipe pipe) { - unsigned int possible_crtcs; struct intel_plane *cursor; int ret, zpos; @@ -16456,10 +16384,8 @@ intel_cursor_plane_create(struct drm_i915_private *dev_priv, if (IS_I845G(dev_priv) || IS_I865G(dev_priv) || HAS_CUR_FBC(dev_priv)) cursor->cursor.size = ~0; - possible_crtcs = BIT(pipe); - ret = drm_universal_plane_init(&dev_priv->drm, &cursor->base, - possible_crtcs, &intel_cursor_plane_funcs, + 0, &intel_cursor_plane_funcs, intel_cursor_formats, ARRAY_SIZE(intel_cursor_formats), cursor_format_modifiers, @@ -16504,6 +16430,7 @@ static const struct drm_crtc_funcs bdw_crtc_funcs = { .get_vblank_counter = g4x_get_vblank_counter, .enable_vblank = bdw_enable_vblank, .disable_vblank = bdw_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static const struct drm_crtc_funcs ilk_crtc_funcs = { @@ -16512,6 +16439,7 @@ static const struct drm_crtc_funcs ilk_crtc_funcs = { .get_vblank_counter = g4x_get_vblank_counter, .enable_vblank = ilk_enable_vblank, .disable_vblank = ilk_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static const struct drm_crtc_funcs g4x_crtc_funcs = { @@ -16520,6 +16448,7 @@ static const struct drm_crtc_funcs g4x_crtc_funcs = { .get_vblank_counter = g4x_get_vblank_counter, .enable_vblank = i965_enable_vblank, .disable_vblank = i965_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static const struct drm_crtc_funcs i965_crtc_funcs = { @@ -16528,6 +16457,7 @@ static const struct drm_crtc_funcs i965_crtc_funcs = { .get_vblank_counter = i915_get_vblank_counter, .enable_vblank = i965_enable_vblank, .disable_vblank = i965_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static const struct drm_crtc_funcs i915gm_crtc_funcs = { @@ -16536,6 +16466,7 @@ static const struct drm_crtc_funcs i915gm_crtc_funcs = { .get_vblank_counter = i915_get_vblank_counter, .enable_vblank = i915gm_enable_vblank, .disable_vblank = i915gm_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static const struct drm_crtc_funcs i915_crtc_funcs = { @@ -16544,6 +16475,7 @@ static const struct drm_crtc_funcs i915_crtc_funcs = { .get_vblank_counter = i915_get_vblank_counter, .enable_vblank = i8xx_enable_vblank, .disable_vblank = i8xx_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static const struct drm_crtc_funcs i8xx_crtc_funcs = { @@ -16552,6 +16484,7 @@ static const struct drm_crtc_funcs i8xx_crtc_funcs = { /* no hw vblank counter */ .enable_vblank = i8xx_enable_vblank, .disable_vblank = i8xx_disable_vblank, + .get_vblank_timestamp = intel_crtc_get_vblank_timestamp, }; static struct intel_crtc *intel_crtc_alloc(void) @@ -16581,6 +16514,18 @@ static void intel_crtc_free(struct intel_crtc *crtc) kfree(crtc); } +static void intel_plane_possible_crtcs_init(struct drm_i915_private *dev_priv) +{ + struct intel_plane *plane; + + for_each_intel_plane(&dev_priv->drm, plane) { + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, + plane->pipe); + + plane->base.possible_crtcs = drm_crtc_mask(&crtc->base); + } +} + static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) { struct intel_plane *primary, *cursor; @@ -16659,7 +16604,9 @@ static int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe) intel_color_init(crtc); - WARN_ON(drm_crtc_index(&crtc->base) != crtc->pipe); + intel_crtc_crc_init(crtc); + + drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe); return 0; @@ -17526,7 +17473,7 @@ static void sanitize_watermarks(struct drm_i915_private *dev_priv) return; state = drm_atomic_state_alloc(&dev_priv->drm); - if (WARN_ON(!state)) + if (drm_WARN_ON(&dev_priv->drm, !state)) return; intel_state = to_intel_atomic_state(state); @@ -17578,7 +17525,8 @@ fail: * If this actually happens, we'll have to just leave the * BIOS-programmed watermarks untouched and hope for the best. */ - WARN(ret, "Could not determine valid watermarks for inherited state\n"); + drm_WARN(&dev_priv->drm, ret, + "Could not determine valid watermarks for inherited state\n"); drm_atomic_state_put(state); @@ -17746,11 +17694,9 @@ static void plane_config_fini(struct intel_initial_plane_config *plane_config) i915_vma_put(plane_config->vma); } -int intel_modeset_init(struct drm_i915_private *i915) +/* part #1: call before irq install */ +int intel_modeset_init_noirq(struct drm_i915_private *i915) { - struct drm_device *dev = &i915->drm; - enum pipe pipe; - struct intel_crtc *crtc; int ret; i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0); @@ -17775,6 +17721,17 @@ int intel_modeset_init(struct drm_i915_private *i915) intel_fbc_init(i915); + return 0; +} + +/* part #2: call after irq install */ +int intel_modeset_init(struct drm_i915_private *i915) +{ + struct drm_device *dev = &i915->drm; + enum pipe pipe; + struct intel_crtc *crtc; + int ret; + intel_init_pm(i915); intel_panel_sanitize_ssc(i915); @@ -17795,6 +17752,7 @@ int intel_modeset_init(struct drm_i915_private *i915) } } + intel_plane_possible_crtcs_init(i915); intel_shared_dpll_init(dev); intel_update_fdi_pll_freq(i915); @@ -17873,7 +17831,8 @@ void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) u32 dpll, fp; int i; - WARN_ON(i9xx_calc_dpll_params(48000, &clock) != 25154); + drm_WARN_ON(&dev_priv->drm, + i9xx_calc_dpll_params(48000, &clock) != 25154); drm_dbg_kms(&dev_priv->drm, "enabling pipe %c due to force quirk (vco=%d dot=%d)\n", @@ -17938,11 +17897,19 @@ void i830_disable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) drm_dbg_kms(&dev_priv->drm, "disabling pipe %c due to force quirk\n", pipe_name(pipe)); - WARN_ON(intel_de_read(dev_priv, DSPCNTR(PLANE_A)) & DISPLAY_PLANE_ENABLE); - WARN_ON(intel_de_read(dev_priv, DSPCNTR(PLANE_B)) & DISPLAY_PLANE_ENABLE); - WARN_ON(intel_de_read(dev_priv, DSPCNTR(PLANE_C)) & DISPLAY_PLANE_ENABLE); - WARN_ON(intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE); - WARN_ON(intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, DSPCNTR(PLANE_A)) & + DISPLAY_PLANE_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, DSPCNTR(PLANE_B)) & + DISPLAY_PLANE_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, DSPCNTR(PLANE_C)) & + DISPLAY_PLANE_ENABLE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, CURCNTR(PIPE_A)) & MCURSOR_MODE); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, CURCNTR(PIPE_B)) & MCURSOR_MODE); intel_de_write(dev_priv, PIPECONF(pipe), 0); intel_de_posting_read(dev_priv, PIPECONF(pipe)); @@ -18191,11 +18158,12 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder) best_encoder = connector->base.state->best_encoder; connector->base.state->best_encoder = &encoder->base; + /* FIXME NULL atomic state passed! */ if (encoder->disable) - encoder->disable(encoder, crtc_state, + encoder->disable(NULL, encoder, crtc_state, connector->base.state); if (encoder->post_disable) - encoder->post_disable(encoder, crtc_state, + encoder->post_disable(NULL, encoder, crtc_state, connector->base.state); connector->base.state->best_encoder = best_encoder; @@ -18263,7 +18231,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) struct intel_connector *connector; struct drm_connector_list_iter conn_iter; u8 active_pipes = 0; - int i; for_each_intel_crtc(dev, crtc) { struct intel_crtc_state *crtc_state = @@ -18292,33 +18259,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) readout_plane_state(dev_priv); - for (i = 0; i < dev_priv->num_shared_dpll; i++) { - struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; - - pll->on = pll->info->funcs->get_hw_state(dev_priv, pll, - &pll->state.hw_state); - - if (IS_ELKHARTLAKE(dev_priv) && pll->on && - pll->info->id == DPLL_ID_EHL_DPLL4) { - pll->wakeref = intel_display_power_get(dev_priv, - POWER_DOMAIN_DPLL_DC_OFF); - } - - pll->state.crtc_mask = 0; - for_each_intel_crtc(dev, crtc) { - struct intel_crtc_state *crtc_state = - to_intel_crtc_state(crtc->base.state); - - if (crtc_state->hw.active && - crtc_state->shared_dpll == pll) - pll->state.crtc_mask |= 1 << crtc->pipe; - } - pll->active_mask = pll->state.crtc_mask; - - drm_dbg_kms(&dev_priv->drm, - "%s hw state readout: crtc_mask 0x%08x, on %i\n", - pll->info->name, pll->state.crtc_mask, pll->on); - } + intel_dpll_readout_hw_state(dev_priv); for_each_intel_encoder(dev, encoder) { pipe = 0; @@ -18446,7 +18387,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev) if (crtc_state->hw.active) { min_cdclk = intel_crtc_compute_min_cdclk(crtc_state); - if (WARN_ON(min_cdclk < 0)) + if (drm_WARN_ON(dev, min_cdclk < 0)) min_cdclk = 0; } @@ -18575,7 +18516,6 @@ intel_modeset_setup_hw_state(struct drm_device *dev, struct intel_encoder *encoder; struct intel_crtc *crtc; intel_wakeref_t wakeref; - int i; wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_INIT); @@ -18628,19 +18568,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev, intel_modeset_update_connector_atomic_state(dev); - for (i = 0; i < dev_priv->num_shared_dpll; i++) { - struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; - - if (!pll->on || pll->active_mask) - continue; - - drm_dbg_kms(&dev_priv->drm, - "%s enabled but not in use, disabling\n", - pll->info->name); - - pll->info->funcs->disable(dev_priv, pll); - pll->on = false; - } + intel_dpll_sanitize_state(dev_priv); if (IS_G4X(dev_priv)) { g4x_wm_get_hw_state(dev_priv); @@ -18660,7 +18588,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev, u64 put_domains; put_domains = modeset_get_crtc_power_domains(crtc_state); - if (WARN_ON(put_domains)) + if (drm_WARN_ON(dev, put_domains)) modeset_put_power_domains(dev_priv, put_domains); } @@ -18727,7 +18655,7 @@ void intel_modeset_driver_remove(struct drm_i915_private *i915) flush_workqueue(i915->modeset_wq); flush_work(&i915->atomic_helper.free_work); - WARN_ON(!llist_empty(&i915->atomic_helper.free_list)); + drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list)); } /* part #2: call after irq uninstall */ @@ -18880,7 +18808,7 @@ intel_display_capture_error_state(struct drm_i915_private *dev_priv) for (i = 0; i < ARRAY_SIZE(error->transcoder); i++) { enum transcoder cpu_transcoder = transcoders[i]; - if (!INTEL_INFO(dev_priv)->trans_offsets[cpu_transcoder]) + if (!HAS_TRANSCODER(dev_priv, cpu_transcoder)) continue; error->transcoder[i].available = true; diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index f92efbbec838..efb4da205ea2 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -26,7 +26,6 @@ #define _INTEL_DISPLAY_H_ #include <drm/drm_util.h> -#include <drm/i915_drm.h> enum link_m_n_set; struct dpll; @@ -40,6 +39,7 @@ struct drm_framebuffer; struct drm_i915_error_state_buf; struct drm_i915_gem_object; struct drm_i915_private; +struct drm_mode_fb_cmd2; struct drm_modeset_acquire_ctx; struct drm_plane; struct drm_plane_state; @@ -47,6 +47,7 @@ struct i915_ggtt_view; struct intel_atomic_state; struct intel_crtc; struct intel_crtc_state; +struct intel_crtc_state; struct intel_digital_port; struct intel_dp; struct intel_encoder; @@ -55,7 +56,6 @@ struct intel_plane; struct intel_plane_state; struct intel_remapped_info; struct intel_rotation_info; -struct intel_crtc_state; enum i915_gpio { GPIOA, @@ -313,15 +313,20 @@ enum phy_fia { }; #define for_each_pipe(__dev_priv, __p) \ - for ((__p) = 0; (__p) < INTEL_NUM_PIPES(__dev_priv); (__p)++) + for ((__p) = 0; (__p) < I915_MAX_PIPES; (__p)++) \ + for_each_if(INTEL_INFO(__dev_priv)->pipe_mask & BIT(__p)) #define for_each_pipe_masked(__dev_priv, __p, __mask) \ - for ((__p) = 0; (__p) < INTEL_NUM_PIPES(__dev_priv); (__p)++) \ + for_each_pipe(__dev_priv, __p) \ for_each_if((__mask) & BIT(__p)) -#define for_each_cpu_transcoder_masked(__dev_priv, __t, __mask) \ +#define for_each_cpu_transcoder(__dev_priv, __t) \ for ((__t) = 0; (__t) < I915_MAX_TRANSCODERS; (__t)++) \ - for_each_if ((__mask) & (1 << (__t))) + for_each_if (INTEL_INFO(__dev_priv)->cpu_transcoder_mask & BIT(__t)) + +#define for_each_cpu_transcoder_masked(__dev_priv, __t, __mask) \ + for_each_cpu_transcoder(__dev_priv, __t) \ + for_each_if ((__mask) & BIT(__t)) #define for_each_universal_plane(__dev_priv, __pipe, __p) \ for ((__p) = 0; \ @@ -578,13 +583,14 @@ void hsw_disable_ips(const struct intel_crtc_state *crtc_state); enum intel_display_power_domain intel_port_to_power_domain(enum port port); enum intel_display_power_domain intel_aux_power_domain(struct intel_digital_port *dig_port); +enum intel_display_power_domain +intel_legacy_aux_to_power_domain(enum aux_ch aux_ch); void intel_mode_from_pipe_config(struct drm_display_mode *mode, struct intel_crtc_state *pipe_config); void intel_crtc_arm_fifo_underrun(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state); u16 skl_scaler_calc_phase(int sub, int scale, bool chroma_center); -int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state); void skl_scaler_disable(const struct intel_crtc_state *old_crtc_state); void ilk_pfit_disable(const struct intel_crtc_state *old_crtc_state); u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state, @@ -614,6 +620,7 @@ intel_format_info_is_yuv_semiplanar(const struct drm_format_info *info, /* modesetting */ void intel_modeset_init_hw(struct drm_i915_private *i915); +int intel_modeset_init_noirq(struct drm_i915_private *i915); int intel_modeset_init(struct drm_i915_private *i915); void intel_modeset_driver_remove(struct drm_i915_private *i915); void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915); diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index 46954cc7b6c0..3d9dc27478b3 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -9,6 +9,7 @@ #include "i915_debugfs.h" #include "intel_csr.h" #include "intel_display_debugfs.h" +#include "intel_display_power.h" #include "intel_display_types.h" #include "intel_dp.h" #include "intel_fbc.h" @@ -920,8 +921,13 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused) int i; drm_modeset_lock_all(dev); - for (i = 0; i < dev_priv->num_shared_dpll; i++) { - struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; + + seq_printf(m, "PLL refclks: non-SSC: %d kHz, SSC: %d kHz\n", + dev_priv->dpll.ref_clks.nssc, + dev_priv->dpll.ref_clks.ssc); + + for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { + struct intel_shared_dpll *pll = &dev_priv->dpll.shared_dplls[i]; seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->info->name, pll->info->id); @@ -1144,6 +1150,51 @@ static int i915_drrs_status(struct seq_file *m, void *unused) return 0; } +#define LPSP_STATUS(COND) (COND ? seq_puts(m, "LPSP: enabled\n") : \ + seq_puts(m, "LPSP: disabled\n")) + +static bool +intel_lpsp_power_well_enabled(struct drm_i915_private *i915, + enum i915_power_well_id power_well_id) +{ + intel_wakeref_t wakeref; + bool is_enabled; + + wakeref = intel_runtime_pm_get(&i915->runtime_pm); + is_enabled = intel_display_power_well_is_enabled(i915, + power_well_id); + intel_runtime_pm_put(&i915->runtime_pm, wakeref); + + return is_enabled; +} + +static int i915_lpsp_status(struct seq_file *m, void *unused) +{ + struct drm_i915_private *i915 = node_to_i915(m->private); + + switch (INTEL_GEN(i915)) { + case 12: + case 11: + LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, ICL_DISP_PW_3)); + break; + case 10: + case 9: + LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, SKL_DISP_PW_2)); + break; + default: + /* + * Apart from HASWELL/BROADWELL other legacy platform doesn't + * support lpsp. + */ + if (IS_HASWELL(i915) || IS_BROADWELL(i915)) + LPSP_STATUS(!intel_lpsp_power_well_enabled(i915, HSW_DISP_PW_GLOBAL)); + else + seq_puts(m, "LPSP: not supported\n"); + } + + return 0; +} + static int i915_dp_mst_info(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -1321,6 +1372,16 @@ static int i915_displayport_test_data_show(struct seq_file *m, void *data) intel_dp->compliance.test_data.vdisplay); seq_printf(m, "bpc: %u\n", intel_dp->compliance.test_data.bpc); + } else if (intel_dp->compliance.test_type == + DP_TEST_LINK_PHY_TEST_PATTERN) { + seq_printf(m, "pattern: %d\n", + intel_dp->compliance.test_data.phytest.phy_pattern); + seq_printf(m, "Number of lanes: %d\n", + intel_dp->compliance.test_data.phytest.num_lanes); + seq_printf(m, "Link Rate: %d\n", + intel_dp->compliance.test_data.phytest.link_rate); + seq_printf(m, "level: %02x\n", + intel_dp->train_set[0]); } } else seq_puts(m, "0"); @@ -1353,7 +1414,7 @@ static int i915_displayport_test_type_show(struct seq_file *m, void *data) if (encoder && connector->status == connector_status_connected) { intel_dp = enc_to_intel_dp(encoder); - seq_printf(m, "%02lx", intel_dp->compliance.test_type); + seq_printf(m, "%02lx\n", intel_dp->compliance.test_type); } else seq_puts(m, "0"); } @@ -1901,6 +1962,7 @@ static const struct drm_info_list intel_display_debugfs_list[] = { {"i915_dp_mst_info", i915_dp_mst_info, 0}, {"i915_ddb_info", i915_ddb_info, 0}, {"i915_drrs_status", i915_drrs_status, 0}, + {"i915_lpsp_status", i915_lpsp_status, 0}, }; static const struct { @@ -1982,6 +2044,48 @@ static int i915_hdcp_sink_capability_show(struct seq_file *m, void *data) } DEFINE_SHOW_ATTRIBUTE(i915_hdcp_sink_capability); +#define LPSP_CAPABLE(COND) (COND ? seq_puts(m, "LPSP: capable\n") : \ + seq_puts(m, "LPSP: incapable\n")) + +static int i915_lpsp_capability_show(struct seq_file *m, void *data) +{ + struct drm_connector *connector = m->private; + struct intel_encoder *encoder = + intel_attached_encoder(to_intel_connector(connector)); + struct drm_i915_private *i915 = to_i915(connector->dev); + + if (connector->status != connector_status_connected) + return -ENODEV; + + switch (INTEL_GEN(i915)) { + case 12: + /* + * Actually TGL can drive LPSP on port till DDI_C + * but there is no physical connected DDI_C on TGL sku's, + * even driver is not initilizing DDI_C port for gen12. + */ + LPSP_CAPABLE(encoder->port <= PORT_B); + break; + case 11: + LPSP_CAPABLE(connector->connector_type == DRM_MODE_CONNECTOR_DSI || + connector->connector_type == DRM_MODE_CONNECTOR_eDP); + break; + case 10: + case 9: + LPSP_CAPABLE(encoder->port == PORT_A && + (connector->connector_type == DRM_MODE_CONNECTOR_DSI || + connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort)); + break; + default: + if (IS_HASWELL(i915) || IS_BROADWELL(i915)) + LPSP_CAPABLE(connector->connector_type == DRM_MODE_CONNECTOR_eDP); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(i915_lpsp_capability); + static int i915_dsc_fec_support_show(struct seq_file *m, void *data) { struct drm_connector *connector = m->private; @@ -2125,5 +2229,16 @@ int intel_connector_debugfs_add(struct drm_connector *connector) debugfs_create_file("i915_dsc_fec_support", S_IRUGO, root, connector, &i915_dsc_fec_support_fops); + /* Legacy panels doesn't lpsp on any platform */ + if ((INTEL_GEN(dev_priv) >= 9 || IS_HASWELL(dev_priv) || + IS_BROADWELL(dev_priv)) && + (connector->connector_type == DRM_MODE_CONNECTOR_DSI || + connector->connector_type == DRM_MODE_CONNECTOR_eDP || + connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIA || + connector->connector_type == DRM_MODE_CONNECTOR_HDMIB)) + debugfs_create_file("i915_lpsp_capability", 0444, root, + connector, &i915_lpsp_capability_fops); + return 0; } diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c index 722399fc2ace..49998906cc61 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.c +++ b/drivers/gpu/drm/i915/display/intel_display_power.c @@ -151,6 +151,8 @@ intel_display_power_domain_str(enum intel_display_power_domain domain) return "GT_IRQ"; case POWER_DOMAIN_DPLL_DC_OFF: return "DPLL_DC_OFF"; + case POWER_DOMAIN_TC_COLD_OFF: + return "TC_COLD_OFF"; default: MISSING_CASE(domain); return "?"; @@ -183,8 +185,9 @@ static void intel_power_well_get(struct drm_i915_private *dev_priv, static void intel_power_well_put(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - WARN(!power_well->count, "Use count on power well %s is already zero", - power_well->desc->name); + drm_WARN(&dev_priv->drm, !power_well->count, + "Use count on power well %s is already zero", + power_well->desc->name); if (!--power_well->count) intel_power_well_disable(dev_priv, power_well); @@ -281,8 +284,51 @@ static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv, gen8_irq_power_well_pre_disable(dev_priv, irq_pipe_mask); } +#define ICL_AUX_PW_TO_CH(pw_idx) \ + ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A) + +#define ICL_TBT_AUX_PW_TO_CH(pw_idx) \ + ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C) + +static enum aux_ch icl_tc_phy_aux_ch(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + int pw_idx = power_well->desc->hsw.idx; + + return power_well->desc->hsw.is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) : + ICL_AUX_PW_TO_CH(pw_idx); +} + +static struct intel_digital_port * +aux_ch_to_digital_port(struct drm_i915_private *dev_priv, + enum aux_ch aux_ch) +{ + struct intel_digital_port *dig_port = NULL; + struct intel_encoder *encoder; + + for_each_intel_encoder(&dev_priv->drm, encoder) { + /* We'll check the MST primary port */ + if (encoder->type == INTEL_OUTPUT_DP_MST) + continue; + + dig_port = enc_to_dig_port(encoder); + if (!dig_port) + continue; + + if (dig_port->aux_ch != aux_ch) { + dig_port = NULL; + continue; + } + + break; + } + + return dig_port; +} + static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) + struct i915_power_well *power_well, + bool timeout_expected) { const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; int pw_idx = power_well->desc->hsw.idx; @@ -293,8 +339,8 @@ static void hsw_wait_for_power_well_enable(struct drm_i915_private *dev_priv, drm_dbg_kms(&dev_priv->drm, "%s power well enable timeout\n", power_well->desc->name); - /* An AUX timeout is expected if the TBT DP tunnel is down. */ - WARN_ON(!power_well->desc->hsw.is_tc_tbt); + drm_WARN_ON(&dev_priv->drm, !timeout_expected); + } } @@ -347,8 +393,9 @@ static void gen9_wait_for_power_well_fuses(struct drm_i915_private *dev_priv, enum skl_power_gate pg) { /* Timeout 5us for PG#0, for other PGs 1us */ - WARN_ON(intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS, - SKL_FUSE_PG_DIST_STATUS(pg), 1)); + drm_WARN_ON(&dev_priv->drm, + intel_de_wait_for_set(dev_priv, SKL_FUSE_STATUS, + SKL_FUSE_PG_DIST_STATUS(pg), 1)); } static void hsw_power_well_enable(struct drm_i915_private *dev_priv, @@ -356,11 +403,11 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv, { const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; int pw_idx = power_well->desc->hsw.idx; - bool wait_fuses = power_well->desc->hsw.has_fuses; - enum skl_power_gate uninitialized_var(pg); u32 val; - if (wait_fuses) { + if (power_well->desc->hsw.has_fuses) { + enum skl_power_gate pg; + pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) : SKL_PW_CTL_IDX_TO_PG(pw_idx); /* @@ -377,19 +424,27 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv, val = intel_de_read(dev_priv, regs->driver); intel_de_write(dev_priv, regs->driver, val | HSW_PWR_WELL_CTL_REQ(pw_idx)); - hsw_wait_for_power_well_enable(dev_priv, power_well); + + hsw_wait_for_power_well_enable(dev_priv, power_well, false); /* Display WA #1178: cnl */ if (IS_CANNONLAKE(dev_priv) && pw_idx >= GLK_PW_CTL_IDX_AUX_B && pw_idx <= CNL_PW_CTL_IDX_AUX_F) { + u32 val; + val = intel_de_read(dev_priv, CNL_AUX_ANAOVRD1(pw_idx)); val |= CNL_AUX_ANAOVRD1_ENABLE | CNL_AUX_ANAOVRD1_LDO_BYPASS; intel_de_write(dev_priv, CNL_AUX_ANAOVRD1(pw_idx), val); } - if (wait_fuses) + if (power_well->desc->hsw.has_fuses) { + enum skl_power_gate pg; + + pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_CTL_IDX_TO_PG(pw_idx) : + SKL_PW_CTL_IDX_TO_PG(pw_idx); gen9_wait_for_power_well_fuses(dev_priv, pg); + } hsw_power_well_post_enable(dev_priv, power_well->desc->hsw.irq_pipe_mask, @@ -423,7 +478,7 @@ icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); u32 val; - WARN_ON(!IS_ICELAKE(dev_priv)); + drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv)); val = intel_de_read(dev_priv, regs->driver); intel_de_write(dev_priv, regs->driver, @@ -435,7 +490,7 @@ icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, val | ICL_LANE_ENABLE_AUX); } - hsw_wait_for_power_well_enable(dev_priv, power_well); + hsw_wait_for_power_well_enable(dev_priv, power_well, false); /* Display WA #1178: icl */ if (pw_idx >= ICL_PW_CTL_IDX_AUX_A && pw_idx <= ICL_PW_CTL_IDX_AUX_B && @@ -455,7 +510,7 @@ icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv, enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); u32 val; - WARN_ON(!IS_ICELAKE(dev_priv)); + drm_WARN_ON(&dev_priv->drm, !IS_ICELAKE(dev_priv)); val = intel_de_read(dev_priv, ICL_PORT_CL_DW12(phy)); intel_de_write(dev_priv, ICL_PORT_CL_DW12(phy), @@ -468,21 +523,6 @@ icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv, hsw_wait_for_power_well_disable(dev_priv, power_well); } -#define ICL_AUX_PW_TO_CH(pw_idx) \ - ((pw_idx) - ICL_PW_CTL_IDX_AUX_A + AUX_CH_A) - -#define ICL_TBT_AUX_PW_TO_CH(pw_idx) \ - ((pw_idx) - ICL_PW_CTL_IDX_AUX_TBT1 + AUX_CH_C) - -static enum aux_ch icl_tc_phy_aux_ch(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) -{ - int pw_idx = power_well->desc->hsw.idx; - - return power_well->desc->hsw.is_tc_tbt ? ICL_TBT_AUX_PW_TO_CH(pw_idx) : - ICL_AUX_PW_TO_CH(pw_idx); -} - #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) static u64 async_put_domains_mask(struct i915_power_domains *power_domains); @@ -493,57 +533,34 @@ static int power_well_async_ref_count(struct drm_i915_private *dev_priv, int refs = hweight64(power_well->desc->domains & async_put_domains_mask(&dev_priv->power_domains)); - WARN_ON(refs > power_well->count); + drm_WARN_ON(&dev_priv->drm, refs > power_well->count); return refs; } static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) + struct i915_power_well *power_well, + struct intel_digital_port *dig_port) { - enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well); - struct intel_digital_port *dig_port = NULL; - struct intel_encoder *encoder; - /* Bypass the check if all references are released asynchronously */ if (power_well_async_ref_count(dev_priv, power_well) == power_well->count) return; - aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well); - - for_each_intel_encoder(&dev_priv->drm, encoder) { - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - - if (!intel_phy_is_tc(dev_priv, phy)) - continue; - - /* We'll check the MST primary port */ - if (encoder->type == INTEL_OUTPUT_DP_MST) - continue; - - dig_port = enc_to_dig_port(encoder); - if (WARN_ON(!dig_port)) - continue; - - if (dig_port->aux_ch != aux_ch) { - dig_port = NULL; - continue; - } - - break; - } + if (drm_WARN_ON(&dev_priv->drm, !dig_port)) + return; - if (WARN_ON(!dig_port)) + if (INTEL_GEN(dev_priv) == 11 && dig_port->tc_legacy_port) return; - WARN_ON(!intel_tc_port_ref_held(dig_port)); + drm_WARN_ON(&dev_priv->drm, !intel_tc_port_ref_held(dig_port)); } #else static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv, - struct i915_power_well *power_well) + struct i915_power_well *power_well, + struct intel_digital_port *dig_port) { } @@ -551,24 +568,65 @@ static void icl_tc_port_assert_ref_held(struct drm_i915_private *dev_priv, #define TGL_AUX_PW_TO_TC_PORT(pw_idx) ((pw_idx) - TGL_PW_CTL_IDX_AUX_TC1) +static void icl_tc_cold_exit(struct drm_i915_private *i915) +{ + int ret, tries = 0; + + while (1) { + ret = sandybridge_pcode_write_timeout(i915, + ICL_PCODE_EXIT_TCCOLD, + 0, 250, 1); + if (ret != -EAGAIN || ++tries == 3) + break; + msleep(1); + } + + /* Spec states that TC cold exit can take up to 1ms to complete */ + if (!ret) + msleep(1); + + /* TODO: turn failure into a error as soon i915 CI updates ICL IFWI */ + drm_dbg_kms(&i915->drm, "TC cold block %s\n", ret ? "failed" : + "succeeded"); +} + static void icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well); + struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch); + const struct i915_power_well_regs *regs = power_well->desc->hsw.regs; + bool is_tbt = power_well->desc->hsw.is_tc_tbt; + bool timeout_expected; u32 val; - icl_tc_port_assert_ref_held(dev_priv, power_well); + icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port); val = intel_de_read(dev_priv, DP_AUX_CH_CTL(aux_ch)); val &= ~DP_AUX_CH_CTL_TBT_IO; - if (power_well->desc->hsw.is_tc_tbt) + if (is_tbt) val |= DP_AUX_CH_CTL_TBT_IO; intel_de_write(dev_priv, DP_AUX_CH_CTL(aux_ch), val); - hsw_power_well_enable(dev_priv, power_well); + val = intel_de_read(dev_priv, regs->driver); + intel_de_write(dev_priv, regs->driver, + val | HSW_PWR_WELL_CTL_REQ(power_well->desc->hsw.idx)); + + /* + * An AUX timeout is expected if the TBT DP tunnel is down, + * or need to enable AUX on a legacy TypeC port as part of the TC-cold + * exit sequence. + */ + timeout_expected = is_tbt; + if (INTEL_GEN(dev_priv) == 11 && dig_port->tc_legacy_port) { + icl_tc_cold_exit(dev_priv); + timeout_expected = true; + } - if (INTEL_GEN(dev_priv) >= 12 && !power_well->desc->hsw.is_tc_tbt) { + hsw_wait_for_power_well_enable(dev_priv, power_well, timeout_expected); + + if (INTEL_GEN(dev_priv) >= 12 && !is_tbt) { enum tc_port tc_port; tc_port = TGL_AUX_PW_TO_TC_PORT(power_well->desc->hsw.idx); @@ -586,11 +644,48 @@ static void icl_tc_phy_aux_power_well_disable(struct drm_i915_private *dev_priv, struct i915_power_well *power_well) { - icl_tc_port_assert_ref_held(dev_priv, power_well); + enum aux_ch aux_ch = icl_tc_phy_aux_ch(dev_priv, power_well); + struct intel_digital_port *dig_port = aux_ch_to_digital_port(dev_priv, aux_ch); + + icl_tc_port_assert_ref_held(dev_priv, power_well, dig_port); hsw_power_well_disable(dev_priv, power_well); } +static void +icl_aux_power_well_enable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + int pw_idx = power_well->desc->hsw.idx; + enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); /* non-TBT only */ + bool is_tbt = power_well->desc->hsw.is_tc_tbt; + + if (is_tbt || intel_phy_is_tc(dev_priv, phy)) + return icl_tc_phy_aux_power_well_enable(dev_priv, power_well); + else if (IS_ICELAKE(dev_priv)) + return icl_combo_phy_aux_power_well_enable(dev_priv, + power_well); + else + return hsw_power_well_enable(dev_priv, power_well); +} + +static void +icl_aux_power_well_disable(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + int pw_idx = power_well->desc->hsw.idx; + enum phy phy = ICL_AUX_PW_TO_PHY(pw_idx); /* non-TBT only */ + bool is_tbt = power_well->desc->hsw.is_tc_tbt; + + if (is_tbt || intel_phy_is_tc(dev_priv, phy)) + return icl_tc_phy_aux_power_well_disable(dev_priv, power_well); + else if (IS_ICELAKE(dev_priv)) + return icl_combo_phy_aux_power_well_disable(dev_priv, + power_well); + else + return hsw_power_well_disable(dev_priv, power_well); +} + /* * We should only use the power well if we explicitly asked the hardware to * enable it, so check if it's enabled and also check if we've requested it to @@ -623,15 +718,19 @@ static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv, static void assert_can_enable_dc9(struct drm_i915_private *dev_priv) { - WARN_ONCE((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC9), - "DC9 already programmed to be enabled.\n"); - WARN_ONCE(intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5, - "DC5 still not disabled to enable DC9.\n"); - WARN_ONCE(intel_de_read(dev_priv, HSW_PWR_WELL_CTL2) & - HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2), - "Power well 2 on.\n"); - WARN_ONCE(intel_irqs_enabled(dev_priv), - "Interrupts not disabled yet.\n"); + drm_WARN_ONCE(&dev_priv->drm, + (intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_DC9), + "DC9 already programmed to be enabled.\n"); + drm_WARN_ONCE(&dev_priv->drm, + intel_de_read(dev_priv, DC_STATE_EN) & + DC_STATE_EN_UPTO_DC5, + "DC5 still not disabled to enable DC9.\n"); + drm_WARN_ONCE(&dev_priv->drm, + intel_de_read(dev_priv, HSW_PWR_WELL_CTL2) & + HSW_PWR_WELL_CTL_REQ(SKL_PW_CTL_IDX_PW_2), + "Power well 2 on.\n"); + drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv), + "Interrupts not disabled yet.\n"); /* * TODO: check for the following to verify the conditions to enter DC9 @@ -644,10 +743,12 @@ static void assert_can_enable_dc9(struct drm_i915_private *dev_priv) static void assert_can_disable_dc9(struct drm_i915_private *dev_priv) { - WARN_ONCE(intel_irqs_enabled(dev_priv), - "Interrupts not disabled yet.\n"); - WARN_ONCE(intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5, - "DC5 still not disabled.\n"); + drm_WARN_ONCE(&dev_priv->drm, intel_irqs_enabled(dev_priv), + "Interrupts not disabled yet.\n"); + drm_WARN_ONCE(&dev_priv->drm, + intel_de_read(dev_priv, DC_STATE_EN) & + DC_STATE_EN_UPTO_DC5, + "DC5 still not disabled.\n"); /* * TODO: check for the following to verify DC9 state was indeed @@ -756,7 +857,8 @@ static void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state) u32 val; u32 mask; - if (WARN_ON_ONCE(state & ~dev_priv->csr.allowed_dc_mask)) + if (drm_WARN_ON_ONCE(&dev_priv->drm, + state & ~dev_priv->csr.allowed_dc_mask)) state &= dev_priv->csr.allowed_dc_mask; val = intel_de_read(dev_priv, DC_STATE_EN); @@ -851,11 +953,13 @@ static void bxt_disable_dc9(struct drm_i915_private *dev_priv) static void assert_csr_loaded(struct drm_i915_private *dev_priv) { - WARN_ONCE(!intel_de_read(dev_priv, CSR_PROGRAM(0)), - "CSR program storage start is NULL\n"); - WARN_ONCE(!intel_de_read(dev_priv, CSR_SSP_BASE), - "CSR SSP Base Not fine\n"); - WARN_ONCE(!intel_de_read(dev_priv, CSR_HTP_SKL), "CSR HTP Not fine\n"); + drm_WARN_ONCE(&dev_priv->drm, + !intel_de_read(dev_priv, CSR_PROGRAM(0)), + "CSR program storage start is NULL\n"); + drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, CSR_SSP_BASE), + "CSR SSP Base Not fine\n"); + drm_WARN_ONCE(&dev_priv->drm, !intel_de_read(dev_priv, CSR_HTP_SKL), + "CSR HTP Not fine\n"); } static struct i915_power_well * @@ -875,7 +979,9 @@ lookup_power_well(struct drm_i915_private *dev_priv, * the first power well and hope the WARN gets reported so we can fix * our driver. */ - WARN(1, "Power well %d not defined for this platform\n", power_well_id); + drm_WARN(&dev_priv->drm, 1, + "Power well %d not defined for this platform\n", + power_well_id); return &dev_priv->power_domains.power_wells[0]; } @@ -898,7 +1004,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv, mutex_lock(&power_domains->lock); power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF); - if (WARN_ON(!power_well)) + if (drm_WARN_ON(&dev_priv->drm, !power_well)) goto unlock; state = sanitize_target_dc_state(dev_priv, state); @@ -926,13 +1032,22 @@ unlock: static void assert_can_enable_dc5(struct drm_i915_private *dev_priv) { - bool pg2_enabled = intel_display_power_well_is_enabled(dev_priv, - SKL_DISP_PW_2); + enum i915_power_well_id high_pg; + + /* Power wells at this level and above must be disabled for DC5 entry */ + if (INTEL_GEN(dev_priv) >= 12) + high_pg = ICL_DISP_PW_3; + else + high_pg = SKL_DISP_PW_2; - WARN_ONCE(pg2_enabled, "PG2 not disabled to enable DC5.\n"); + drm_WARN_ONCE(&dev_priv->drm, + intel_display_power_well_is_enabled(dev_priv, high_pg), + "Power wells above platform's DC5 limit still enabled.\n"); - WARN_ONCE((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC5), - "DC5 already programmed to be enabled.\n"); + drm_WARN_ONCE(&dev_priv->drm, + (intel_de_read(dev_priv, DC_STATE_EN) & + DC_STATE_EN_UPTO_DC5), + "DC5 already programmed to be enabled.\n"); assert_rpm_wakelock_held(&dev_priv->runtime_pm); assert_csr_loaded(dev_priv); @@ -954,10 +1069,13 @@ static void gen9_enable_dc5(struct drm_i915_private *dev_priv) static void assert_can_enable_dc6(struct drm_i915_private *dev_priv) { - WARN_ONCE(intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE, - "Backlight is not disabled.\n"); - WARN_ONCE((intel_de_read(dev_priv, DC_STATE_EN) & DC_STATE_EN_UPTO_DC6), - "DC6 already programmed to be enabled.\n"); + drm_WARN_ONCE(&dev_priv->drm, + intel_de_read(dev_priv, UTIL_PIN_CTL) & UTIL_PIN_ENABLE, + "Backlight is not disabled.\n"); + drm_WARN_ONCE(&dev_priv->drm, + (intel_de_read(dev_priv, DC_STATE_EN) & + DC_STATE_EN_UPTO_DC6), + "DC6 already programmed to be enabled.\n"); assert_csr_loaded(dev_priv); } @@ -1045,10 +1163,11 @@ static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv) u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv); u8 enabled_dbuf_slices = dev_priv->enabled_dbuf_slices_mask; - WARN(hw_enabled_dbuf_slices != enabled_dbuf_slices, - "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n", - hw_enabled_dbuf_slices, - enabled_dbuf_slices); + drm_WARN(&dev_priv->drm, + hw_enabled_dbuf_slices != enabled_dbuf_slices, + "Unexpected DBuf power power state (0x%08x, expected 0x%08x)\n", + hw_enabled_dbuf_slices, + enabled_dbuf_slices); } static void gen9_disable_dc_states(struct drm_i915_private *dev_priv) @@ -1064,7 +1183,9 @@ static void gen9_disable_dc_states(struct drm_i915_private *dev_priv) dev_priv->display.get_cdclk(dev_priv, &cdclk_config); /* Can't read out voltage_level so can't use intel_cdclk_changed() */ - WARN_ON(intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, &cdclk_config)); + drm_WARN_ON(&dev_priv->drm, + intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, + &cdclk_config)); gen9_assert_dbuf_enabled(dev_priv); @@ -1221,8 +1342,8 @@ static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, * We only ever set the power-on and power-gate states, anything * else is unexpected. */ - WARN_ON(state != PUNIT_PWRGT_PWR_ON(pw_idx) && - state != PUNIT_PWRGT_PWR_GATE(pw_idx)); + drm_WARN_ON(&dev_priv->drm, state != PUNIT_PWRGT_PWR_ON(pw_idx) && + state != PUNIT_PWRGT_PWR_GATE(pw_idx)); if (state == ctrl) enabled = true; @@ -1231,7 +1352,7 @@ static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv, * is poking at the power controls too. */ ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask; - WARN_ON(ctrl != state); + drm_WARN_ON(&dev_priv->drm, ctrl != state); vlv_punit_put(dev_priv); @@ -1260,7 +1381,7 @@ static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv) MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE); intel_de_write(dev_priv, CBR1_VLV, 0); - WARN_ON(RUNTIME_INFO(dev_priv)->rawclk_freq == 0); + drm_WARN_ON(&dev_priv->drm, RUNTIME_INFO(dev_priv)->rawclk_freq == 0); intel_de_write(dev_priv, RAWCLK_FREQ_VLV, DIV_ROUND_CLOSEST(RUNTIME_INFO(dev_priv)->rawclk_freq, 1000)); @@ -1502,8 +1623,9 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv, enum pipe pipe; u32 tmp; - WARN_ON_ONCE(power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC && - power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D); + drm_WARN_ON_ONCE(&dev_priv->drm, + power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC && + power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D); if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) { pipe = PIPE_A; @@ -1564,8 +1686,9 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv, { enum dpio_phy phy; - WARN_ON_ONCE(power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC && - power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D); + drm_WARN_ON_ONCE(&dev_priv->drm, + power_well->desc->id != VLV_DISP_PW_DPIO_CMN_BC && + power_well->desc->id != CHV_DISP_PW_DPIO_CMN_D); if (power_well->desc->id == VLV_DISP_PW_DPIO_CMN_BC) { phy = DPIO_PHY0; @@ -1647,11 +1770,13 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi actual = val >> DPIO_ANYDL_POWERDOWN_SHIFT_CH1; actual &= DPIO_ALLDL_POWERDOWN | DPIO_ANYDL_POWERDOWN; - WARN(actual != expected, - "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n", - !!(actual & DPIO_ALLDL_POWERDOWN), !!(actual & DPIO_ANYDL_POWERDOWN), - !!(expected & DPIO_ALLDL_POWERDOWN), !!(expected & DPIO_ANYDL_POWERDOWN), - reg, val); + drm_WARN(&dev_priv->drm, actual != expected, + "Unexpected DPIO lane power down: all %d, any %d. Expected: all %d, any %d. (0x%x = 0x%08x)\n", + !!(actual & DPIO_ALLDL_POWERDOWN), + !!(actual & DPIO_ANYDL_POWERDOWN), + !!(expected & DPIO_ALLDL_POWERDOWN), + !!(expected & DPIO_ANYDL_POWERDOWN), + reg, val); } bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy, @@ -1733,7 +1858,8 @@ static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, * We only ever set the power-on and power-gate states, anything * else is unexpected. */ - WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe)); + drm_WARN_ON(&dev_priv->drm, state != DP_SSS_PWR_ON(pipe) && + state != DP_SSS_PWR_GATE(pipe)); enabled = state == DP_SSS_PWR_ON(pipe); /* @@ -1741,7 +1867,7 @@ static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv, * is poking at the power controls too. */ ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe); - WARN_ON(ctrl << 16 != state); + drm_WARN_ON(&dev_priv->drm, ctrl << 16 != state); vlv_punit_put(dev_priv); @@ -1840,20 +1966,27 @@ __async_put_domains_state_ok(struct i915_power_domains *power_domains) static void print_power_domains(struct i915_power_domains *power_domains, const char *prefix, u64 mask) { + struct drm_i915_private *i915 = container_of(power_domains, + struct drm_i915_private, + power_domains); enum intel_display_power_domain domain; - DRM_DEBUG_DRIVER("%s (%lu):\n", prefix, hweight64(mask)); + drm_dbg(&i915->drm, "%s (%lu):\n", prefix, hweight64(mask)); for_each_power_domain(domain, mask) - DRM_DEBUG_DRIVER("%s use_count %d\n", - intel_display_power_domain_str(domain), - power_domains->domain_use_count[domain]); + drm_dbg(&i915->drm, "%s use_count %d\n", + intel_display_power_domain_str(domain), + power_domains->domain_use_count[domain]); } static void print_async_put_domains_state(struct i915_power_domains *power_domains) { - DRM_DEBUG_DRIVER("async_put_wakeref %u\n", - power_domains->async_put_wakeref); + struct drm_i915_private *i915 = container_of(power_domains, + struct drm_i915_private, + power_domains); + + drm_dbg(&i915->drm, "async_put_wakeref %u\n", + power_domains->async_put_wakeref); print_power_domains(power_domains, "async_put_domains[0]", power_domains->async_put_domains[0]); @@ -2019,12 +2152,13 @@ __intel_display_power_put_domain(struct drm_i915_private *dev_priv, power_domains = &dev_priv->power_domains; - WARN(!power_domains->domain_use_count[domain], - "Use count on domain %s is already zero\n", - name); - WARN(async_put_domains_mask(power_domains) & BIT_ULL(domain), - "Async disabling of domain %s is pending\n", - name); + drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain], + "Use count on domain %s is already zero\n", + name); + drm_WARN(&dev_priv->drm, + async_put_domains_mask(power_domains) & BIT_ULL(domain), + "Async disabling of domain %s is pending\n", + name); power_domains->domain_use_count[domain]--; @@ -2169,7 +2303,7 @@ void __intel_display_power_put_async(struct drm_i915_private *i915, goto out_verify; } - WARN_ON(power_domains->domain_use_count[domain] != 1); + drm_WARN_ON(&i915->drm, power_domains->domain_use_count[domain] != 1); /* Let a pending work requeue itself or queue a new one. */ if (power_domains->async_put_wakeref) { @@ -2244,7 +2378,7 @@ intel_display_power_flush_work_sync(struct drm_i915_private *i915) verify_async_put_domains_state(power_domains); - WARN_ON(power_domains->async_put_wakeref); + drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref); } #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_RUNTIME_PM) @@ -2712,7 +2846,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, BIT_ULL(POWER_DOMAIN_INIT)) #define TGL_DISPLAY_DC_OFF_POWER_DOMAINS ( \ - TGL_PW_2_POWER_DOMAINS | \ + TGL_PW_3_POWER_DOMAINS | \ BIT_ULL(POWER_DOMAIN_MODESET) | \ BIT_ULL(POWER_DOMAIN_AUX_A) | \ BIT_ULL(POWER_DOMAIN_AUX_B) | \ @@ -2764,6 +2898,21 @@ void intel_display_power_put(struct drm_i915_private *dev_priv, #define TGL_AUX_I_TBT6_IO_POWER_DOMAINS ( \ BIT_ULL(POWER_DOMAIN_AUX_I_TBT)) +#define TGL_TC_COLD_OFF_POWER_DOMAINS ( \ + BIT_ULL(POWER_DOMAIN_AUX_D) | \ + BIT_ULL(POWER_DOMAIN_AUX_E) | \ + BIT_ULL(POWER_DOMAIN_AUX_F) | \ + BIT_ULL(POWER_DOMAIN_AUX_G) | \ + BIT_ULL(POWER_DOMAIN_AUX_H) | \ + BIT_ULL(POWER_DOMAIN_AUX_I) | \ + BIT_ULL(POWER_DOMAIN_AUX_D_TBT) | \ + BIT_ULL(POWER_DOMAIN_AUX_E_TBT) | \ + BIT_ULL(POWER_DOMAIN_AUX_F_TBT) | \ + BIT_ULL(POWER_DOMAIN_AUX_G_TBT) | \ + BIT_ULL(POWER_DOMAIN_AUX_H_TBT) | \ + BIT_ULL(POWER_DOMAIN_AUX_I_TBT) | \ + BIT_ULL(POWER_DOMAIN_TC_COLD_OFF)) + static const struct i915_power_well_ops i9xx_always_on_power_well_ops = { .sync_hw = i9xx_power_well_sync_hw_noop, .enable = i9xx_always_on_power_well_noop, @@ -3462,17 +3611,10 @@ static const struct i915_power_well_desc cnl_power_wells[] = { }, }; -static const struct i915_power_well_ops icl_combo_phy_aux_power_well_ops = { - .sync_hw = hsw_power_well_sync_hw, - .enable = icl_combo_phy_aux_power_well_enable, - .disable = icl_combo_phy_aux_power_well_disable, - .is_enabled = hsw_power_well_enabled, -}; - -static const struct i915_power_well_ops icl_tc_phy_aux_power_well_ops = { +static const struct i915_power_well_ops icl_aux_power_well_ops = { .sync_hw = hsw_power_well_sync_hw, - .enable = icl_tc_phy_aux_power_well_enable, - .disable = icl_tc_phy_aux_power_well_disable, + .enable = icl_aux_power_well_enable, + .disable = icl_aux_power_well_disable, .is_enabled = hsw_power_well_enabled, }; @@ -3530,7 +3672,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { .name = "power well 3", .domains = ICL_PW_3_POWER_DOMAINS, .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, + .id = ICL_DISP_PW_3, { .hsw.regs = &hsw_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_PW_3, @@ -3602,7 +3744,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX A", .domains = ICL_AUX_A_IO_POWER_DOMAINS, - .ops = &icl_combo_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3612,7 +3754,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX B", .domains = ICL_AUX_B_IO_POWER_DOMAINS, - .ops = &icl_combo_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3622,7 +3764,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX C TC1", .domains = ICL_AUX_C_TC1_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3633,7 +3775,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX D TC2", .domains = ICL_AUX_D_TC2_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3644,7 +3786,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX E TC3", .domains = ICL_AUX_E_TC3_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3655,7 +3797,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX F TC4", .domains = ICL_AUX_F_TC4_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3666,7 +3808,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX C TBT1", .domains = ICL_AUX_C_TBT1_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3677,7 +3819,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX D TBT2", .domains = ICL_AUX_D_TBT2_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3688,7 +3830,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX E TBT3", .domains = ICL_AUX_E_TBT3_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3699,7 +3841,7 @@ static const struct i915_power_well_desc icl_power_wells[] = { { .name = "AUX F TBT4", .domains = ICL_AUX_F_TBT4_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -3721,149 +3863,89 @@ static const struct i915_power_well_desc icl_power_wells[] = { }, }; -static const struct i915_power_well_desc ehl_power_wells[] = { - { - .name = "always-on", - .always_on = true, - .domains = POWER_DOMAIN_MASK, - .ops = &i9xx_always_on_power_well_ops, - .id = DISP_PW_ID_NONE, - }, - { - .name = "power well 1", - /* Handled by the DMC firmware */ - .always_on = true, - .domains = 0, - .ops = &hsw_power_well_ops, - .id = SKL_DISP_PW_1, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_PW_1, - .hsw.has_fuses = true, - }, - }, - { - .name = "DC off", - .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS, - .ops = &gen9_dc_off_power_well_ops, - .id = SKL_DISP_DC_OFF, - }, - { - .name = "power well 2", - .domains = ICL_PW_2_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = SKL_DISP_PW_2, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_PW_2, - .hsw.has_fuses = true, - }, - }, - { - .name = "power well 3", - .domains = ICL_PW_3_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_PW_3, - .hsw.irq_pipe_mask = BIT(PIPE_B), - .hsw.has_vga = true, - .hsw.has_fuses = true, - }, - }, - { - .name = "DDI A IO", - .domains = ICL_DDI_IO_A_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_ddi_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_DDI_A, - }, - }, - { - .name = "DDI B IO", - .domains = ICL_DDI_IO_B_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_ddi_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_DDI_B, - }, - }, - { - .name = "DDI C IO", - .domains = ICL_DDI_IO_C_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_ddi_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_DDI_C, - }, - }, - { - .name = "DDI D IO", - .domains = ICL_DDI_IO_D_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_ddi_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_DDI_D, - }, - }, - { - .name = "AUX A", - .domains = ICL_AUX_A_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_aux_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_AUX_A, - }, - }, - { - .name = "AUX B", - .domains = ICL_AUX_B_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_aux_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_AUX_B, - }, - }, - { - .name = "AUX C", - .domains = ICL_AUX_C_TC1_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_aux_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_AUX_C, - }, - }, - { - .name = "AUX D", - .domains = ICL_AUX_D_TC2_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &icl_aux_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_AUX_D, - }, - }, - { - .name = "power well 4", - .domains = ICL_PW_4_POWER_DOMAINS, - .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, - { - .hsw.regs = &hsw_power_well_regs, - .hsw.idx = ICL_PW_CTL_IDX_PW_4, - .hsw.has_fuses = true, - .hsw.irq_pipe_mask = BIT(PIPE_C), - }, - }, +static void +tgl_tc_cold_request(struct drm_i915_private *i915, bool block) +{ + u8 tries = 0; + int ret; + + while (1) { + u32 low_val = 0, high_val; + + if (block) + high_val = TGL_PCODE_EXIT_TCCOLD_DATA_H_BLOCK_REQ; + else + high_val = TGL_PCODE_EXIT_TCCOLD_DATA_H_UNBLOCK_REQ; + + /* + * Spec states that we should timeout the request after 200us + * but the function below will timeout after 500us + */ + ret = sandybridge_pcode_read(i915, TGL_PCODE_TCCOLD, &low_val, + &high_val); + if (ret == 0) { + if (block && + (low_val & TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED)) + ret = -EIO; + else + break; + } + + if (++tries == 3) + break; + + if (ret == -EAGAIN) + msleep(1); + } + + if (ret) + drm_err(&i915->drm, "TC cold %sblock failed\n", + block ? "" : "un"); + else + drm_dbg_kms(&i915->drm, "TC cold %sblock succeeded\n", + block ? "" : "un"); +} + +static void +tgl_tc_cold_off_power_well_enable(struct drm_i915_private *i915, + struct i915_power_well *power_well) +{ + tgl_tc_cold_request(i915, true); +} + +static void +tgl_tc_cold_off_power_well_disable(struct drm_i915_private *i915, + struct i915_power_well *power_well) +{ + tgl_tc_cold_request(i915, false); +} + +static void +tgl_tc_cold_off_power_well_sync_hw(struct drm_i915_private *i915, + struct i915_power_well *power_well) +{ + if (power_well->count > 0) + tgl_tc_cold_off_power_well_enable(i915, power_well); + else + tgl_tc_cold_off_power_well_disable(i915, power_well); +} + +static bool +tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv, + struct i915_power_well *power_well) +{ + /* + * Not the correctly implementation but there is no way to just read it + * from PCODE, so returning count to avoid state mismatch errors + */ + return power_well->count; +} + +static const struct i915_power_well_ops tgl_tc_cold_off_ops = { + .sync_hw = tgl_tc_cold_off_power_well_sync_hw, + .enable = tgl_tc_cold_off_power_well_enable, + .disable = tgl_tc_cold_off_power_well_disable, + .is_enabled = tgl_tc_cold_off_power_well_is_enabled, }; static const struct i915_power_well_desc tgl_power_wells[] = { @@ -3908,7 +3990,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { .name = "power well 3", .domains = TGL_PW_3_POWER_DOMAINS, .ops = &hsw_power_well_ops, - .id = DISP_PW_ID_NONE, + .id = ICL_DISP_PW_3, { .hsw.regs = &hsw_power_well_regs, .hsw.idx = ICL_PW_CTL_IDX_PW_3, @@ -4010,7 +4092,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX A", .domains = TGL_AUX_A_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4020,7 +4102,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX B", .domains = TGL_AUX_B_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4030,7 +4112,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX C", .domains = TGL_AUX_C_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4040,7 +4122,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX D TC1", .domains = TGL_AUX_D_TC1_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4051,7 +4133,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX E TC2", .domains = TGL_AUX_E_TC2_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4062,7 +4144,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX F TC3", .domains = TGL_AUX_F_TC3_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4073,7 +4155,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX G TC4", .domains = TGL_AUX_G_TC4_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4084,7 +4166,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX H TC5", .domains = TGL_AUX_H_TC5_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4095,7 +4177,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX I TC6", .domains = TGL_AUX_I_TC6_IO_POWER_DOMAINS, - .ops = &icl_tc_phy_aux_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4106,7 +4188,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX D TBT1", .domains = TGL_AUX_D_TBT1_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4117,7 +4199,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX E TBT2", .domains = TGL_AUX_E_TBT2_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4128,7 +4210,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX F TBT3", .domains = TGL_AUX_F_TBT3_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4139,7 +4221,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX G TBT4", .domains = TGL_AUX_G_TBT4_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4150,7 +4232,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX H TBT5", .domains = TGL_AUX_H_TBT5_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4161,7 +4243,7 @@ static const struct i915_power_well_desc tgl_power_wells[] = { { .name = "AUX I TBT6", .domains = TGL_AUX_I_TBT6_IO_POWER_DOMAINS, - .ops = &hsw_power_well_ops, + .ops = &icl_aux_power_well_ops, .id = DISP_PW_ID_NONE, { .hsw.regs = &icl_aux_power_well_regs, @@ -4193,6 +4275,12 @@ static const struct i915_power_well_desc tgl_power_wells[] = { .hsw.irq_pipe_mask = BIT(PIPE_D), }, }, + { + .name = "TC cold off", + .domains = TGL_TC_COLD_OFF_POWER_DOMAINS, + .ops = &tgl_tc_cold_off_ops, + .id = DISP_PW_ID_NONE, + }, }; static int @@ -4342,8 +4430,6 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv) */ if (IS_GEN(dev_priv, 12)) { err = set_power_wells(power_domains, tgl_power_wells); - } else if (IS_ELKHARTLAKE(dev_priv)) { - err = set_power_wells(power_domains, ehl_power_wells); } else if (IS_GEN(dev_priv, 11)) { err = set_power_wells(power_domains, icl_power_wells); } else if (IS_CANNONLAKE(dev_priv)) { @@ -4405,9 +4491,8 @@ static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv) mutex_unlock(&power_domains->lock); } -static inline -bool intel_dbuf_slice_set(struct drm_i915_private *dev_priv, - i915_reg_t reg, bool enable) +static bool intel_dbuf_slice_set(struct drm_i915_private *dev_priv, + i915_reg_t reg, bool enable) { u32 val, status; @@ -4443,10 +4528,11 @@ void icl_dbuf_slices_update(struct drm_i915_private *dev_priv, int max_slices = INTEL_INFO(dev_priv)->num_supported_dbuf_slices; struct i915_power_domains *power_domains = &dev_priv->power_domains; - WARN(hweight8(req_slices) > max_slices, - "Invalid number of dbuf slices requested\n"); + drm_WARN(&dev_priv->drm, hweight8(req_slices) > max_slices, + "Invalid number of dbuf slices requested\n"); - DRM_DEBUG_KMS("Updating dbuf slices to 0x%x\n", req_slices); + drm_dbg_kms(&dev_priv->drm, "Updating dbuf slices to 0x%x\n", + req_slices); /* * Might be running this in parallel to gen9_dc_off_power_well_enable @@ -4486,14 +4572,22 @@ static void icl_dbuf_disable(struct drm_i915_private *dev_priv) static void icl_mbus_init(struct drm_i915_private *dev_priv) { - u32 val; + u32 mask, val; + mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK | + MBUS_ABOX_BT_CREDIT_POOL2_MASK | + MBUS_ABOX_B_CREDIT_MASK | + MBUS_ABOX_BW_CREDIT_MASK; val = MBUS_ABOX_BT_CREDIT_POOL1(16) | - MBUS_ABOX_BT_CREDIT_POOL2(16) | - MBUS_ABOX_B_CREDIT(1) | - MBUS_ABOX_BW_CREDIT(1); + MBUS_ABOX_BT_CREDIT_POOL2(16) | + MBUS_ABOX_B_CREDIT(1) | + MBUS_ABOX_BW_CREDIT(1); - intel_de_write(dev_priv, MBUS_ABOX_CTL, val); + intel_de_rmw(dev_priv, MBUS_ABOX_CTL, mask, val); + if (INTEL_GEN(dev_priv) >= 12) { + intel_de_rmw(dev_priv, MBUS_ABOX1_CTL, mask, val); + intel_de_rmw(dev_priv, MBUS_ABOX2_CTL, mask, val); + } } static void hsw_assert_cdclk(struct drm_i915_private *dev_priv) @@ -4974,7 +5068,7 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv) const struct buddy_page_mask *table; int i; - if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0)) + if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_B0)) /* Wa_1409767108: tgl */ table = wa_1409767108_buddy_page_masks; else @@ -4995,6 +5089,14 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv) table[i].page_mask); intel_de_write(dev_priv, BW_BUDDY2_PAGE_MASK, table[i].page_mask); + + /* Wa_22010178259:tgl */ + intel_de_rmw(dev_priv, BW_BUDDY1_CTL, + BW_BUDDY_TLB_REQ_TIMER_MASK, + REG_FIELD_PREP(BW_BUDDY_TLB_REQ_TIMER_MASK, 0x8)); + intel_de_rmw(dev_priv, BW_BUDDY2_CTL, + BW_BUDDY_TLB_REQ_TIMER_MASK, + REG_FIELD_PREP(BW_BUDDY_TLB_REQ_TIMER_MASK, 0x8)); } } @@ -5195,8 +5297,9 @@ static bool vlv_punit_is_power_gated(struct drm_i915_private *dev_priv, u32 reg0 static void assert_ved_power_gated(struct drm_i915_private *dev_priv) { - WARN(!vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0), - "VED not power gated\n"); + drm_WARN(&dev_priv->drm, + !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_VEDSSPM0), + "VED not power gated\n"); } static void assert_isp_power_gated(struct drm_i915_private *dev_priv) @@ -5207,9 +5310,9 @@ static void assert_isp_power_gated(struct drm_i915_private *dev_priv) {} }; - WARN(!pci_dev_present(isp_ids) && - !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0), - "ISP not power gated\n"); + drm_WARN(&dev_priv->drm, !pci_dev_present(isp_ids) && + !vlv_punit_is_power_gated(dev_priv, PUNIT_REG_ISPSSPM0), + "ISP not power gated\n"); } static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv); @@ -5339,7 +5442,7 @@ void intel_power_domains_disable(struct drm_i915_private *i915) { struct i915_power_domains *power_domains = &i915->power_domains; - WARN_ON(power_domains->wakeref); + drm_WARN_ON(&i915->drm, power_domains->wakeref); power_domains->wakeref = intel_display_power_get(i915, POWER_DOMAIN_INIT); @@ -5421,7 +5524,7 @@ void intel_power_domains_resume(struct drm_i915_private *i915) intel_power_domains_init_hw(i915, true); power_domains->display_core_suspended = false; } else { - WARN_ON(power_domains->wakeref); + drm_WARN_ON(&i915->drm, power_domains->wakeref); power_domains->wakeref = intel_display_power_get(i915, POWER_DOMAIN_INIT); } diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h index 601e000ffd0d..6c917699293b 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.h +++ b/drivers/gpu/drm/i915/display/intel_display_power.h @@ -76,6 +76,7 @@ enum intel_display_power_domain { POWER_DOMAIN_MODESET, POWER_DOMAIN_GT_IRQ, POWER_DOMAIN_DPLL_DC_OFF, + POWER_DOMAIN_TC_COLD_OFF, POWER_DOMAIN_INIT, POWER_DOMAIN_NUM, @@ -100,6 +101,7 @@ enum i915_power_well_id { SKL_DISP_PW_MISC_IO, SKL_DISP_PW_1, SKL_DISP_PW_2, + ICL_DISP_PW_3, SKL_DISP_DC_OFF, }; @@ -265,6 +267,8 @@ intel_display_power_domain_str(enum intel_display_power_domain domain); bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); +bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv, + enum i915_power_well_id power_well_id); bool __intel_display_power_is_enabled(struct drm_i915_private *dev_priv, enum intel_display_power_domain domain); intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index e6147364c413..1b689d6bac4c 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -39,7 +39,6 @@ #include <drm/drm_probe_helper.h> #include <drm/drm_rect.h> #include <drm/drm_vblank.h> -#include <drm/i915_drm.h> #include <drm/i915_mei_hdcp_interface.h> #include <media/cec-notifier.h> @@ -133,8 +132,7 @@ struct intel_encoder { u16 cloneable; u8 pipe_mask; enum intel_hotplug_state (*hotplug)(struct intel_encoder *encoder, - struct intel_connector *connector, - bool irq_received); + struct intel_connector *connector); enum intel_output_type (*compute_output_type)(struct intel_encoder *, struct intel_crtc_state *, struct drm_connector_state *); @@ -147,28 +145,35 @@ struct intel_encoder { void (*update_prepare)(struct intel_atomic_state *, struct intel_encoder *, struct intel_crtc *); - void (*pre_pll_enable)(struct intel_encoder *, + void (*pre_pll_enable)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); - void (*pre_enable)(struct intel_encoder *, + void (*pre_enable)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); - void (*enable)(struct intel_encoder *, + void (*enable)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); void (*update_complete)(struct intel_atomic_state *, struct intel_encoder *, struct intel_crtc *); - void (*disable)(struct intel_encoder *, + void (*disable)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); - void (*post_disable)(struct intel_encoder *, + void (*post_disable)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); - void (*post_pll_disable)(struct intel_encoder *, + void (*post_pll_disable)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); - void (*update_pipe)(struct intel_encoder *, + void (*update_pipe)(struct intel_atomic_state *, + struct intel_encoder *, const struct intel_crtc_state *, const struct drm_connector_state *); /* Read out the current hw state of this connector, returning true if @@ -426,6 +431,9 @@ struct intel_connector { struct edid *edid; struct edid *detect_edid; + /* Number of times hotplug detection was tried after an HPD interrupt */ + int hotplug_retries; + /* since POLL and HPD connectors may use the same HPD line keep the native state of connector->polled in case hotplug storm detection changes it */ u8 polled; @@ -641,6 +649,24 @@ struct intel_crtc_scaler_state { #define I915_MODE_FLAG_GET_SCANLINE_FROM_TIMESTAMP (1<<1) /* Flag to use the scanline counter instead of the pixel counter */ #define I915_MODE_FLAG_USE_SCANLINE_COUNTER (1<<2) +/* + * TE0 or TE1 flag is set if the crtc has a DSI encoder which + * is operating in command mode. + * Flag to use TE from DSI0 instead of VBI in command mode + */ +#define I915_MODE_FLAG_DSI_USE_TE0 (1<<3) +/* Flag to use TE from DSI1 instead of VBI in command mode */ +#define I915_MODE_FLAG_DSI_USE_TE1 (1<<4) +/* Flag to indicate mipi dsi periodic command mode where we do not get TE */ +#define I915_MODE_FLAG_DSI_PERIODIC_CMD_MODE (1<<5) + +struct intel_wm_level { + bool enable; + u32 pri_val; + u32 spr_val; + u32 cur_val; + u32 fbc_val; +}; struct intel_pipe_wm { struct intel_wm_level wm[5]; @@ -650,6 +676,14 @@ struct intel_pipe_wm { bool sprites_scaled; }; +struct skl_wm_level { + u16 min_ddb_alloc; + u16 plane_res_b; + u8 plane_res_l; + bool plane_en; + bool ignore_lines; +}; + struct skl_plane_wm { struct skl_wm_level wm[8]; struct skl_wm_level uv_wm[8]; @@ -940,8 +974,7 @@ struct intel_crtc_state { /* Panel fitter placement and size for Ironlake+ */ struct { - u32 pos; - u32 size; + struct drm_rect dst; bool enabled; bool force_thru; } pch_pfit; @@ -1000,6 +1033,7 @@ struct intel_crtc_state { union hdmi_infoframe spd; union hdmi_infoframe hdmi; union hdmi_infoframe drm; + struct drm_dp_vsc_sdp vsc; } infoframes; /* HDMI scrambling status */ @@ -1046,6 +1080,32 @@ struct intel_crtc_state { enum transcoder mst_master_transcoder; }; +enum intel_pipe_crc_source { + INTEL_PIPE_CRC_SOURCE_NONE, + INTEL_PIPE_CRC_SOURCE_PLANE1, + INTEL_PIPE_CRC_SOURCE_PLANE2, + INTEL_PIPE_CRC_SOURCE_PLANE3, + INTEL_PIPE_CRC_SOURCE_PLANE4, + INTEL_PIPE_CRC_SOURCE_PLANE5, + INTEL_PIPE_CRC_SOURCE_PLANE6, + INTEL_PIPE_CRC_SOURCE_PLANE7, + INTEL_PIPE_CRC_SOURCE_PIPE, + /* TV/DP on pre-gen5/vlv can't use the pipe source. */ + INTEL_PIPE_CRC_SOURCE_TV, + INTEL_PIPE_CRC_SOURCE_DP_B, + INTEL_PIPE_CRC_SOURCE_DP_C, + INTEL_PIPE_CRC_SOURCE_DP_D, + INTEL_PIPE_CRC_SOURCE_AUTO, + INTEL_PIPE_CRC_SOURCE_MAX, +}; + +#define INTEL_PIPE_CRC_ENTRIES_NR 128 +struct intel_pipe_crc { + spinlock_t lock; + int skipped; + enum intel_pipe_crc_source source; +}; + struct intel_crtc { struct drm_crtc base; enum pipe pipe; @@ -1089,6 +1149,10 @@ struct intel_crtc { /* per pipe DSB related info */ struct intel_dsb dsb; + +#ifdef CONFIG_DEBUG_FS + struct intel_pipe_crc pipe_crc; +#endif }; struct intel_plane { @@ -1168,8 +1232,6 @@ struct intel_hdmi { }; struct intel_dp_mst_encoder; -#define DP_MAX_DOWNSTREAM_PORTS 0x10 - /* * enum link_m_n_set: * When platform provides two set of M_N registers for dp, we can @@ -1195,6 +1257,7 @@ struct intel_dp_compliance_data { u8 video_pattern; u16 hdisplay, vdisplay; u8 bpc; + struct drm_dp_phy_test_params phytest; }; struct intel_dp_compliance { @@ -1237,6 +1300,7 @@ struct intel_dp { int max_link_rate; /* sink or branch descriptor */ struct drm_dp_desc desc; + u32 edid_quirks; struct drm_dp_aux aux; u32 aux_busy_last_status; u8 train_set[4]; @@ -1303,6 +1367,9 @@ struct intel_dp { /* This is called before a link training is starterd */ void (*prepare_link_retrain)(struct intel_dp *intel_dp); + void (*set_link_train)(struct intel_dp *intel_dp, u8 dp_train_pat); + void (*set_idle_link_train)(struct intel_dp *intel_dp); + void (*set_signal_levels)(struct intel_dp *intel_dp); /* Displayport compliance testing */ struct intel_dp_compliance compliance; @@ -1357,6 +1424,7 @@ struct intel_digital_port { const struct drm_connector_state *conn_state); u32 (*infoframes_enabled)(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config); + bool (*connected)(struct intel_encoder *encoder); }; struct intel_dp_mst_encoder { @@ -1409,8 +1477,17 @@ vlv_pipe_to_channel(enum pipe pipe) } static inline struct intel_crtc * +intel_get_first_crtc(struct drm_i915_private *dev_priv) +{ + return to_intel_crtc(drm_crtc_from_index(&dev_priv->drm, 0)); +} + +static inline struct intel_crtc * intel_get_crtc_for_pipe(struct drm_i915_private *dev_priv, enum pipe pipe) { + /* pipe_to_crtc_mapping may have hole on any of 3 display pipe system */ + drm_WARN_ON(&dev_priv->drm, + !(INTEL_INFO(dev_priv)->pipe_mask & BIT(pipe))); return dev_priv->pipe_to_crtc_mapping[pipe]; } @@ -1600,11 +1677,15 @@ intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state) (1 << INTEL_OUTPUT_DP_MST) | (1 << INTEL_OUTPUT_EDP)); } + static inline void intel_wait_for_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) { - drm_wait_one_vblank(&dev_priv->drm, pipe); + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + + drm_crtc_wait_one_vblank(&crtc->base); } + static inline void intel_wait_for_vblank_if_active(struct drm_i915_private *dev_priv, enum pipe pipe) { diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 7d69b4f43fe2..408c3c1c5e81 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -40,7 +40,6 @@ #include <drm/drm_edid.h> #include <drm/drm_hdcp.h> #include <drm/drm_probe_helper.h> -#include <drm/i915_drm.h> #include "i915_debugfs.h" #include "i915_drv.h" @@ -49,7 +48,6 @@ #include "intel_audio.h" #include "intel_connector.h" #include "intel_ddi.h" -#include "intel_display_debugfs.h" #include "intel_display_types.h" #include "intel_dp.h" #include "intel_dp_link_training.h" @@ -165,6 +163,17 @@ static void intel_dp_set_sink_rates(struct intel_dp *intel_dp) }; int i, max_rate; + if (drm_dp_has_quirk(&intel_dp->desc, 0, + DP_DPCD_QUIRK_CAN_DO_MAX_LINK_RATE_3_24_GBPS)) { + /* Needed, e.g., for Apple MBP 2017, 15 inch eDP Retina panel */ + static const int quirk_rates[] = { 162000, 270000, 324000 }; + + memcpy(intel_dp->sink_rates, quirk_rates, sizeof(quirk_rates)); + intel_dp->num_sink_rates = ARRAY_SIZE(quirk_rates); + + return; + } + max_rate = drm_dp_bw_code_to_link_rate(intel_dp->dpcd[DP_MAX_LINK_RATE]); for (i = 0; i < ARRAY_SIZE(dp_rates); i++) { @@ -325,7 +334,8 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp) int size, max_rate = 0, vbt_max_rate; /* This should only be done once */ - WARN_ON(intel_dp->source_rates || intel_dp->num_source_rates); + drm_WARN_ON(&dev_priv->drm, + intel_dp->source_rates || intel_dp->num_source_rates); if (INTEL_GEN(dev_priv) >= 10) { source_rates = cnl_rates; @@ -452,6 +462,7 @@ static bool intel_dp_can_link_train_fallback_for_edp(struct intel_dp *intel_dp, int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, int link_rate, u8 lane_count) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); int index; index = intel_dp_rate_index(intel_dp->common_rates, @@ -462,7 +473,8 @@ int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, !intel_dp_can_link_train_fallback_for_edp(intel_dp, intel_dp->common_rates[index - 1], lane_count)) { - DRM_DEBUG_KMS("Retrying Link training for eDP with same parameters\n"); + drm_dbg_kms(&i915->drm, + "Retrying Link training for eDP with same parameters\n"); return 0; } intel_dp->max_link_rate = intel_dp->common_rates[index - 1]; @@ -472,13 +484,14 @@ int intel_dp_get_link_train_fallback_values(struct intel_dp *intel_dp, !intel_dp_can_link_train_fallback_for_edp(intel_dp, intel_dp_max_common_rate(intel_dp), lane_count >> 1)) { - DRM_DEBUG_KMS("Retrying Link training for eDP with same parameters\n"); + drm_dbg_kms(&i915->drm, + "Retrying Link training for eDP with same parameters\n"); return 0; } intel_dp->max_link_rate = intel_dp_max_common_rate(intel_dp); intel_dp->max_link_lane_count = lane_count >> 1; } else { - DRM_ERROR("Link Training Unsuccessful\n"); + drm_err(&i915->drm, "Link Training Unsuccessful\n"); return -1; } @@ -553,6 +566,7 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915, static u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, int mode_clock, int mode_hdisplay) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 min_slice_count, i; int max_slice_width; @@ -565,8 +579,9 @@ static u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, max_slice_width = drm_dp_dsc_sink_max_slice_width(intel_dp->dsc_dpcd); if (max_slice_width < DP_DSC_MIN_SLICE_WIDTH_VALUE) { - DRM_DEBUG_KMS("Unsupported slice width %d by DP DSC Sink device\n", - max_slice_width); + drm_dbg_kms(&i915->drm, + "Unsupported slice width %d by DP DSC Sink device\n", + max_slice_width); return 0; } /* Also take into account max slice width */ @@ -584,7 +599,8 @@ static u8 intel_dp_dsc_get_slice_count(struct intel_dp *intel_dp, return valid_dsc_slicecount[i]; } - DRM_DEBUG_KMS("Unsupported Slice Count %d\n", min_slice_count); + drm_dbg_kms(&i915->drm, "Unsupported Slice Count %d\n", + min_slice_count); return 0; } @@ -757,10 +773,11 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp) enum dpio_channel ch = vlv_pipe_to_channel(pipe); u32 DP; - if (WARN(intel_de_read(dev_priv, intel_dp->output_reg) & DP_PORT_EN, - "skipping pipe %c power sequencer kick due to [ENCODER:%d:%s] being active\n", - pipe_name(pipe), intel_dig_port->base.base.base.id, - intel_dig_port->base.base.name)) + if (drm_WARN(&dev_priv->drm, + intel_de_read(dev_priv, intel_dp->output_reg) & DP_PORT_EN, + "skipping pipe %c power sequencer kick due to [ENCODER:%d:%s] being active\n", + pipe_name(pipe), intel_dig_port->base.base.base.id, + intel_dig_port->base.base.name)) return; drm_dbg_kms(&dev_priv->drm, @@ -836,13 +853,16 @@ static enum pipe vlv_find_free_pps(struct drm_i915_private *dev_priv) struct intel_dp *intel_dp = enc_to_intel_dp(encoder); if (encoder->type == INTEL_OUTPUT_EDP) { - WARN_ON(intel_dp->active_pipe != INVALID_PIPE && - intel_dp->active_pipe != intel_dp->pps_pipe); + drm_WARN_ON(&dev_priv->drm, + intel_dp->active_pipe != INVALID_PIPE && + intel_dp->active_pipe != + intel_dp->pps_pipe); if (intel_dp->pps_pipe != INVALID_PIPE) pipes &= ~(1 << intel_dp->pps_pipe); } else { - WARN_ON(intel_dp->pps_pipe != INVALID_PIPE); + drm_WARN_ON(&dev_priv->drm, + intel_dp->pps_pipe != INVALID_PIPE); if (intel_dp->active_pipe != INVALID_PIPE) pipes &= ~(1 << intel_dp->active_pipe); @@ -865,10 +885,10 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp) lockdep_assert_held(&dev_priv->pps_mutex); /* We should never land here with regular DP ports */ - WARN_ON(!intel_dp_is_edp(intel_dp)); + drm_WARN_ON(&dev_priv->drm, !intel_dp_is_edp(intel_dp)); - WARN_ON(intel_dp->active_pipe != INVALID_PIPE && - intel_dp->active_pipe != intel_dp->pps_pipe); + drm_WARN_ON(&dev_priv->drm, intel_dp->active_pipe != INVALID_PIPE && + intel_dp->active_pipe != intel_dp->pps_pipe); if (intel_dp->pps_pipe != INVALID_PIPE) return intel_dp->pps_pipe; @@ -879,7 +899,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp) * Didn't find one. This should not happen since there * are two power sequencers and up to two eDP ports. */ - if (WARN_ON(pipe == INVALID_PIPE)) + if (drm_WARN_ON(&dev_priv->drm, pipe == INVALID_PIPE)) pipe = PIPE_A; vlv_steal_power_sequencer(dev_priv, pipe); @@ -913,7 +933,7 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp) lockdep_assert_held(&dev_priv->pps_mutex); /* We should never land here with regular DP ports */ - WARN_ON(!intel_dp_is_edp(intel_dp)); + drm_WARN_ON(&dev_priv->drm, !intel_dp_is_edp(intel_dp)); if (!intel_dp->pps_reset) return backlight_controller; @@ -1018,8 +1038,10 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv) { struct intel_encoder *encoder; - if (WARN_ON(!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) && - !IS_GEN9_LP(dev_priv))) + if (drm_WARN_ON(&dev_priv->drm, + !(IS_VALLEYVIEW(dev_priv) || + IS_CHERRYVIEW(dev_priv) || + IS_GEN9_LP(dev_priv)))) return; /* @@ -1035,7 +1057,8 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv) for_each_intel_dp(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - WARN_ON(intel_dp->active_pipe != INVALID_PIPE); + drm_WARN_ON(&dev_priv->drm, + intel_dp->active_pipe != INVALID_PIPE); if (encoder->type != INTEL_OUTPUT_EDP) continue; @@ -1170,7 +1193,8 @@ intel_dp_check_edp(struct intel_dp *intel_dp) return; if (!edp_have_panel_power(intel_dp) && !edp_have_panel_vdd(intel_dp)) { - WARN(1, "eDP powered off while attempting aux channel communication.\n"); + drm_WARN(&dev_priv->drm, 1, + "eDP powered off while attempting aux channel communication.\n"); drm_dbg_kms(&dev_priv->drm, "Status 0x%08x Control 0x%08x\n", intel_de_read(dev_priv, _pp_stat_reg(intel_dp)), intel_de_read(dev_priv, _pp_ctrl_reg(intel_dp))); @@ -1366,7 +1390,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp, * lowest possible wakeup latency and so prevent the cpu from going into * deep sleep states. */ - pm_qos_update_request(&i915->pm_qos, 0); + cpu_latency_qos_update_request(&i915->pm_qos, 0); intel_dp_check_edp(intel_dp); @@ -1384,8 +1408,9 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp, const u32 status = intel_uncore_read(uncore, ch_ctl); if (status != intel_dp->aux_busy_last_status) { - WARN(1, "%s: not started (status 0x%08x)\n", - intel_dp->aux.name, status); + drm_WARN(&i915->drm, 1, + "%s: not started (status 0x%08x)\n", + intel_dp->aux.name, status); intel_dp->aux_busy_last_status = status; } @@ -1394,7 +1419,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp, } /* Only 5 data registers! */ - if (WARN_ON(send_bytes > 20 || recv_size > 20)) { + if (drm_WARN_ON(&i915->drm, send_bytes > 20 || recv_size > 20)) { ret = -E2BIG; goto out; } @@ -1498,7 +1523,7 @@ done: ret = recv_bytes; out: - pm_qos_update_request(&i915->pm_qos, PM_QOS_DEFAULT_VALUE); + cpu_latency_qos_update_request(&i915->pm_qos, PM_QOS_DEFAULT_VALUE); if (vdd) edp_panel_vdd_off(intel_dp, false); @@ -1823,6 +1848,7 @@ static void snprintf_int_array(char *str, size_t len, static void intel_dp_print_rates(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); char str[128]; /* FIXME: too big for stack? */ if (!drm_debug_enabled(DRM_UT_KMS)) @@ -1830,15 +1856,15 @@ static void intel_dp_print_rates(struct intel_dp *intel_dp) snprintf_int_array(str, sizeof(str), intel_dp->source_rates, intel_dp->num_source_rates); - DRM_DEBUG_KMS("source rates: %s\n", str); + drm_dbg_kms(&i915->drm, "source rates: %s\n", str); snprintf_int_array(str, sizeof(str), intel_dp->sink_rates, intel_dp->num_sink_rates); - DRM_DEBUG_KMS("sink rates: %s\n", str); + drm_dbg_kms(&i915->drm, "sink rates: %s\n", str); snprintf_int_array(str, sizeof(str), intel_dp->common_rates, intel_dp->num_common_rates); - DRM_DEBUG_KMS("common rates: %s\n", str); + drm_dbg_kms(&i915->drm, "common rates: %s\n", str); } int @@ -1945,6 +1971,8 @@ intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, struct intel_crtc_state *pipe_config, struct link_config_limits *limits) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + /* For DP Compliance we override the computed bpp for the pipe */ if (intel_dp->compliance.test_data.bpc != 0) { int bpp = 3 * intel_dp->compliance.test_data.bpc; @@ -1952,7 +1980,7 @@ intel_dp_adjust_compliance_config(struct intel_dp *intel_dp, limits->min_bpp = limits->max_bpp = bpp; pipe_config->dither_force_disable = bpp == 6 * 3; - DRM_DEBUG_KMS("Setting pipe_bpp to %d\n", bpp); + drm_dbg_kms(&i915->drm, "Setting pipe_bpp to %d\n", bpp); } /* Use values requested by Compliance Test Request */ @@ -2046,6 +2074,7 @@ static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 dsc_max_bpc) static int intel_dp_dsc_compute_params(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; u8 line_buf_depth; @@ -2080,7 +2109,8 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder, line_buf_depth = drm_dp_dsc_sink_line_buf_depth(intel_dp->dsc_dpcd); if (!line_buf_depth) { - DRM_DEBUG_KMS("DSC Sink Line Buffer Depth invalid\n"); + drm_dbg_kms(&i915->drm, + "DSC Sink Line Buffer Depth invalid\n"); return -EINVAL; } @@ -2105,7 +2135,8 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp, { struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp); struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev); - struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; + const struct drm_display_mode *adjusted_mode = + &pipe_config->hw.adjusted_mode; u8 dsc_max_bpc; int pipe_bpp; int ret; @@ -2220,7 +2251,9 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { - struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; + struct drm_i915_private *i915 = to_i915(encoder->base.dev); + const struct drm_display_mode *adjusted_mode = + &pipe_config->hw.adjusted_mode; struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct link_config_limits limits; int common_len; @@ -2255,11 +2288,11 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, intel_dp_adjust_compliance_config(intel_dp, pipe_config, &limits); - DRM_DEBUG_KMS("DP link computation with max lane count %i " - "max rate %d max bpp %d pixel clock %iKHz\n", - limits.max_lane_count, - intel_dp->common_rates[limits.max_clock], - limits.max_bpp, adjusted_mode->crtc_clock); + drm_dbg_kms(&i915->drm, "DP link computation with max lane count %i " + "max rate %d max bpp %d pixel clock %iKHz\n", + limits.max_lane_count, + intel_dp->common_rates[limits.max_clock], + limits.max_bpp, adjusted_mode->crtc_clock); /* * Optimize for slow and wide. This is the place to add alternative @@ -2268,7 +2301,7 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits); /* enable compression if the mode doesn't fit available BW */ - DRM_DEBUG_KMS("Force DSC en = %d\n", intel_dp->force_dsc_en); + drm_dbg_kms(&i915->drm, "Force DSC en = %d\n", intel_dp->force_dsc_en); if (ret || intel_dp->force_dsc_en) { ret = intel_dp_dsc_compute_config(intel_dp, pipe_config, conn_state, &limits); @@ -2277,40 +2310,42 @@ intel_dp_compute_link_config(struct intel_encoder *encoder, } if (pipe_config->dsc.compression_enable) { - DRM_DEBUG_KMS("DP lane count %d clock %d Input bpp %d Compressed bpp %d\n", - pipe_config->lane_count, pipe_config->port_clock, - pipe_config->pipe_bpp, - pipe_config->dsc.compressed_bpp); - - DRM_DEBUG_KMS("DP link rate required %i available %i\n", - intel_dp_link_required(adjusted_mode->crtc_clock, - pipe_config->dsc.compressed_bpp), - intel_dp_max_data_rate(pipe_config->port_clock, - pipe_config->lane_count)); + drm_dbg_kms(&i915->drm, + "DP lane count %d clock %d Input bpp %d Compressed bpp %d\n", + pipe_config->lane_count, pipe_config->port_clock, + pipe_config->pipe_bpp, + pipe_config->dsc.compressed_bpp); + + drm_dbg_kms(&i915->drm, + "DP link rate required %i available %i\n", + intel_dp_link_required(adjusted_mode->crtc_clock, + pipe_config->dsc.compressed_bpp), + intel_dp_max_data_rate(pipe_config->port_clock, + pipe_config->lane_count)); } else { - DRM_DEBUG_KMS("DP lane count %d clock %d bpp %d\n", - pipe_config->lane_count, pipe_config->port_clock, - pipe_config->pipe_bpp); + drm_dbg_kms(&i915->drm, "DP lane count %d clock %d bpp %d\n", + pipe_config->lane_count, pipe_config->port_clock, + pipe_config->pipe_bpp); - DRM_DEBUG_KMS("DP link rate required %i available %i\n", - intel_dp_link_required(adjusted_mode->crtc_clock, - pipe_config->pipe_bpp), - intel_dp_max_data_rate(pipe_config->port_clock, - pipe_config->lane_count)); + drm_dbg_kms(&i915->drm, + "DP link rate required %i available %i\n", + intel_dp_link_required(adjusted_mode->crtc_clock, + pipe_config->pipe_bpp), + intel_dp_max_data_rate(pipe_config->port_clock, + pipe_config->lane_count)); } return 0; } static int intel_dp_ycbcr420_config(struct intel_dp *intel_dp, - struct drm_connector *connector, - struct intel_crtc_state *crtc_state) + struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { + struct drm_connector *connector = conn_state->connector; const struct drm_display_info *info = &connector->display_info; const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; - struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - int ret; if (!drm_mode_is_420_only(info, adjusted_mode) || !intel_dp_get_colorimetry_status(intel_dp) || @@ -2319,16 +2354,7 @@ intel_dp_ycbcr420_config(struct intel_dp *intel_dp, crtc_state->output_format = INTEL_OUTPUT_FORMAT_YCBCR420; - /* YCBCR 420 output conversion needs a scaler */ - ret = skl_update_scaler_crtc(crtc_state); - if (ret) { - DRM_DEBUG_KMS("Scaler allocation for output failed\n"); - return ret; - } - - intel_pch_panel_fitting(crtc, crtc_state, DRM_MODE_SCALE_FULLSCREEN); - - return 0; + return intel_pch_panel_fitting(crtc_state, conn_state); } bool intel_dp_limited_color_range(const struct intel_crtc_state *crtc_state, @@ -2375,6 +2401,164 @@ static bool intel_dp_port_has_audio(struct drm_i915_private *dev_priv, return true; } +static void intel_dp_compute_vsc_colorimetry(const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, + struct drm_dp_vsc_sdp *vsc) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + + /* + * Prepare VSC Header for SU as per DP 1.4 spec, Table 2-118 + * VSC SDP supporting 3D stereo, PSR2, and Pixel Encoding/ + * Colorimetry Format indication. + */ + vsc->revision = 0x5; + vsc->length = 0x13; + + /* DP 1.4a spec, Table 2-120 */ + switch (crtc_state->output_format) { + case INTEL_OUTPUT_FORMAT_YCBCR444: + vsc->pixelformat = DP_PIXELFORMAT_YUV444; + break; + case INTEL_OUTPUT_FORMAT_YCBCR420: + vsc->pixelformat = DP_PIXELFORMAT_YUV420; + break; + case INTEL_OUTPUT_FORMAT_RGB: + default: + vsc->pixelformat = DP_PIXELFORMAT_RGB; + } + + switch (conn_state->colorspace) { + case DRM_MODE_COLORIMETRY_BT709_YCC: + vsc->colorimetry = DP_COLORIMETRY_BT709_YCC; + break; + case DRM_MODE_COLORIMETRY_XVYCC_601: + vsc->colorimetry = DP_COLORIMETRY_XVYCC_601; + break; + case DRM_MODE_COLORIMETRY_XVYCC_709: + vsc->colorimetry = DP_COLORIMETRY_XVYCC_709; + break; + case DRM_MODE_COLORIMETRY_SYCC_601: + vsc->colorimetry = DP_COLORIMETRY_SYCC_601; + break; + case DRM_MODE_COLORIMETRY_OPYCC_601: + vsc->colorimetry = DP_COLORIMETRY_OPYCC_601; + break; + case DRM_MODE_COLORIMETRY_BT2020_CYCC: + vsc->colorimetry = DP_COLORIMETRY_BT2020_CYCC; + break; + case DRM_MODE_COLORIMETRY_BT2020_RGB: + vsc->colorimetry = DP_COLORIMETRY_BT2020_RGB; + break; + case DRM_MODE_COLORIMETRY_BT2020_YCC: + vsc->colorimetry = DP_COLORIMETRY_BT2020_YCC; + break; + case DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65: + case DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER: + vsc->colorimetry = DP_COLORIMETRY_DCI_P3_RGB; + break; + default: + /* + * RGB->YCBCR color conversion uses the BT.709 + * color space. + */ + if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) + vsc->colorimetry = DP_COLORIMETRY_BT709_YCC; + else + vsc->colorimetry = DP_COLORIMETRY_DEFAULT; + break; + } + + vsc->bpc = crtc_state->pipe_bpp / 3; + + /* only RGB pixelformat supports 6 bpc */ + drm_WARN_ON(&dev_priv->drm, + vsc->bpc == 6 && vsc->pixelformat != DP_PIXELFORMAT_RGB); + + /* all YCbCr are always limited range */ + vsc->dynamic_range = DP_DYNAMIC_RANGE_CTA; + vsc->content_type = DP_CONTENT_TYPE_NOT_DEFINED; +} + +static void intel_dp_compute_vsc_sdp(struct intel_dp *intel_dp, + struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct drm_dp_vsc_sdp *vsc = &crtc_state->infoframes.vsc; + + /* When a crtc state has PSR, VSC SDP will be handled by PSR routine */ + if (crtc_state->has_psr) + return; + + if (!intel_dp_needs_vsc_sdp(crtc_state, conn_state)) + return; + + crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC); + vsc->sdp_type = DP_SDP_VSC; + intel_dp_compute_vsc_colorimetry(crtc_state, conn_state, + &crtc_state->infoframes.vsc); +} + +void intel_dp_compute_psr_vsc_sdp(struct intel_dp *intel_dp, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state, + struct drm_dp_vsc_sdp *vsc) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + + vsc->sdp_type = DP_SDP_VSC; + + if (dev_priv->psr.psr2_enabled) { + if (dev_priv->psr.colorimetry_support && + intel_dp_needs_vsc_sdp(crtc_state, conn_state)) { + /* [PSR2, +Colorimetry] */ + intel_dp_compute_vsc_colorimetry(crtc_state, conn_state, + vsc); + } else { + /* + * [PSR2, -Colorimetry] + * Prepare VSC Header for SU as per eDP 1.4 spec, Table 6-11 + * 3D stereo + PSR/PSR2 + Y-coordinate. + */ + vsc->revision = 0x4; + vsc->length = 0xe; + } + } else { + /* + * [PSR1] + * Prepare VSC Header for SU as per DP 1.4 spec, Table 2-118 + * VSC SDP supporting 3D stereo + PSR (applies to eDP v1.3 or + * higher). + */ + vsc->revision = 0x2; + vsc->length = 0x8; + } +} + +static void +intel_dp_compute_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp, + struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + int ret; + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + struct hdmi_drm_infoframe *drm_infoframe = &crtc_state->infoframes.drm.drm; + + if (!conn_state->hdr_output_metadata) + return; + + ret = drm_hdmi_infoframe_set_hdr_metadata(drm_infoframe, conn_state); + + if (ret) { + drm_dbg_kms(&dev_priv->drm, "couldn't set HDR metadata in infoframe\n"); + return; + } + + crtc_state->infoframes.enable |= + intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GAMUT_METADATA); +} + int intel_dp_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, @@ -2385,11 +2569,10 @@ intel_dp_compute_config(struct intel_encoder *encoder, struct intel_dp *intel_dp = enc_to_intel_dp(encoder); struct intel_lspcon *lspcon = enc_to_intel_lspcon(encoder); enum port port = encoder->port; - struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->uapi.crtc); struct intel_connector *intel_connector = intel_dp->attached_connector; struct intel_digital_connector_state *intel_conn_state = to_intel_digital_connector_state(conn_state); - bool constant_n = drm_dp_has_quirk(&intel_dp->desc, + bool constant_n = drm_dp_has_quirk(&intel_dp->desc, 0, DP_DPCD_QUIRK_CONSTANT_N); int ret = 0, output_bpp; @@ -2401,9 +2584,8 @@ intel_dp_compute_config(struct intel_encoder *encoder, if (lspcon->active) lspcon_ycbcr420_config(&intel_connector->base, pipe_config); else - ret = intel_dp_ycbcr420_config(intel_dp, &intel_connector->base, - pipe_config); - + ret = intel_dp_ycbcr420_config(intel_dp, pipe_config, + conn_state); if (ret) return ret; @@ -2419,18 +2601,12 @@ intel_dp_compute_config(struct intel_encoder *encoder, intel_fixed_panel_mode(intel_connector->panel.fixed_mode, adjusted_mode); - if (INTEL_GEN(dev_priv) >= 9) { - ret = skl_update_scaler_crtc(pipe_config); - if (ret) - return ret; - } - if (HAS_GMCH(dev_priv)) - intel_gmch_panel_fitting(intel_crtc, pipe_config, - conn_state->scaling_mode); + ret = intel_gmch_panel_fitting(pipe_config, conn_state); else - intel_pch_panel_fitting(intel_crtc, pipe_config, - conn_state->scaling_mode); + ret = intel_pch_panel_fitting(pipe_config, conn_state); + if (ret) + return ret; } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) @@ -2480,6 +2656,8 @@ intel_dp_compute_config(struct intel_encoder *encoder, intel_dp_set_clock(encoder, pipe_config); intel_psr_compute_config(intel_dp, pipe_config); + intel_dp_compute_vsc_sdp(intel_dp, pipe_config, conn_state); + intel_dp_compute_hdr_metadata_infoframe_sdp(intel_dp, pipe_config, conn_state); return 0; } @@ -2508,9 +2686,6 @@ static void intel_dp_prepare(struct intel_encoder *encoder, intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DP_MST)); - intel_dp->regs.dp_tp_ctl = DP_TP_CTL(port); - intel_dp->regs.dp_tp_status = DP_TP_STATUS(port); - /* * There are four kinds of DP registers: * @@ -2624,22 +2799,27 @@ static void wait_panel_status(struct intel_dp *intel_dp, static void wait_panel_on(struct intel_dp *intel_dp) { - DRM_DEBUG_KMS("Wait for panel power on\n"); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + + drm_dbg_kms(&i915->drm, "Wait for panel power on\n"); wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE); } static void wait_panel_off(struct intel_dp *intel_dp) { - DRM_DEBUG_KMS("Wait for panel power off time\n"); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + + drm_dbg_kms(&i915->drm, "Wait for panel power off time\n"); wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE); } static void wait_panel_power_cycle(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); ktime_t panel_power_on_time; s64 panel_power_off_duration; - DRM_DEBUG_KMS("Wait for panel power cycle\n"); + drm_dbg_kms(&i915->drm, "Wait for panel power cycle\n"); /* take the difference of currrent time and panel power off time * and then make panel wait for t11_t12 if needed. */ @@ -2679,8 +2859,8 @@ static u32 ilk_get_pp_control(struct intel_dp *intel_dp) lockdep_assert_held(&dev_priv->pps_mutex); control = intel_de_read(dev_priv, _pp_ctrl_reg(intel_dp)); - if (WARN_ON(!HAS_DDI(dev_priv) && - (control & PANEL_UNLOCK_MASK) != PANEL_UNLOCK_REGS)) { + if (drm_WARN_ON(&dev_priv->drm, !HAS_DDI(dev_priv) && + (control & PANEL_UNLOCK_MASK) != PANEL_UNLOCK_REGS)) { control &= ~PANEL_UNLOCK_MASK; control |= PANEL_UNLOCK_REGS; } @@ -2779,7 +2959,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp) lockdep_assert_held(&dev_priv->pps_mutex); - WARN_ON(intel_dp->want_panel_vdd); + drm_WARN_ON(&dev_priv->drm, intel_dp->want_panel_vdd); if (!edp_have_panel_vdd(intel_dp)) return; @@ -2876,10 +3056,10 @@ static void edp_panel_on(struct intel_dp *intel_dp) dp_to_dig_port(intel_dp)->base.base.base.id, dp_to_dig_port(intel_dp)->base.base.name); - if (WARN(edp_have_panel_power(intel_dp), - "[ENCODER:%d:%s] panel power already on\n", - dp_to_dig_port(intel_dp)->base.base.base.id, - dp_to_dig_port(intel_dp)->base.base.name)) + if (drm_WARN(&dev_priv->drm, edp_have_panel_power(intel_dp), + "[ENCODER:%d:%s] panel power already on\n", + dp_to_dig_port(intel_dp)->base.base.base.id, + dp_to_dig_port(intel_dp)->base.base.name)) return; wait_panel_power_cycle(intel_dp); @@ -2937,8 +3117,9 @@ static void edp_panel_off(struct intel_dp *intel_dp) drm_dbg_kms(&dev_priv->drm, "Turn [ENCODER:%d:%s] panel power off\n", dig_port->base.base.base.id, dig_port->base.base.name); - WARN(!intel_dp->want_panel_vdd, "Need [ENCODER:%d:%s] VDD to turn off panel\n", - dig_port->base.base.base.id, dig_port->base.base.name); + drm_WARN(&dev_priv->drm, !intel_dp->want_panel_vdd, + "Need [ENCODER:%d:%s] VDD to turn off panel\n", + dig_port->base.base.base.id, dig_port->base.base.name); pp = ilk_get_pp_control(intel_dp); /* We need to switch off panel power _and_ force vdd, for otherwise some @@ -3002,11 +3183,12 @@ void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { struct intel_dp *intel_dp = enc_to_intel_dp(to_intel_encoder(conn_state->best_encoder)); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); if (!intel_dp_is_edp(intel_dp)) return; - DRM_DEBUG_KMS("\n"); + drm_dbg_kms(&i915->drm, "\n"); intel_panel_enable_backlight(crtc_state, conn_state); _intel_edp_backlight_on(intel_dp); @@ -3040,11 +3222,12 @@ static void _intel_edp_backlight_off(struct intel_dp *intel_dp) void intel_edp_backlight_off(const struct drm_connector_state *old_conn_state) { struct intel_dp *intel_dp = enc_to_intel_dp(to_intel_encoder(old_conn_state->best_encoder)); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); if (!intel_dp_is_edp(intel_dp)) return; - DRM_DEBUG_KMS("\n"); + drm_dbg_kms(&i915->drm, "\n"); _intel_edp_backlight_off(intel_dp); intel_panel_disable_backlight(old_conn_state); @@ -3057,6 +3240,7 @@ void intel_edp_backlight_off(const struct drm_connector_state *old_conn_state) static void intel_edp_backlight_power(struct intel_connector *connector, bool enable) { + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_dp *intel_dp = intel_attached_dp(connector); intel_wakeref_t wakeref; bool is_enabled; @@ -3067,8 +3251,8 @@ static void intel_edp_backlight_power(struct intel_connector *connector, if (is_enabled == enable) return; - DRM_DEBUG_KMS("panel power control backlight %s\n", - enable ? "enable" : "disable"); + drm_dbg_kms(&i915->drm, "panel power control backlight %s\n", + enable ? "enable" : "disable"); if (enable) _intel_edp_backlight_on(intel_dp); @@ -3170,7 +3354,7 @@ static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp) * FIXME should really check all downstream ports... */ return intel_dp->dpcd[DP_DPCD_REV] == 0x11 && - intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT && + drm_dp_is_branch(intel_dp->dpcd) && intel_dp->downstream_ports[0] & DP_DS_PORT_HPD; } @@ -3178,6 +3362,7 @@ void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, bool enable) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); int ret; if (!crtc_state->dsc.compression_enable) @@ -3186,13 +3371,15 @@ void intel_dp_sink_set_decompression_state(struct intel_dp *intel_dp, ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_DSC_ENABLE, enable ? DP_DECOMPRESSION_EN : 0); if (ret < 0) - DRM_DEBUG_KMS("Failed to %s sink decompression state\n", - enable ? "enable" : "disable"); + drm_dbg_kms(&i915->drm, + "Failed to %s sink decompression state\n", + enable ? "enable" : "disable"); } /* If the sink supports it, try to set the power state appropriately */ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); int ret, i; /* Should have a valid DPCD by this point */ @@ -3225,8 +3412,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode) } if (ret != 1) - DRM_DEBUG_KMS("failed to %s sink power state\n", - mode == DRM_MODE_DPMS_ON ? "enable" : "disable"); + drm_dbg_kms(&i915->drm, "failed to %s sink power state\n", + mode == DRM_MODE_DPMS_ON ? "enable" : "disable"); } static bool cpt_dp_port_selected(struct drm_i915_private *dev_priv, @@ -3383,7 +3570,8 @@ static void intel_dp_get_config(struct intel_encoder *encoder, } } -static void intel_disable_dp(struct intel_encoder *encoder, +static void intel_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3403,21 +3591,24 @@ static void intel_disable_dp(struct intel_encoder *encoder, intel_edp_panel_off(intel_dp); } -static void g4x_disable_dp(struct intel_encoder *encoder, +static void g4x_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_disable_dp(encoder, old_crtc_state, old_conn_state); + intel_disable_dp(state, encoder, old_crtc_state, old_conn_state); } -static void vlv_disable_dp(struct intel_encoder *encoder, +static void vlv_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_disable_dp(encoder, old_crtc_state, old_conn_state); + intel_disable_dp(state, encoder, old_crtc_state, old_conn_state); } -static void g4x_post_disable_dp(struct intel_encoder *encoder, +static void g4x_post_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3437,14 +3628,16 @@ static void g4x_post_disable_dp(struct intel_encoder *encoder, ilk_edp_pll_off(intel_dp, old_crtc_state); } -static void vlv_post_disable_dp(struct intel_encoder *encoder, +static void vlv_post_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { intel_dp_link_down(encoder, old_crtc_state); } -static void chv_post_disable_dp(struct intel_encoder *encoder, +static void chv_post_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3461,90 +3654,63 @@ static void chv_post_disable_dp(struct intel_encoder *encoder, } static void -_intel_dp_set_link_train(struct intel_dp *intel_dp, - u32 *DP, - u8 dp_train_pat) +cpt_set_link_train(struct intel_dp *intel_dp, + u8 dp_train_pat) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - enum port port = intel_dig_port->base.port; - u8 train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); + u32 *DP = &intel_dp->DP; - if (dp_train_pat & train_pat_mask) - drm_dbg_kms(&dev_priv->drm, - "Using DP training pattern TPS%d\n", - dp_train_pat & train_pat_mask); - - if (HAS_DDI(dev_priv)) { - u32 temp = intel_de_read(dev_priv, intel_dp->regs.dp_tp_ctl); - - if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE) - temp |= DP_TP_CTL_SCRAMBLE_DISABLE; - else - temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE; - - temp &= ~DP_TP_CTL_LINK_TRAIN_MASK; - switch (dp_train_pat & train_pat_mask) { - case DP_TRAINING_PATTERN_DISABLE: - temp |= DP_TP_CTL_LINK_TRAIN_NORMAL; + *DP &= ~DP_LINK_TRAIN_MASK_CPT; - break; - case DP_TRAINING_PATTERN_1: - temp |= DP_TP_CTL_LINK_TRAIN_PAT1; - break; - case DP_TRAINING_PATTERN_2: - temp |= DP_TP_CTL_LINK_TRAIN_PAT2; - break; - case DP_TRAINING_PATTERN_3: - temp |= DP_TP_CTL_LINK_TRAIN_PAT3; - break; - case DP_TRAINING_PATTERN_4: - temp |= DP_TP_CTL_LINK_TRAIN_PAT4; - break; - } - intel_de_write(dev_priv, intel_dp->regs.dp_tp_ctl, temp); + switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { + case DP_TRAINING_PATTERN_DISABLE: + *DP |= DP_LINK_TRAIN_OFF_CPT; + break; + case DP_TRAINING_PATTERN_1: + *DP |= DP_LINK_TRAIN_PAT_1_CPT; + break; + case DP_TRAINING_PATTERN_2: + *DP |= DP_LINK_TRAIN_PAT_2_CPT; + break; + case DP_TRAINING_PATTERN_3: + drm_dbg_kms(&dev_priv->drm, + "TPS3 not supported, using TPS2 instead\n"); + *DP |= DP_LINK_TRAIN_PAT_2_CPT; + break; + } - } else if ((IS_IVYBRIDGE(dev_priv) && port == PORT_A) || - (HAS_PCH_CPT(dev_priv) && port != PORT_A)) { - *DP &= ~DP_LINK_TRAIN_MASK_CPT; + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); + intel_de_posting_read(dev_priv, intel_dp->output_reg); +} - switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { - case DP_TRAINING_PATTERN_DISABLE: - *DP |= DP_LINK_TRAIN_OFF_CPT; - break; - case DP_TRAINING_PATTERN_1: - *DP |= DP_LINK_TRAIN_PAT_1_CPT; - break; - case DP_TRAINING_PATTERN_2: - *DP |= DP_LINK_TRAIN_PAT_2_CPT; - break; - case DP_TRAINING_PATTERN_3: - drm_dbg_kms(&dev_priv->drm, - "TPS3 not supported, using TPS2 instead\n"); - *DP |= DP_LINK_TRAIN_PAT_2_CPT; - break; - } +static void +g4x_set_link_train(struct intel_dp *intel_dp, + u8 dp_train_pat) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + u32 *DP = &intel_dp->DP; - } else { - *DP &= ~DP_LINK_TRAIN_MASK; + *DP &= ~DP_LINK_TRAIN_MASK; - switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { - case DP_TRAINING_PATTERN_DISABLE: - *DP |= DP_LINK_TRAIN_OFF; - break; - case DP_TRAINING_PATTERN_1: - *DP |= DP_LINK_TRAIN_PAT_1; - break; - case DP_TRAINING_PATTERN_2: - *DP |= DP_LINK_TRAIN_PAT_2; - break; - case DP_TRAINING_PATTERN_3: - drm_dbg_kms(&dev_priv->drm, - "TPS3 not supported, using TPS2 instead\n"); - *DP |= DP_LINK_TRAIN_PAT_2; - break; - } + switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) { + case DP_TRAINING_PATTERN_DISABLE: + *DP |= DP_LINK_TRAIN_OFF; + break; + case DP_TRAINING_PATTERN_1: + *DP |= DP_LINK_TRAIN_PAT_1; + break; + case DP_TRAINING_PATTERN_2: + *DP |= DP_LINK_TRAIN_PAT_2; + break; + case DP_TRAINING_PATTERN_3: + drm_dbg_kms(&dev_priv->drm, + "TPS3 not supported, using TPS2 instead\n"); + *DP |= DP_LINK_TRAIN_PAT_2; + break; } + + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); + intel_de_posting_read(dev_priv, intel_dp->output_reg); } static void intel_dp_enable_port(struct intel_dp *intel_dp, @@ -3570,7 +3736,8 @@ static void intel_dp_enable_port(struct intel_dp *intel_dp, intel_de_posting_read(dev_priv, intel_dp->output_reg); } -static void intel_enable_dp(struct intel_encoder *encoder, +static void intel_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -3581,7 +3748,7 @@ static void intel_enable_dp(struct intel_encoder *encoder, enum pipe pipe = crtc->pipe; intel_wakeref_t wakeref; - if (WARN_ON(dp_reg & DP_PORT_EN)) + if (drm_WARN_ON(&dev_priv->drm, dp_reg & DP_PORT_EN)) return; with_pps_lock(intel_dp, wakeref) { @@ -3616,22 +3783,25 @@ static void intel_enable_dp(struct intel_encoder *encoder, } } -static void g4x_enable_dp(struct intel_encoder *encoder, +static void g4x_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { - intel_enable_dp(encoder, pipe_config, conn_state); + intel_enable_dp(state, encoder, pipe_config, conn_state); intel_edp_backlight_on(pipe_config, conn_state); } -static void vlv_enable_dp(struct intel_encoder *encoder, +static void vlv_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { intel_edp_backlight_on(pipe_config, conn_state); } -static void g4x_pre_enable_dp(struct intel_encoder *encoder, +static void g4x_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -3652,9 +3822,9 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp) enum pipe pipe = intel_dp->pps_pipe; i915_reg_t pp_on_reg = PP_ON_DELAYS(pipe); - WARN_ON(intel_dp->active_pipe != INVALID_PIPE); + drm_WARN_ON(&dev_priv->drm, intel_dp->active_pipe != INVALID_PIPE); - if (WARN_ON(pipe != PIPE_A && pipe != PIPE_B)) + if (drm_WARN_ON(&dev_priv->drm, pipe != PIPE_A && pipe != PIPE_B)) return; edp_panel_vdd_off_sync(intel_dp); @@ -3688,10 +3858,10 @@ static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv, for_each_intel_dp(&dev_priv->drm, encoder) { struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - WARN(intel_dp->active_pipe == pipe, - "stealing pipe %c power sequencer from active [ENCODER:%d:%s]\n", - pipe_name(pipe), encoder->base.base.id, - encoder->base.name); + drm_WARN(&dev_priv->drm, intel_dp->active_pipe == pipe, + "stealing pipe %c power sequencer from active [ENCODER:%d:%s]\n", + pipe_name(pipe), encoder->base.base.id, + encoder->base.name); if (intel_dp->pps_pipe != pipe) continue; @@ -3715,7 +3885,7 @@ static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder, lockdep_assert_held(&dev_priv->pps_mutex); - WARN_ON(intel_dp->active_pipe != INVALID_PIPE); + drm_WARN_ON(&dev_priv->drm, intel_dp->active_pipe != INVALID_PIPE); if (intel_dp->pps_pipe != INVALID_PIPE && intel_dp->pps_pipe != crtc->pipe) { @@ -3751,16 +3921,18 @@ static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder, intel_dp_init_panel_power_sequencer_registers(intel_dp, true); } -static void vlv_pre_enable_dp(struct intel_encoder *encoder, +static void vlv_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { vlv_phy_pre_encoder_enable(encoder, pipe_config); - intel_enable_dp(encoder, pipe_config, conn_state); + intel_enable_dp(state, encoder, pipe_config, conn_state); } -static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder, +static void vlv_dp_pre_pll_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -3769,19 +3941,21 @@ static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder, vlv_phy_pre_pll_enable(encoder, pipe_config); } -static void chv_pre_enable_dp(struct intel_encoder *encoder, +static void chv_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { chv_phy_pre_encoder_enable(encoder, pipe_config); - intel_enable_dp(encoder, pipe_config, conn_state); + intel_enable_dp(state, encoder, pipe_config, conn_state); /* Second common lane will stay alive on its own now */ chv_phy_release_cl2_override(encoder); } -static void chv_dp_pre_pll_enable(struct intel_encoder *encoder, +static void chv_dp_pre_pll_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -3790,7 +3964,8 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder, chv_phy_pre_pll_enable(encoder, pipe_config); } -static void chv_dp_post_pll_disable(struct intel_encoder *encoder, +static void chv_dp_post_pll_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -3874,7 +4049,7 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, u8 voltage_swing) } } -static u32 vlv_signal_levels(struct intel_dp *intel_dp) +static void vlv_set_signal_levels(struct intel_dp *intel_dp) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; unsigned long demph_reg_value, preemph_reg_value, @@ -3902,7 +4077,7 @@ static u32 vlv_signal_levels(struct intel_dp *intel_dp) uniqtranscale_reg_value = 0x5598DA3A; break; default: - return 0; + return; } break; case DP_TRAIN_PRE_EMPH_LEVEL_1: @@ -3921,7 +4096,7 @@ static u32 vlv_signal_levels(struct intel_dp *intel_dp) uniqtranscale_reg_value = 0x55ADDA3A; break; default: - return 0; + return; } break; case DP_TRAIN_PRE_EMPH_LEVEL_2: @@ -3936,7 +4111,7 @@ static u32 vlv_signal_levels(struct intel_dp *intel_dp) uniqtranscale_reg_value = 0x55ADDA3A; break; default: - return 0; + return; } break; case DP_TRAIN_PRE_EMPH_LEVEL_3: @@ -3947,20 +4122,18 @@ static u32 vlv_signal_levels(struct intel_dp *intel_dp) uniqtranscale_reg_value = 0x55ADDA3A; break; default: - return 0; + return; } break; default: - return 0; + return; } vlv_set_phy_signal_level(encoder, demph_reg_value, preemph_reg_value, uniqtranscale_reg_value, 0); - - return 0; } -static u32 chv_signal_levels(struct intel_dp *intel_dp) +static void chv_set_signal_levels(struct intel_dp *intel_dp) { struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; u32 deemph_reg_value, margin_reg_value; @@ -3988,7 +4161,7 @@ static u32 chv_signal_levels(struct intel_dp *intel_dp) uniq_trans_scale = true; break; default: - return 0; + return; } break; case DP_TRAIN_PRE_EMPH_LEVEL_1: @@ -4006,7 +4179,7 @@ static u32 chv_signal_levels(struct intel_dp *intel_dp) margin_reg_value = 154; break; default: - return 0; + return; } break; case DP_TRAIN_PRE_EMPH_LEVEL_2: @@ -4020,7 +4193,7 @@ static u32 chv_signal_levels(struct intel_dp *intel_dp) margin_reg_value = 154; break; default: - return 0; + return; } break; case DP_TRAIN_PRE_EMPH_LEVEL_3: @@ -4030,21 +4203,18 @@ static u32 chv_signal_levels(struct intel_dp *intel_dp) margin_reg_value = 154; break; default: - return 0; + return; } break; default: - return 0; + return; } chv_set_phy_signal_level(encoder, deemph_reg_value, margin_reg_value, uniq_trans_scale); - - return 0; } -static u32 -g4x_signal_levels(u8 train_set) +static u32 g4x_signal_levels(u8 train_set) { u32 signal_levels = 0; @@ -4081,12 +4251,31 @@ g4x_signal_levels(u8 train_set) return signal_levels; } +static void +g4x_set_signal_levels(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + u8 train_set = intel_dp->train_set[0]; + u32 signal_levels; + + signal_levels = g4x_signal_levels(train_set); + + drm_dbg_kms(&dev_priv->drm, "Using signal levels %08x\n", + signal_levels); + + intel_dp->DP &= ~(DP_VOLTAGE_MASK | DP_PRE_EMPHASIS_MASK); + intel_dp->DP |= signal_levels; + + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); + intel_de_posting_read(dev_priv, intel_dp->output_reg); +} + /* SNB CPU eDP voltage swing and pre-emphasis control */ -static u32 -snb_cpu_edp_signal_levels(u8 train_set) +static u32 snb_cpu_edp_signal_levels(u8 train_set) { - int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | - DP_TRAIN_PRE_EMPHASIS_MASK); + u8 signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | + DP_TRAIN_PRE_EMPHASIS_MASK); + switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0: @@ -4109,12 +4298,31 @@ snb_cpu_edp_signal_levels(u8 train_set) } } +static void +snb_cpu_edp_set_signal_levels(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + u8 train_set = intel_dp->train_set[0]; + u32 signal_levels; + + signal_levels = snb_cpu_edp_signal_levels(train_set); + + drm_dbg_kms(&dev_priv->drm, "Using signal levels %08x\n", + signal_levels); + + intel_dp->DP &= ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB; + intel_dp->DP |= signal_levels; + + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); + intel_de_posting_read(dev_priv, intel_dp->output_reg); +} + /* IVB CPU eDP voltage swing and pre-emphasis control */ -static u32 -ivb_cpu_edp_signal_levels(u8 train_set) +static u32 ivb_cpu_edp_signal_levels(u8 train_set) { - int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | - DP_TRAIN_PRE_EMPHASIS_MASK); + u8 signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | + DP_TRAIN_PRE_EMPHASIS_MASK); + switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0: return EDP_LINK_TRAIN_400MV_0DB_IVB; @@ -4140,38 +4348,29 @@ ivb_cpu_edp_signal_levels(u8 train_set) } } -void -intel_dp_set_signal_levels(struct intel_dp *intel_dp) +static void +ivb_cpu_edp_set_signal_levels(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - enum port port = intel_dig_port->base.port; - u32 signal_levels, mask = 0; u8 train_set = intel_dp->train_set[0]; + u32 signal_levels; - if (IS_GEN9_LP(dev_priv) || INTEL_GEN(dev_priv) >= 10) { - signal_levels = bxt_signal_levels(intel_dp); - } else if (HAS_DDI(dev_priv)) { - signal_levels = ddi_signal_levels(intel_dp); - mask = DDI_BUF_EMP_MASK; - } else if (IS_CHERRYVIEW(dev_priv)) { - signal_levels = chv_signal_levels(intel_dp); - } else if (IS_VALLEYVIEW(dev_priv)) { - signal_levels = vlv_signal_levels(intel_dp); - } else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) { - signal_levels = ivb_cpu_edp_signal_levels(train_set); - mask = EDP_LINK_TRAIN_VOL_EMP_MASK_IVB; - } else if (IS_GEN(dev_priv, 6) && port == PORT_A) { - signal_levels = snb_cpu_edp_signal_levels(train_set); - mask = EDP_LINK_TRAIN_VOL_EMP_MASK_SNB; - } else { - signal_levels = g4x_signal_levels(train_set); - mask = DP_VOLTAGE_MASK | DP_PRE_EMPHASIS_MASK; - } + signal_levels = ivb_cpu_edp_signal_levels(train_set); + + drm_dbg_kms(&dev_priv->drm, "Using signal levels %08x\n", + signal_levels); + + intel_dp->DP &= ~EDP_LINK_TRAIN_VOL_EMP_MASK_IVB; + intel_dp->DP |= signal_levels; + + intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); + intel_de_posting_read(dev_priv, intel_dp->output_reg); +} - if (mask) - drm_dbg_kms(&dev_priv->drm, "Using signal levels %08x\n", - signal_levels); +void intel_dp_set_signal_levels(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + u8 train_set = intel_dp->train_set[0]; drm_dbg_kms(&dev_priv->drm, "Using vswing level %d%s\n", train_set & DP_TRAIN_VOLTAGE_SWING_MASK, @@ -4182,55 +4381,28 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp) train_set & DP_TRAIN_MAX_PRE_EMPHASIS_REACHED ? " (max)" : ""); - intel_dp->DP = (intel_dp->DP & ~mask) | signal_levels; - - intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); - intel_de_posting_read(dev_priv, intel_dp->output_reg); + intel_dp->set_signal_levels(intel_dp); } void intel_dp_program_link_training_pattern(struct intel_dp *intel_dp, u8 dp_train_pat) { - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = - to_i915(intel_dig_port->base.base.dev); + struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + u8 train_pat_mask = drm_dp_training_pattern_mask(intel_dp->dpcd); - _intel_dp_set_link_train(intel_dp, &intel_dp->DP, dp_train_pat); + if (dp_train_pat & train_pat_mask) + drm_dbg_kms(&dev_priv->drm, + "Using DP training pattern TPS%d\n", + dp_train_pat & train_pat_mask); - intel_de_write(dev_priv, intel_dp->output_reg, intel_dp->DP); - intel_de_posting_read(dev_priv, intel_dp->output_reg); + intel_dp->set_link_train(intel_dp, dp_train_pat); } void intel_dp_set_idle_link_train(struct intel_dp *intel_dp) { - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - enum port port = intel_dig_port->base.port; - u32 val; - - if (!HAS_DDI(dev_priv)) - return; - - val = intel_de_read(dev_priv, intel_dp->regs.dp_tp_ctl); - val &= ~DP_TP_CTL_LINK_TRAIN_MASK; - val |= DP_TP_CTL_LINK_TRAIN_IDLE; - intel_de_write(dev_priv, intel_dp->regs.dp_tp_ctl, val); - - /* - * Until TGL on PORT_A we can have only eDP in SST mode. There the only - * reason we need to set idle transmission mode is to work around a HW - * issue where we enable the pipe while not in idle link-training mode. - * In this case there is requirement to wait for a minimum number of - * idle patterns to be sent. - */ - if (port == PORT_A && INTEL_GEN(dev_priv) < 12) - return; - - if (intel_de_wait_for_set(dev_priv, intel_dp->regs.dp_tp_status, - DP_TP_STATUS_IDLE_DONE, 1)) - drm_err(&dev_priv->drm, - "Timed out waiting for DP idle patterns\n"); + if (intel_dp->set_idle_link_train) + intel_dp->set_idle_link_train(intel_dp); } static void @@ -4243,7 +4415,9 @@ intel_dp_link_down(struct intel_encoder *encoder, enum port port = encoder->port; u32 DP = intel_dp->DP; - if (WARN_ON((intel_de_read(dev_priv, intel_dp->output_reg) & DP_PORT_EN) == 0)) + if (drm_WARN_ON(&dev_priv->drm, + (intel_de_read(dev_priv, intel_dp->output_reg) & + DP_PORT_EN) == 0)) return; drm_dbg_kms(&dev_priv->drm, "\n"); @@ -4307,6 +4481,7 @@ intel_dp_link_down(struct intel_encoder *encoder, static void intel_dp_extended_receiver_capabilities(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 dpcd_ext[6]; /* @@ -4322,20 +4497,22 @@ intel_dp_extended_receiver_capabilities(struct intel_dp *intel_dp) if (drm_dp_dpcd_read(&intel_dp->aux, DP_DP13_DPCD_REV, &dpcd_ext, sizeof(dpcd_ext)) != sizeof(dpcd_ext)) { - DRM_ERROR("DPCD failed read at extended capabilities\n"); + drm_err(&i915->drm, + "DPCD failed read at extended capabilities\n"); return; } if (intel_dp->dpcd[DP_DPCD_REV] > dpcd_ext[DP_DPCD_REV]) { - DRM_DEBUG_KMS("DPCD extended DPCD rev less than base DPCD rev\n"); + drm_dbg_kms(&i915->drm, + "DPCD extended DPCD rev less than base DPCD rev\n"); return; } if (!memcmp(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext))) return; - DRM_DEBUG_KMS("Base DPCD: %*ph\n", - (int)sizeof(intel_dp->dpcd), intel_dp->dpcd); + drm_dbg_kms(&i915->drm, "Base DPCD: %*ph\n", + (int)sizeof(intel_dp->dpcd), intel_dp->dpcd); memcpy(intel_dp->dpcd, dpcd_ext, sizeof(dpcd_ext)); } @@ -4343,13 +4520,16 @@ intel_dp_extended_receiver_capabilities(struct intel_dp *intel_dp) bool intel_dp_read_dpcd(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd, sizeof(intel_dp->dpcd)) < 0) return false; /* aux transfer failed */ intel_dp_extended_receiver_capabilities(intel_dp); - DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd); + drm_dbg_kms(&i915->drm, "DPCD: %*ph\n", (int)sizeof(intel_dp->dpcd), + intel_dp->dpcd); return intel_dp->dpcd[DP_DPCD_REV] != 0; } @@ -4366,6 +4546,8 @@ bool intel_dp_get_colorimetry_status(struct intel_dp *intel_dp) static void intel_dp_get_dsc_sink_cap(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + /* * Clear the cached register set to avoid using stale values * for the sinks that do not support DSC. @@ -4381,20 +4563,23 @@ static void intel_dp_get_dsc_sink_cap(struct intel_dp *intel_dp) if (drm_dp_dpcd_read(&intel_dp->aux, DP_DSC_SUPPORT, intel_dp->dsc_dpcd, sizeof(intel_dp->dsc_dpcd)) < 0) - DRM_ERROR("Failed to read DPCD register 0x%x\n", - DP_DSC_SUPPORT); + drm_err(&i915->drm, + "Failed to read DPCD register 0x%x\n", + DP_DSC_SUPPORT); - DRM_DEBUG_KMS("DSC DPCD: %*ph\n", - (int)sizeof(intel_dp->dsc_dpcd), - intel_dp->dsc_dpcd); + drm_dbg_kms(&i915->drm, "DSC DPCD: %*ph\n", + (int)sizeof(intel_dp->dsc_dpcd), + intel_dp->dsc_dpcd); /* FEC is supported only on DP 1.4 */ if (!intel_dp_is_edp(intel_dp) && drm_dp_dpcd_readb(&intel_dp->aux, DP_FEC_CAPABILITY, &intel_dp->fec_capable) < 0) - DRM_ERROR("Failed to read FEC DPCD register\n"); + drm_err(&i915->drm, + "Failed to read FEC DPCD register\n"); - DRM_DEBUG_KMS("FEC CAPABILITY: %x\n", intel_dp->fec_capable); + drm_dbg_kms(&i915->drm, "FEC CAPABILITY: %x\n", + intel_dp->fec_capable); } } @@ -4405,7 +4590,7 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp) to_i915(dp_to_dig_port(intel_dp)->base.base.dev); /* this function is meant to be called only once */ - WARN_ON(intel_dp->dpcd[DP_DPCD_REV] != 0); + drm_WARN_ON(&dev_priv->drm, intel_dp->dpcd[DP_DPCD_REV] != 0); if (!intel_dp_read_dpcd(intel_dp)) return false; @@ -4502,7 +4687,8 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp) * it don't care about read it here and in intel_edp_init_dpcd(). */ if (!intel_dp_is_edp(intel_dp) && - !drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_SINK_COUNT)) { + !drm_dp_has_quirk(&intel_dp->desc, 0, + DP_DPCD_QUIRK_NO_SINK_COUNT)) { u8 count; ssize_t r; @@ -4567,14 +4753,16 @@ intel_dp_can_mst(struct intel_dp *intel_dp) static void intel_dp_configure_mst(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base; bool sink_can_mst = intel_dp_sink_can_mst(intel_dp); - DRM_DEBUG_KMS("[ENCODER:%d:%s] MST support: port: %s, sink: %s, modparam: %s\n", - encoder->base.base.id, encoder->base.name, - yesno(intel_dp->can_mst), yesno(sink_can_mst), - yesno(i915_modparams.enable_dp_mst)); + drm_dbg_kms(&i915->drm, + "[ENCODER:%d:%s] MST support: port: %s, sink: %s, modparam: %s\n", + encoder->base.base.id, encoder->base.name, + yesno(intel_dp->can_mst), yesno(sink_can_mst), + yesno(i915_modparams.enable_dp_mst)); if (!intel_dp->can_mst) return; @@ -4620,158 +4808,92 @@ intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state, return false; } -static void -intel_dp_setup_vsc_sdp(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static ssize_t intel_dp_vsc_sdp_pack(const struct drm_dp_vsc_sdp *vsc, + struct dp_sdp *sdp, size_t size) { - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct dp_sdp vsc_sdp = {}; + size_t length = sizeof(struct dp_sdp); - /* Prepare VSC Header for SU as per DP 1.4a spec, Table 2-119 */ - vsc_sdp.sdp_header.HB0 = 0; - vsc_sdp.sdp_header.HB1 = 0x7; + if (size < length) + return -ENOSPC; + + memset(sdp, 0, size); /* - * VSC SDP supporting 3D stereo, PSR2, and Pixel Encoding/ - * Colorimetry Format indication. + * Prepare VSC Header for SU as per DP 1.4a spec, Table 2-119 + * VSC SDP Header Bytes */ - vsc_sdp.sdp_header.HB2 = 0x5; + sdp->sdp_header.HB0 = 0; /* Secondary-Data Packet ID = 0 */ + sdp->sdp_header.HB1 = vsc->sdp_type; /* Secondary-data Packet Type */ + sdp->sdp_header.HB2 = vsc->revision; /* Revision Number */ + sdp->sdp_header.HB3 = vsc->length; /* Number of Valid Data Bytes */ /* - * VSC SDP supporting 3D stereo, + PSR2, + Pixel Encoding/ - * Colorimetry Format indication (HB2 = 05h). + * Only revision 0x5 supports Pixel Encoding/Colorimetry Format as + * per DP 1.4a spec. */ - vsc_sdp.sdp_header.HB3 = 0x13; - - /* DP 1.4a spec, Table 2-120 */ - switch (crtc_state->output_format) { - case INTEL_OUTPUT_FORMAT_YCBCR444: - vsc_sdp.db[16] = 0x1 << 4; /* YCbCr 444 : DB16[7:4] = 1h */ - break; - case INTEL_OUTPUT_FORMAT_YCBCR420: - vsc_sdp.db[16] = 0x3 << 4; /* YCbCr 420 : DB16[7:4] = 3h */ - break; - case INTEL_OUTPUT_FORMAT_RGB: - default: - /* RGB: DB16[7:4] = 0h */ - break; - } + if (vsc->revision != 0x5) + goto out; - switch (conn_state->colorspace) { - case DRM_MODE_COLORIMETRY_BT709_YCC: - vsc_sdp.db[16] |= 0x1; - break; - case DRM_MODE_COLORIMETRY_XVYCC_601: - vsc_sdp.db[16] |= 0x2; - break; - case DRM_MODE_COLORIMETRY_XVYCC_709: - vsc_sdp.db[16] |= 0x3; - break; - case DRM_MODE_COLORIMETRY_SYCC_601: - vsc_sdp.db[16] |= 0x4; - break; - case DRM_MODE_COLORIMETRY_OPYCC_601: - vsc_sdp.db[16] |= 0x5; - break; - case DRM_MODE_COLORIMETRY_BT2020_CYCC: - case DRM_MODE_COLORIMETRY_BT2020_RGB: - vsc_sdp.db[16] |= 0x6; - break; - case DRM_MODE_COLORIMETRY_BT2020_YCC: - vsc_sdp.db[16] |= 0x7; - break; - case DRM_MODE_COLORIMETRY_DCI_P3_RGB_D65: - case DRM_MODE_COLORIMETRY_DCI_P3_RGB_THEATER: - vsc_sdp.db[16] |= 0x4; /* DCI-P3 (SMPTE RP 431-2) */ - break; - default: - /* sRGB (IEC 61966-2-1) / ITU-R BT.601: DB16[0:3] = 0h */ + /* VSC SDP Payload for DB16 through DB18 */ + /* Pixel Encoding and Colorimetry Formats */ + sdp->db[16] = (vsc->pixelformat & 0xf) << 4; /* DB16[7:4] */ + sdp->db[16] |= vsc->colorimetry & 0xf; /* DB16[3:0] */ - /* RGB->YCBCR color conversion uses the BT.709 color space. */ - if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) - vsc_sdp.db[16] |= 0x1; /* 0x1, ITU-R BT.709 */ + switch (vsc->bpc) { + case 6: + /* 6bpc: 0x0 */ break; - } - - /* - * For pixel encoding formats YCbCr444, YCbCr422, YCbCr420, and Y Only, - * the following Component Bit Depth values are defined: - * 001b = 8bpc. - * 010b = 10bpc. - * 011b = 12bpc. - * 100b = 16bpc. - */ - switch (crtc_state->pipe_bpp) { - case 24: /* 8bpc */ - vsc_sdp.db[17] = 0x1; + case 8: + sdp->db[17] = 0x1; /* DB17[3:0] */ break; - case 30: /* 10bpc */ - vsc_sdp.db[17] = 0x2; + case 10: + sdp->db[17] = 0x2; break; - case 36: /* 12bpc */ - vsc_sdp.db[17] = 0x3; + case 12: + sdp->db[17] = 0x3; break; - case 48: /* 16bpc */ - vsc_sdp.db[17] = 0x4; + case 16: + sdp->db[17] = 0x4; break; default: - MISSING_CASE(crtc_state->pipe_bpp); + MISSING_CASE(vsc->bpc); break; } + /* Dynamic Range and Component Bit Depth */ + if (vsc->dynamic_range == DP_DYNAMIC_RANGE_CTA) + sdp->db[17] |= 0x80; /* DB17[7] */ - /* - * Dynamic Range (Bit 7) - * 0 = VESA range, 1 = CTA range. - * all YCbCr are always limited range - */ - vsc_sdp.db[17] |= 0x80; + /* Content Type */ + sdp->db[18] = vsc->content_type & 0x7; - /* - * Content Type (Bits 2:0) - * 000b = Not defined. - * 001b = Graphics. - * 010b = Photo. - * 011b = Video. - * 100b = Game - * All other values are RESERVED. - * Note: See CTA-861-G for the definition and expected - * processing by a stream sink for the above contect types. - */ - vsc_sdp.db[18] = 0; - - intel_dig_port->write_infoframe(&intel_dig_port->base, - crtc_state, DP_SDP_VSC, &vsc_sdp, sizeof(vsc_sdp)); +out: + return length; } -static void -intel_dp_setup_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static ssize_t +intel_dp_hdr_metadata_infoframe_sdp_pack(const struct hdmi_drm_infoframe *drm_infoframe, + struct dp_sdp *sdp, + size_t size) { - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct dp_sdp infoframe_sdp = {}; - struct hdmi_drm_infoframe drm_infoframe = {}; + size_t length = sizeof(struct dp_sdp); const int infoframe_size = HDMI_INFOFRAME_HEADER_SIZE + HDMI_DRM_INFOFRAME_SIZE; unsigned char buf[HDMI_INFOFRAME_HEADER_SIZE + HDMI_DRM_INFOFRAME_SIZE]; ssize_t len; - int ret; - ret = drm_hdmi_infoframe_set_hdr_metadata(&drm_infoframe, conn_state); - if (ret) { - DRM_DEBUG_KMS("couldn't set HDR metadata in infoframe\n"); - return; - } + if (size < length) + return -ENOSPC; + + memset(sdp, 0, size); - len = hdmi_drm_infoframe_pack_only(&drm_infoframe, buf, sizeof(buf)); + len = hdmi_drm_infoframe_pack_only(drm_infoframe, buf, sizeof(buf)); if (len < 0) { DRM_DEBUG_KMS("buffer size is smaller than hdr metadata infoframe\n"); - return; + return -ENOSPC; } if (len != infoframe_size) { DRM_DEBUG_KMS("wrong static hdr metadata size\n"); - return; + return -ENOSPC; } /* @@ -4780,34 +4902,37 @@ intel_dp_setup_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp, * Table 2-100 and Table 2-101 */ - /* Packet ID, 00h for non-Audio INFOFRAME */ - infoframe_sdp.sdp_header.HB0 = 0; + /* Secondary-Data Packet ID, 00h for non-Audio INFOFRAME */ + sdp->sdp_header.HB0 = 0; /* * Packet Type 80h + Non-audio INFOFRAME Type value - * HDMI_INFOFRAME_TYPE_DRM: 0x87, + * HDMI_INFOFRAME_TYPE_DRM: 0x87 + * - 80h + Non-audio INFOFRAME Type value + * - InfoFrame Type: 0x07 + * [CTA-861-G Table-42 Dynamic Range and Mastering InfoFrame] */ - infoframe_sdp.sdp_header.HB1 = drm_infoframe.type; + sdp->sdp_header.HB1 = drm_infoframe->type; /* * Least Significant Eight Bits of (Data Byte Count – 1) - * infoframe_size - 1, + * infoframe_size - 1 */ - infoframe_sdp.sdp_header.HB2 = 0x1D; + sdp->sdp_header.HB2 = 0x1D; /* INFOFRAME SDP Version Number */ - infoframe_sdp.sdp_header.HB3 = (0x13 << 2); + sdp->sdp_header.HB3 = (0x13 << 2); /* CTA Header Byte 2 (INFOFRAME Version Number) */ - infoframe_sdp.db[0] = drm_infoframe.version; + sdp->db[0] = drm_infoframe->version; /* CTA Header Byte 3 (Length of INFOFRAME): HDMI_DRM_INFOFRAME_SIZE */ - infoframe_sdp.db[1] = drm_infoframe.length; + sdp->db[1] = drm_infoframe->length; /* * Copy HDMI_DRM_INFOFRAME_SIZE size from a buffer after * HDMI_INFOFRAME_HEADER_SIZE */ - BUILD_BUG_ON(sizeof(infoframe_sdp.db) < HDMI_DRM_INFOFRAME_SIZE + 2); - memcpy(&infoframe_sdp.db[2], &buf[HDMI_INFOFRAME_HEADER_SIZE], + BUILD_BUG_ON(sizeof(sdp->db) < HDMI_DRM_INFOFRAME_SIZE + 2); + memcpy(&sdp->db[2], &buf[HDMI_INFOFRAME_HEADER_SIZE], HDMI_DRM_INFOFRAME_SIZE); /* - * Size of DP infoframe sdp packet for HDR static metadata is consist of + * Size of DP infoframe sdp packet for HDR static metadata consists of * - DP SDP Header(struct dp_sdp_header): 4 bytes * - Two Data Blocks: 2 bytes * CTA Header Byte2 (INFOFRAME Version Number) @@ -4818,36 +4943,286 @@ intel_dp_setup_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp, * infoframe size. But GEN11+ has larger than that size, write_infoframe * will pad rest of the size. */ - intel_dig_port->write_infoframe(&intel_dig_port->base, crtc_state, - HDMI_PACKET_TYPE_GAMUT_METADATA, - &infoframe_sdp, - sizeof(struct dp_sdp_header) + 2 + HDMI_DRM_INFOFRAME_SIZE); + return sizeof(struct dp_sdp_header) + 2 + HDMI_DRM_INFOFRAME_SIZE; } -void intel_dp_vsc_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +static void intel_write_dp_sdp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + unsigned int type) { - if (!intel_dp_needs_vsc_sdp(crtc_state, conn_state)) + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct dp_sdp sdp = {}; + ssize_t len; + + if ((crtc_state->infoframes.enable & + intel_hdmi_infoframe_enable(type)) == 0) return; - intel_dp_setup_vsc_sdp(intel_dp, crtc_state, conn_state); + switch (type) { + case DP_SDP_VSC: + len = intel_dp_vsc_sdp_pack(&crtc_state->infoframes.vsc, &sdp, + sizeof(sdp)); + break; + case HDMI_PACKET_TYPE_GAMUT_METADATA: + len = intel_dp_hdr_metadata_infoframe_sdp_pack(&crtc_state->infoframes.drm.drm, + &sdp, sizeof(sdp)); + break; + default: + MISSING_CASE(type); + return; + } + + if (drm_WARN_ON(&dev_priv->drm, len < 0)) + return; + + intel_dig_port->write_infoframe(encoder, crtc_state, type, &sdp, len); } -void intel_dp_hdr_metadata_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state) +void intel_write_dp_vsc_sdp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + struct drm_dp_vsc_sdp *vsc) { - if (!conn_state->hdr_output_metadata) + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct dp_sdp sdp = {}; + ssize_t len; + + len = intel_dp_vsc_sdp_pack(vsc, &sdp, sizeof(sdp)); + + if (drm_WARN_ON(&dev_priv->drm, len < 0)) return; - intel_dp_setup_hdr_metadata_infoframe_sdp(intel_dp, - crtc_state, - conn_state); + intel_dig_port->write_infoframe(encoder, crtc_state, DP_SDP_VSC, + &sdp, len); +} + +void intel_dp_set_infoframes(struct intel_encoder *encoder, + bool enable, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) +{ + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + i915_reg_t reg = HSW_TVIDEO_DIP_CTL(crtc_state->cpu_transcoder); + u32 dip_enable = VIDEO_DIP_ENABLE_AVI_HSW | VIDEO_DIP_ENABLE_GCP_HSW | + VIDEO_DIP_ENABLE_VS_HSW | VIDEO_DIP_ENABLE_GMP_HSW | + VIDEO_DIP_ENABLE_SPD_HSW | VIDEO_DIP_ENABLE_DRM_GLK; + u32 val = intel_de_read(dev_priv, reg); + + /* TODO: Add DSC case (DIP_ENABLE_PPS) */ + /* When PSR is enabled, this routine doesn't disable VSC DIP */ + if (intel_psr_enabled(intel_dp)) + val &= ~dip_enable; + else + val &= ~(dip_enable | VIDEO_DIP_ENABLE_VSC_HSW); + + if (!enable) { + intel_de_write(dev_priv, reg, val); + intel_de_posting_read(dev_priv, reg); + return; + } + + intel_de_write(dev_priv, reg, val); + intel_de_posting_read(dev_priv, reg); + + /* When PSR is enabled, VSC SDP is handled by PSR routine */ + if (!intel_psr_enabled(intel_dp)) + intel_write_dp_sdp(encoder, crtc_state, DP_SDP_VSC); + + intel_write_dp_sdp(encoder, crtc_state, HDMI_PACKET_TYPE_GAMUT_METADATA); +} + +static int intel_dp_vsc_sdp_unpack(struct drm_dp_vsc_sdp *vsc, + const void *buffer, size_t size) +{ + const struct dp_sdp *sdp = buffer; + + if (size < sizeof(struct dp_sdp)) + return -EINVAL; + + memset(vsc, 0, size); + + if (sdp->sdp_header.HB0 != 0) + return -EINVAL; + + if (sdp->sdp_header.HB1 != DP_SDP_VSC) + return -EINVAL; + + vsc->sdp_type = sdp->sdp_header.HB1; + vsc->revision = sdp->sdp_header.HB2; + vsc->length = sdp->sdp_header.HB3; + + if ((sdp->sdp_header.HB2 == 0x2 && sdp->sdp_header.HB3 == 0x8) || + (sdp->sdp_header.HB2 == 0x4 && sdp->sdp_header.HB3 == 0xe)) { + /* + * - HB2 = 0x2, HB3 = 0x8 + * VSC SDP supporting 3D stereo + PSR + * - HB2 = 0x4, HB3 = 0xe + * VSC SDP supporting 3D stereo + PSR2 with Y-coordinate of + * first scan line of the SU region (applies to eDP v1.4b + * and higher). + */ + return 0; + } else if (sdp->sdp_header.HB2 == 0x5 && sdp->sdp_header.HB3 == 0x13) { + /* + * - HB2 = 0x5, HB3 = 0x13 + * VSC SDP supporting 3D stereo + PSR2 + Pixel Encoding/Colorimetry + * Format. + */ + vsc->pixelformat = (sdp->db[16] >> 4) & 0xf; + vsc->colorimetry = sdp->db[16] & 0xf; + vsc->dynamic_range = (sdp->db[17] >> 7) & 0x1; + + switch (sdp->db[17] & 0x7) { + case 0x0: + vsc->bpc = 6; + break; + case 0x1: + vsc->bpc = 8; + break; + case 0x2: + vsc->bpc = 10; + break; + case 0x3: + vsc->bpc = 12; + break; + case 0x4: + vsc->bpc = 16; + break; + default: + MISSING_CASE(sdp->db[17] & 0x7); + return -EINVAL; + } + + vsc->content_type = sdp->db[18] & 0x7; + } else { + return -EINVAL; + } + + return 0; +} + +static int +intel_dp_hdr_metadata_infoframe_sdp_unpack(struct hdmi_drm_infoframe *drm_infoframe, + const void *buffer, size_t size) +{ + int ret; + + const struct dp_sdp *sdp = buffer; + + if (size < sizeof(struct dp_sdp)) + return -EINVAL; + + if (sdp->sdp_header.HB0 != 0) + return -EINVAL; + + if (sdp->sdp_header.HB1 != HDMI_INFOFRAME_TYPE_DRM) + return -EINVAL; + + /* + * Least Significant Eight Bits of (Data Byte Count – 1) + * 1Dh (i.e., Data Byte Count = 30 bytes). + */ + if (sdp->sdp_header.HB2 != 0x1D) + return -EINVAL; + + /* Most Significant Two Bits of (Data Byte Count – 1), Clear to 00b. */ + if ((sdp->sdp_header.HB3 & 0x3) != 0) + return -EINVAL; + + /* INFOFRAME SDP Version Number */ + if (((sdp->sdp_header.HB3 >> 2) & 0x3f) != 0x13) + return -EINVAL; + + /* CTA Header Byte 2 (INFOFRAME Version Number) */ + if (sdp->db[0] != 1) + return -EINVAL; + + /* CTA Header Byte 3 (Length of INFOFRAME): HDMI_DRM_INFOFRAME_SIZE */ + if (sdp->db[1] != HDMI_DRM_INFOFRAME_SIZE) + return -EINVAL; + + ret = hdmi_drm_infoframe_unpack_only(drm_infoframe, &sdp->db[2], + HDMI_DRM_INFOFRAME_SIZE); + + return ret; +} + +static void intel_read_dp_vsc_sdp(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state, + struct drm_dp_vsc_sdp *vsc) +{ + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct intel_dp *intel_dp = enc_to_intel_dp(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + unsigned int type = DP_SDP_VSC; + struct dp_sdp sdp = {}; + int ret; + + /* When PSR is enabled, VSC SDP is handled by PSR routine */ + if (intel_psr_enabled(intel_dp)) + return; + + if ((crtc_state->infoframes.enable & + intel_hdmi_infoframe_enable(type)) == 0) + return; + + intel_dig_port->read_infoframe(encoder, crtc_state, type, &sdp, sizeof(sdp)); + + ret = intel_dp_vsc_sdp_unpack(vsc, &sdp, sizeof(sdp)); + + if (ret) + drm_dbg_kms(&dev_priv->drm, "Failed to unpack DP VSC SDP\n"); +} + +static void intel_read_dp_hdr_metadata_infoframe_sdp(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state, + struct hdmi_drm_infoframe *drm_infoframe) +{ + struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder); + struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + unsigned int type = HDMI_PACKET_TYPE_GAMUT_METADATA; + struct dp_sdp sdp = {}; + int ret; + + if ((crtc_state->infoframes.enable & + intel_hdmi_infoframe_enable(type)) == 0) + return; + + intel_dig_port->read_infoframe(encoder, crtc_state, type, &sdp, + sizeof(sdp)); + + ret = intel_dp_hdr_metadata_infoframe_sdp_unpack(drm_infoframe, &sdp, + sizeof(sdp)); + + if (ret) + drm_dbg_kms(&dev_priv->drm, + "Failed to unpack DP HDR Metadata Infoframe SDP\n"); +} + +void intel_read_dp_sdp(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state, + unsigned int type) +{ + switch (type) { + case DP_SDP_VSC: + intel_read_dp_vsc_sdp(encoder, crtc_state, + &crtc_state->infoframes.vsc); + break; + case HDMI_PACKET_TYPE_GAMUT_METADATA: + intel_read_dp_hdr_metadata_infoframe_sdp(encoder, crtc_state, + &crtc_state->infoframes.drm.drm); + break; + default: + MISSING_CASE(type); + break; + } } static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); int status = 0; int test_link_rate; u8 test_lane_count, test_link_bw; @@ -4859,7 +5234,7 @@ static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp) &test_lane_count); if (status <= 0) { - DRM_DEBUG_KMS("Lane count read failed\n"); + drm_dbg_kms(&i915->drm, "Lane count read failed\n"); return DP_TEST_NAK; } test_lane_count &= DP_MAX_LANE_COUNT_MASK; @@ -4867,7 +5242,7 @@ static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp) status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_LINK_RATE, &test_link_bw); if (status <= 0) { - DRM_DEBUG_KMS("Link Rate read failed\n"); + drm_dbg_kms(&i915->drm, "Link Rate read failed\n"); return DP_TEST_NAK; } test_link_rate = drm_dp_bw_code_to_link_rate(test_link_bw); @@ -4885,6 +5260,7 @@ static u8 intel_dp_autotest_link_training(struct intel_dp *intel_dp) static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 test_pattern; u8 test_misc; __be16 h_width, v_height; @@ -4894,7 +5270,7 @@ static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_PATTERN, &test_pattern); if (status <= 0) { - DRM_DEBUG_KMS("Test pattern read failed\n"); + drm_dbg_kms(&i915->drm, "Test pattern read failed\n"); return DP_TEST_NAK; } if (test_pattern != DP_COLOR_RAMP) @@ -4903,21 +5279,21 @@ static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_H_WIDTH_HI, &h_width, 2); if (status <= 0) { - DRM_DEBUG_KMS("H Width read failed\n"); + drm_dbg_kms(&i915->drm, "H Width read failed\n"); return DP_TEST_NAK; } status = drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_V_HEIGHT_HI, &v_height, 2); if (status <= 0) { - DRM_DEBUG_KMS("V Height read failed\n"); + drm_dbg_kms(&i915->drm, "V Height read failed\n"); return DP_TEST_NAK; } status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_MISC0, &test_misc); if (status <= 0) { - DRM_DEBUG_KMS("TEST MISC read failed\n"); + drm_dbg_kms(&i915->drm, "TEST MISC read failed\n"); return DP_TEST_NAK; } if ((test_misc & DP_TEST_COLOR_FORMAT_MASK) != DP_COLOR_FORMAT_RGB) @@ -4946,6 +5322,7 @@ static u8 intel_dp_autotest_video_pattern(struct intel_dp *intel_dp) static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 test_result = DP_TEST_ACK; struct intel_connector *intel_connector = intel_dp->attached_connector; struct drm_connector *connector = &intel_connector->base; @@ -4962,9 +5339,10 @@ static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp) */ if (intel_dp->aux.i2c_nack_count > 0 || intel_dp->aux.i2c_defer_count > 0) - DRM_DEBUG_KMS("EDID read had %d NACKs, %d DEFERs\n", - intel_dp->aux.i2c_nack_count, - intel_dp->aux.i2c_defer_count); + drm_dbg_kms(&i915->drm, + "EDID read had %d NACKs, %d DEFERs\n", + intel_dp->aux.i2c_nack_count, + intel_dp->aux.i2c_defer_count); intel_dp->compliance.test_data.edid = INTEL_DP_RESOLUTION_FAILSAFE; } else { struct edid *block = intel_connector->detect_edid; @@ -4976,7 +5354,8 @@ static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp) if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_EDID_CHECKSUM, block->checksum) <= 0) - DRM_DEBUG_KMS("Failed to write EDID checksum\n"); + drm_dbg_kms(&i915->drm, + "Failed to write EDID checksum\n"); test_result = DP_TEST_ACK | DP_TEST_EDID_CHECKSUM_WRITE; intel_dp->compliance.test_data.edid = INTEL_DP_RESOLUTION_PREFERRED; @@ -4988,43 +5367,217 @@ static u8 intel_dp_autotest_edid(struct intel_dp *intel_dp) return test_result; } +static u8 intel_dp_prepare_phytest(struct intel_dp *intel_dp) +{ + struct drm_dp_phy_test_params *data = + &intel_dp->compliance.test_data.phytest; + + if (drm_dp_get_phy_test_pattern(&intel_dp->aux, data)) { + DRM_DEBUG_KMS("DP Phy Test pattern AUX read failure\n"); + return DP_TEST_NAK; + } + + /* + * link_mst is set to false to avoid executing mst related code + * during compliance testing. + */ + intel_dp->link_mst = false; + + return DP_TEST_ACK; +} + +static void intel_dp_phy_pattern_update(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv = + to_i915(dp_to_dig_port(intel_dp)->base.base.dev); + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_dp_phy_test_params *data = + &intel_dp->compliance.test_data.phytest; + struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); + enum pipe pipe = crtc->pipe; + u32 pattern_val; + + switch (data->phy_pattern) { + case DP_PHY_TEST_PATTERN_NONE: + DRM_DEBUG_KMS("Disable Phy Test Pattern\n"); + intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), 0x0); + break; + case DP_PHY_TEST_PATTERN_D10_2: + DRM_DEBUG_KMS("Set D10.2 Phy Test Pattern\n"); + intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), + DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_D10_2); + break; + case DP_PHY_TEST_PATTERN_ERROR_COUNT: + DRM_DEBUG_KMS("Set Error Count Phy Test Pattern\n"); + intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), + DDI_DP_COMP_CTL_ENABLE | + DDI_DP_COMP_CTL_SCRAMBLED_0); + break; + case DP_PHY_TEST_PATTERN_PRBS7: + DRM_DEBUG_KMS("Set PRBS7 Phy Test Pattern\n"); + intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), + DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_PRBS7); + break; + case DP_PHY_TEST_PATTERN_80BIT_CUSTOM: + /* + * FIXME: Ideally pattern should come from DPCD 0x250. As + * current firmware of DPR-100 could not set it, so hardcoding + * now for complaince test. + */ + DRM_DEBUG_KMS("Set 80Bit Custom Phy Test Pattern 0x3e0f83e0 0x0f83e0f8 0x0000f83e\n"); + pattern_val = 0x3e0f83e0; + intel_de_write(dev_priv, DDI_DP_COMP_PAT(pipe, 0), pattern_val); + pattern_val = 0x0f83e0f8; + intel_de_write(dev_priv, DDI_DP_COMP_PAT(pipe, 1), pattern_val); + pattern_val = 0x0000f83e; + intel_de_write(dev_priv, DDI_DP_COMP_PAT(pipe, 2), pattern_val); + intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), + DDI_DP_COMP_CTL_ENABLE | + DDI_DP_COMP_CTL_CUSTOM80); + break; + case DP_PHY_TEST_PATTERN_CP2520: + /* + * FIXME: Ideally pattern should come from DPCD 0x24A. As + * current firmware of DPR-100 could not set it, so hardcoding + * now for complaince test. + */ + DRM_DEBUG_KMS("Set HBR2 compliance Phy Test Pattern\n"); + pattern_val = 0xFB; + intel_de_write(dev_priv, DDI_DP_COMP_CTL(pipe), + DDI_DP_COMP_CTL_ENABLE | DDI_DP_COMP_CTL_HBR2 | + pattern_val); + break; + default: + WARN(1, "Invalid Phy Test Pattern\n"); + } +} + +static void +intel_dp_autotest_phy_ddi_disable(struct intel_dp *intel_dp) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); + enum pipe pipe = crtc->pipe; + u32 trans_ddi_func_ctl_value, trans_conf_value, dp_tp_ctl_value; + + trans_ddi_func_ctl_value = intel_de_read(dev_priv, + TRANS_DDI_FUNC_CTL(pipe)); + trans_conf_value = intel_de_read(dev_priv, PIPECONF(pipe)); + dp_tp_ctl_value = intel_de_read(dev_priv, TGL_DP_TP_CTL(pipe)); + + trans_ddi_func_ctl_value &= ~(TRANS_DDI_FUNC_ENABLE | + TGL_TRANS_DDI_PORT_MASK); + trans_conf_value &= ~PIPECONF_ENABLE; + dp_tp_ctl_value &= ~DP_TP_CTL_ENABLE; + + intel_de_write(dev_priv, PIPECONF(pipe), trans_conf_value); + intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(pipe), + trans_ddi_func_ctl_value); + intel_de_write(dev_priv, TGL_DP_TP_CTL(pipe), dp_tp_ctl_value); +} + +static void +intel_dp_autotest_phy_ddi_enable(struct intel_dp *intel_dp, uint8_t lane_cnt) +{ + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct drm_device *dev = intel_dig_port->base.base.dev; + struct drm_i915_private *dev_priv = to_i915(dev); + enum port port = intel_dig_port->base.port; + struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc); + enum pipe pipe = crtc->pipe; + u32 trans_ddi_func_ctl_value, trans_conf_value, dp_tp_ctl_value; + + trans_ddi_func_ctl_value = intel_de_read(dev_priv, + TRANS_DDI_FUNC_CTL(pipe)); + trans_conf_value = intel_de_read(dev_priv, PIPECONF(pipe)); + dp_tp_ctl_value = intel_de_read(dev_priv, TGL_DP_TP_CTL(pipe)); + + trans_ddi_func_ctl_value |= TRANS_DDI_FUNC_ENABLE | + TGL_TRANS_DDI_SELECT_PORT(port); + trans_conf_value |= PIPECONF_ENABLE; + dp_tp_ctl_value |= DP_TP_CTL_ENABLE; + + intel_de_write(dev_priv, PIPECONF(pipe), trans_conf_value); + intel_de_write(dev_priv, TGL_DP_TP_CTL(pipe), dp_tp_ctl_value); + intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(pipe), + trans_ddi_func_ctl_value); +} + +void intel_dp_process_phy_request(struct intel_dp *intel_dp) +{ + struct drm_dp_phy_test_params *data = + &intel_dp->compliance.test_data.phytest; + u8 link_status[DP_LINK_STATUS_SIZE]; + + if (!intel_dp_get_link_status(intel_dp, link_status)) { + DRM_DEBUG_KMS("failed to get link status\n"); + return; + } + + /* retrieve vswing & pre-emphasis setting */ + intel_dp_get_adjust_train(intel_dp, link_status); + + intel_dp_autotest_phy_ddi_disable(intel_dp); + + intel_dp_set_signal_levels(intel_dp); + + intel_dp_phy_pattern_update(intel_dp); + + intel_dp_autotest_phy_ddi_enable(intel_dp, data->num_lanes); + + drm_dp_set_phy_test_pattern(&intel_dp->aux, data, + link_status[DP_DPCD_REV]); +} + static u8 intel_dp_autotest_phy_pattern(struct intel_dp *intel_dp) { - u8 test_result = DP_TEST_NAK; + u8 test_result; + + test_result = intel_dp_prepare_phytest(intel_dp); + if (test_result != DP_TEST_ACK) + DRM_ERROR("Phy test preparation failed\n"); + + intel_dp_process_phy_request(intel_dp); + return test_result; } static void intel_dp_handle_test_request(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 response = DP_TEST_NAK; u8 request = 0; int status; status = drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_REQUEST, &request); if (status <= 0) { - DRM_DEBUG_KMS("Could not read test request from sink\n"); + drm_dbg_kms(&i915->drm, + "Could not read test request from sink\n"); goto update_status; } switch (request) { case DP_TEST_LINK_TRAINING: - DRM_DEBUG_KMS("LINK_TRAINING test requested\n"); + drm_dbg_kms(&i915->drm, "LINK_TRAINING test requested\n"); response = intel_dp_autotest_link_training(intel_dp); break; case DP_TEST_LINK_VIDEO_PATTERN: - DRM_DEBUG_KMS("TEST_PATTERN test requested\n"); + drm_dbg_kms(&i915->drm, "TEST_PATTERN test requested\n"); response = intel_dp_autotest_video_pattern(intel_dp); break; case DP_TEST_LINK_EDID_READ: - DRM_DEBUG_KMS("EDID test requested\n"); + drm_dbg_kms(&i915->drm, "EDID test requested\n"); response = intel_dp_autotest_edid(intel_dp); break; case DP_TEST_LINK_PHY_TEST_PATTERN: - DRM_DEBUG_KMS("PHY_PATTERN test requested\n"); + drm_dbg_kms(&i915->drm, "PHY_PATTERN test requested\n"); response = intel_dp_autotest_phy_pattern(intel_dp); break; default: - DRM_DEBUG_KMS("Invalid test request '%02x'\n", request); + drm_dbg_kms(&i915->drm, "Invalid test request '%02x'\n", + request); break; } @@ -5034,64 +5587,59 @@ static void intel_dp_handle_test_request(struct intel_dp *intel_dp) update_status: status = drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_RESPONSE, response); if (status <= 0) - DRM_DEBUG_KMS("Could not write test response to sink\n"); + drm_dbg_kms(&i915->drm, + "Could not write test response to sink\n"); } static int intel_dp_check_mst_status(struct intel_dp *intel_dp) { - bool bret; + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + bool need_retrain = false; - if (intel_dp->is_mst) { - u8 esi[DP_DPRX_ESI_LEN] = { 0 }; - int ret = 0; + if (!intel_dp->is_mst) + return -EINVAL; + + WARN_ON_ONCE(intel_dp->active_mst_links < 0); + + for (;;) { + u8 esi[DP_DPRX_ESI_LEN] = {}; + bool bret, handled; int retry; - bool handled; - WARN_ON_ONCE(intel_dp->active_mst_links < 0); bret = intel_dp_get_sink_irq_esi(intel_dp, esi); -go_again: - if (bret == true) { - - /* check link status - esi[10] = 0x200c */ - if (intel_dp->active_mst_links > 0 && - !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) { - DRM_DEBUG_KMS("channel EQ not ok, retraining\n"); - intel_dp_start_link_train(intel_dp); - intel_dp_stop_link_train(intel_dp); - } + if (!bret) { + drm_dbg_kms(&i915->drm, + "failed to get ESI - device may have failed\n"); + return -EINVAL; + } - DRM_DEBUG_KMS("got esi %3ph\n", esi); - ret = drm_dp_mst_hpd_irq(&intel_dp->mst_mgr, esi, &handled); - - if (handled) { - for (retry = 0; retry < 3; retry++) { - int wret; - wret = drm_dp_dpcd_write(&intel_dp->aux, - DP_SINK_COUNT_ESI+1, - &esi[1], 3); - if (wret == 3) { - break; - } - } + /* check link status - esi[10] = 0x200c */ + if (intel_dp->active_mst_links > 0 && !need_retrain && + !drm_dp_channel_eq_ok(&esi[10], intel_dp->lane_count)) { + drm_dbg_kms(&i915->drm, + "channel EQ not ok, retraining\n"); + need_retrain = true; + } - bret = intel_dp_get_sink_irq_esi(intel_dp, esi); - if (bret == true) { - DRM_DEBUG_KMS("got esi2 %3ph\n", esi); - goto go_again; - } - } else - ret = 0; + drm_dbg_kms(&i915->drm, "got esi %3ph\n", esi); - return ret; - } else { - DRM_DEBUG_KMS("failed to get ESI - device may have failed\n"); - intel_dp->is_mst = false; - drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, - intel_dp->is_mst); + drm_dp_mst_hpd_irq(&intel_dp->mst_mgr, esi, &handled); + if (!handled) + break; + + for (retry = 0; retry < 3; retry++) { + int wret; + + wret = drm_dp_dpcd_write(&intel_dp->aux, + DP_SINK_COUNT_ESI+1, + &esi[1], 3); + if (wret == 3) + break; } } - return -EINVAL; + + return need_retrain; } static bool @@ -5128,20 +5676,102 @@ intel_dp_needs_link_retrain(struct intel_dp *intel_dp) return !drm_dp_channel_eq_ok(link_status, intel_dp->lane_count); } +static bool intel_dp_has_connector(struct intel_dp *intel_dp, + const struct drm_connector_state *conn_state) +{ + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct intel_encoder *encoder; + enum pipe pipe; + + if (!conn_state->best_encoder) + return false; + + /* SST */ + encoder = &dp_to_dig_port(intel_dp)->base; + if (conn_state->best_encoder == &encoder->base) + return true; + + /* MST */ + for_each_pipe(i915, pipe) { + encoder = &intel_dp->mst_encoders[pipe]->base; + if (conn_state->best_encoder == &encoder->base) + return true; + } + + return false; +} + +static int intel_dp_prep_link_retrain(struct intel_dp *intel_dp, + struct drm_modeset_acquire_ctx *ctx, + u32 *crtc_mask) +{ + struct drm_i915_private *i915 = dp_to_i915(intel_dp); + struct drm_connector_list_iter conn_iter; + struct intel_connector *connector; + int ret = 0; + + *crtc_mask = 0; + + if (!intel_dp_needs_link_retrain(intel_dp)) + return 0; + + drm_connector_list_iter_begin(&i915->drm, &conn_iter); + for_each_intel_connector_iter(connector, &conn_iter) { + struct drm_connector_state *conn_state = + connector->base.state; + struct intel_crtc_state *crtc_state; + struct intel_crtc *crtc; + + if (!intel_dp_has_connector(intel_dp, conn_state)) + continue; + + crtc = to_intel_crtc(conn_state->crtc); + if (!crtc) + continue; + + ret = drm_modeset_lock(&crtc->base.mutex, ctx); + if (ret) + break; + + crtc_state = to_intel_crtc_state(crtc->base.state); + + drm_WARN_ON(&i915->drm, !intel_crtc_has_dp_encoder(crtc_state)); + + if (!crtc_state->hw.active) + continue; + + if (conn_state->commit && + !try_wait_for_completion(&conn_state->commit->hw_done)) + continue; + + *crtc_mask |= drm_crtc_mask(&crtc->base); + } + drm_connector_list_iter_end(&conn_iter); + + if (!intel_dp_needs_link_retrain(intel_dp)) + *crtc_mask = 0; + + return ret; +} + +static bool intel_dp_is_connected(struct intel_dp *intel_dp) +{ + struct intel_connector *connector = intel_dp->attached_connector; + + return connector->base.status == connector_status_connected || + intel_dp->is_mst; +} + int intel_dp_retrain_link(struct intel_encoder *encoder, struct drm_modeset_acquire_ctx *ctx) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(encoder); - struct intel_connector *connector = intel_dp->attached_connector; - struct drm_connector_state *conn_state; - struct intel_crtc_state *crtc_state; struct intel_crtc *crtc; + u32 crtc_mask; int ret; - /* FIXME handle the MST connectors as well */ - - if (!connector || connector->base.status != connector_status_connected) + if (!intel_dp_is_connected(intel_dp)) return 0; ret = drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, @@ -5149,46 +5779,42 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, if (ret) return ret; - conn_state = connector->base.state; - - crtc = to_intel_crtc(conn_state->crtc); - if (!crtc) - return 0; - - ret = drm_modeset_lock(&crtc->base.mutex, ctx); + ret = intel_dp_prep_link_retrain(intel_dp, ctx, &crtc_mask); if (ret) return ret; - crtc_state = to_intel_crtc_state(crtc->base.state); - - WARN_ON(!intel_crtc_has_dp_encoder(crtc_state)); - - if (!crtc_state->hw.active) + if (crtc_mask == 0) return 0; - if (conn_state->commit && - !try_wait_for_completion(&conn_state->commit->hw_done)) - return 0; + drm_dbg_kms(&dev_priv->drm, "[ENCODER:%d:%s] retraining link\n", + encoder->base.base.id, encoder->base.name); - if (!intel_dp_needs_link_retrain(intel_dp)) - return 0; + for_each_intel_crtc_mask(&dev_priv->drm, crtc, crtc_mask) { + const struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); - /* Suppress underruns caused by re-training */ - intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); - if (crtc_state->has_pch_encoder) - intel_set_pch_fifo_underrun_reporting(dev_priv, - intel_crtc_pch_transcoder(crtc), false); + /* Suppress underruns caused by re-training */ + intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, false); + if (crtc_state->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, + intel_crtc_pch_transcoder(crtc), false); + } intel_dp_start_link_train(intel_dp); intel_dp_stop_link_train(intel_dp); - /* Keep underrun reporting disabled until things are stable */ - intel_wait_for_vblank(dev_priv, crtc->pipe); + for_each_intel_crtc_mask(&dev_priv->drm, crtc, crtc_mask) { + const struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + + /* Keep underrun reporting disabled until things are stable */ + intel_wait_for_vblank(dev_priv, crtc->pipe); - intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); - if (crtc_state->has_pch_encoder) - intel_set_pch_fifo_underrun_reporting(dev_priv, - intel_crtc_pch_transcoder(crtc), true); + intel_set_cpu_fifo_underrun_reporting(dev_priv, crtc->pipe, true); + if (crtc_state->has_pch_encoder) + intel_set_pch_fifo_underrun_reporting(dev_priv, + intel_crtc_pch_transcoder(crtc), true); + } return 0; } @@ -5207,14 +5833,13 @@ int intel_dp_retrain_link(struct intel_encoder *encoder, */ static enum intel_hotplug_state intel_dp_hotplug(struct intel_encoder *encoder, - struct intel_connector *connector, - bool irq_received) + struct intel_connector *connector) { struct drm_modeset_acquire_ctx ctx; enum intel_hotplug_state state; int ret; - state = intel_encoder_hotplug(encoder, connector, irq_received); + state = intel_encoder_hotplug(encoder, connector); drm_modeset_acquire_init(&ctx, 0); @@ -5238,7 +5863,7 @@ intel_dp_hotplug(struct intel_encoder *encoder, * Keeping it consistent with intel_ddi_hotplug() and * intel_hdmi_hotplug(). */ - if (state == INTEL_HOTPLUG_UNCHANGED && irq_received) + if (state == INTEL_HOTPLUG_UNCHANGED && !connector->hotplug_retries) state = INTEL_HOTPLUG_RETRY; return state; @@ -5246,6 +5871,7 @@ intel_dp_hotplug(struct intel_encoder *encoder, static void intel_dp_check_service_irq(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 val; if (intel_dp->dpcd[DP_DPCD_REV] < 0x11) @@ -5264,7 +5890,7 @@ static void intel_dp_check_service_irq(struct intel_dp *intel_dp) intel_hdcp_handle_cp_irq(intel_dp->attached_connector); if (val & DP_SINK_SPECIFIC_IRQ) - DRM_DEBUG_DRIVER("Sink specific irq unhandled\n"); + drm_dbg_kms(&i915->drm, "Sink specific irq unhandled\n"); } /* @@ -5331,6 +5957,7 @@ intel_dp_short_pulse(struct intel_dp *intel_dp) static enum drm_connector_status intel_dp_detect_dpcd(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct intel_lspcon *lspcon = dp_to_lspcon(intel_dp); u8 *dpcd = intel_dp->dpcd; u8 type; @@ -5378,7 +6005,7 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp) } /* Anything else is out of spec, warn and ignore */ - DRM_DEBUG_KMS("Broken DP branch device, ignoring\n"); + drm_dbg_kms(&i915->drm, "Broken DP branch device, ignoring\n"); return connector_status_disconnected; } @@ -5391,64 +6018,7 @@ edp_detect(struct intel_dp *intel_dp) static bool ibx_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit; - - switch (encoder->hpd_pin) { - case HPD_PORT_B: - bit = SDE_PORTB_HOTPLUG; - break; - case HPD_PORT_C: - bit = SDE_PORTC_HOTPLUG; - break; - case HPD_PORT_D: - bit = SDE_PORTD_HOTPLUG; - break; - default: - MISSING_CASE(encoder->hpd_pin); - return false; - } - - return intel_de_read(dev_priv, SDEISR) & bit; -} - -static bool cpt_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit; - - switch (encoder->hpd_pin) { - case HPD_PORT_B: - bit = SDE_PORTB_HOTPLUG_CPT; - break; - case HPD_PORT_C: - bit = SDE_PORTC_HOTPLUG_CPT; - break; - case HPD_PORT_D: - bit = SDE_PORTD_HOTPLUG_CPT; - break; - default: - MISSING_CASE(encoder->hpd_pin); - return false; - } - - return intel_de_read(dev_priv, SDEISR) & bit; -} - -static bool spt_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit; - - switch (encoder->hpd_pin) { - case HPD_PORT_A: - bit = SDE_PORTA_HOTPLUG_SPT; - break; - case HPD_PORT_E: - bit = SDE_PORTE_HOTPLUG_SPT; - break; - default: - return cpt_digital_port_connected(encoder); - } + u32 bit = dev_priv->hotplug.pch_hpd[encoder->hpd_pin]; return intel_de_read(dev_priv, SDEISR) & bit; } @@ -5502,89 +6072,9 @@ static bool gm45_digital_port_connected(struct intel_encoder *encoder) static bool ilk_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin]; - if (encoder->hpd_pin == HPD_PORT_A) - return intel_de_read(dev_priv, DEISR) & DE_DP_A_HOTPLUG; - else - return ibx_digital_port_connected(encoder); -} - -static bool snb_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - - if (encoder->hpd_pin == HPD_PORT_A) - return intel_de_read(dev_priv, DEISR) & DE_DP_A_HOTPLUG; - else - return cpt_digital_port_connected(encoder); -} - -static bool ivb_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - - if (encoder->hpd_pin == HPD_PORT_A) - return intel_de_read(dev_priv, DEISR) & DE_DP_A_HOTPLUG_IVB; - else - return cpt_digital_port_connected(encoder); -} - -static bool bdw_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - - if (encoder->hpd_pin == HPD_PORT_A) - return intel_de_read(dev_priv, GEN8_DE_PORT_ISR) & GEN8_PORT_DP_A_HOTPLUG; - else - return cpt_digital_port_connected(encoder); -} - -static bool bxt_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - u32 bit; - - switch (encoder->hpd_pin) { - case HPD_PORT_A: - bit = BXT_DE_PORT_HP_DDIA; - break; - case HPD_PORT_B: - bit = BXT_DE_PORT_HP_DDIB; - break; - case HPD_PORT_C: - bit = BXT_DE_PORT_HP_DDIC; - break; - default: - MISSING_CASE(encoder->hpd_pin); - return false; - } - - return intel_de_read(dev_priv, GEN8_DE_PORT_ISR) & bit; -} - -static bool intel_combo_phy_connected(struct drm_i915_private *dev_priv, - enum phy phy) -{ - if (HAS_PCH_MCC(dev_priv) && phy == PHY_C) - return intel_de_read(dev_priv, SDEISR) & SDE_TC_HOTPLUG_ICP(PORT_TC1); - - return intel_de_read(dev_priv, SDEISR) & SDE_DDI_HOTPLUG_ICP(phy); -} - -static bool icp_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - struct intel_digital_port *dig_port = enc_to_dig_port(encoder); - enum phy phy = intel_port_to_phy(dev_priv, encoder->port); - - if (intel_phy_is_combo(dev_priv, phy)) - return intel_combo_phy_connected(dev_priv, phy); - else if (intel_phy_is_tc(dev_priv, phy)) - return intel_tc_port_connected(dig_port); - else - MISSING_CASE(encoder->hpd_pin); - - return false; + return intel_de_read(dev_priv, DEISR) & bit; } /* @@ -5598,44 +6088,15 @@ static bool icp_digital_port_connected(struct intel_encoder *encoder) * * Return %true if port is connected, %false otherwise. */ -static bool __intel_digital_port_connected(struct intel_encoder *encoder) -{ - struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - - if (HAS_GMCH(dev_priv)) { - if (IS_GM45(dev_priv)) - return gm45_digital_port_connected(encoder); - else - return g4x_digital_port_connected(encoder); - } - - if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) - return icp_digital_port_connected(encoder); - else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT) - return spt_digital_port_connected(encoder); - else if (IS_GEN9_LP(dev_priv)) - return bxt_digital_port_connected(encoder); - else if (IS_GEN(dev_priv, 8)) - return bdw_digital_port_connected(encoder); - else if (IS_GEN(dev_priv, 7)) - return ivb_digital_port_connected(encoder); - else if (IS_GEN(dev_priv, 6)) - return snb_digital_port_connected(encoder); - else if (IS_GEN(dev_priv, 5)) - return ilk_digital_port_connected(encoder); - - MISSING_CASE(INTEL_GEN(dev_priv)); - return false; -} - bool intel_digital_port_connected(struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); + struct intel_digital_port *dig_port = enc_to_dig_port(encoder); bool is_connected = false; intel_wakeref_t wakeref; with_intel_display_power(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref) - is_connected = __intel_digital_port_connected(encoder); + is_connected = dig_port->connected(encoder); return is_connected; } @@ -5669,6 +6130,7 @@ intel_dp_set_edid(struct intel_dp *intel_dp) intel_dp->has_audio = drm_detect_monitor_audio(edid); drm_dp_cec_set_edid(&intel_dp->aux, edid); + intel_dp->edid_quirks = drm_dp_get_edid_quirks(edid); } static void @@ -5681,6 +6143,7 @@ intel_dp_unset_edid(struct intel_dp *intel_dp) intel_connector->detect_edid = NULL; intel_dp->has_audio = false; + intel_dp->edid_quirks = 0; } static int @@ -5696,7 +6159,8 @@ intel_dp_detect(struct drm_connector *connector, drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); - WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex)); + drm_WARN_ON(&dev_priv->drm, + !drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex)); /* Can't disconnect eDP */ if (intel_dp_is_edp(intel_dp)) @@ -5847,6 +6311,7 @@ static int intel_dp_get_modes(struct drm_connector *connector) static int intel_dp_connector_register(struct drm_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->dev); struct intel_dp *intel_dp = intel_attached_dp(to_intel_connector(connector)); int ret; @@ -5854,10 +6319,8 @@ intel_dp_connector_register(struct drm_connector *connector) if (ret) return ret; - intel_connector_debugfs_add(connector); - - DRM_DEBUG_KMS("registering %s bus for %s\n", - intel_dp->aux.name, connector->kdev->kobj.name); + drm_dbg_kms(&i915->drm, "registering %s bus for %s\n", + intel_dp->aux.name, connector->kdev->kobj.name); intel_dp->aux.dev = connector->kdev; ret = drm_dp_aux_register(&intel_dp->aux); @@ -5943,6 +6406,7 @@ static int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, u8 *an) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); struct intel_dp *intel_dp = enc_to_intel_dp(to_intel_encoder(&intel_dig_port->base.base)); static const struct drm_dp_aux_msg msg = { .request = DP_AUX_NATIVE_WRITE, @@ -5957,8 +6421,9 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, dpcd_ret = drm_dp_dpcd_write(&intel_dig_port->dp.aux, DP_AUX_HDCP_AN, an, DRM_HDCP_AN_LEN); if (dpcd_ret != DRM_HDCP_AN_LEN) { - DRM_DEBUG_KMS("Failed to write An over DP/AUX (%zd)\n", - dpcd_ret); + drm_dbg_kms(&i915->drm, + "Failed to write An over DP/AUX (%zd)\n", + dpcd_ret); return dpcd_ret >= 0 ? -EIO : dpcd_ret; } @@ -5974,17 +6439,19 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, rxbuf, sizeof(rxbuf), DP_AUX_CH_CTL_AUX_AKSV_SELECT); if (ret < 0) { - DRM_DEBUG_KMS("Write Aksv over DP/AUX failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, + "Write Aksv over DP/AUX failed (%d)\n", ret); return ret; } else if (ret == 0) { - DRM_DEBUG_KMS("Aksv write over DP/AUX was empty\n"); + drm_dbg_kms(&i915->drm, "Aksv write over DP/AUX was empty\n"); return -EIO; } reply = (rxbuf[0] >> 4) & DP_AUX_NATIVE_REPLY_MASK; if (reply != DP_AUX_NATIVE_REPLY_ACK) { - DRM_DEBUG_KMS("Aksv write: no DP_AUX_NATIVE_REPLY_ACK %x\n", - reply); + drm_dbg_kms(&i915->drm, + "Aksv write: no DP_AUX_NATIVE_REPLY_ACK %x\n", + reply); return -EIO; } return 0; @@ -5993,11 +6460,14 @@ int intel_dp_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, static int intel_dp_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, u8 *bksv) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; + ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BKSV, bksv, DRM_HDCP_KSV_LEN); if (ret != DRM_HDCP_KSV_LEN) { - DRM_DEBUG_KMS("Read Bksv from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, + "Read Bksv from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -6006,7 +6476,9 @@ static int intel_dp_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, static int intel_dp_hdcp_read_bstatus(struct intel_digital_port *intel_dig_port, u8 *bstatus) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; + /* * For some reason the HDMI and DP HDCP specs call this register * definition by different names. In the HDMI spec, it's called BSTATUS, @@ -6015,7 +6487,8 @@ static int intel_dp_hdcp_read_bstatus(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BINFO, bstatus, DRM_HDCP_BSTATUS_LEN); if (ret != DRM_HDCP_BSTATUS_LEN) { - DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, + "Read bstatus from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -6025,12 +6498,14 @@ static int intel_dp_hdcp_read_bcaps(struct intel_digital_port *intel_dig_port, u8 *bcaps) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BCAPS, bcaps, 1); if (ret != 1) { - DRM_DEBUG_KMS("Read bcaps from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, + "Read bcaps from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } @@ -6056,11 +6531,14 @@ static int intel_dp_hdcp_read_ri_prime(struct intel_digital_port *intel_dig_port, u8 *ri_prime) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; + ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_RI_PRIME, ri_prime, DRM_HDCP_RI_LEN); if (ret != DRM_HDCP_RI_LEN) { - DRM_DEBUG_KMS("Read Ri' from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, "Read Ri' from DP/AUX failed (%zd)\n", + ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -6070,12 +6548,15 @@ static int intel_dp_hdcp_read_ksv_ready(struct intel_digital_port *intel_dig_port, bool *ksv_ready) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; u8 bstatus; + ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, &bstatus, 1); if (ret != 1) { - DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, + "Read bstatus from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } *ksv_ready = bstatus & DP_BSTATUS_READY; @@ -6086,6 +6567,7 @@ static int intel_dp_hdcp_read_ksv_fifo(struct intel_digital_port *intel_dig_port, int num_downstream, u8 *ksv_fifo) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; int i; @@ -6097,8 +6579,9 @@ int intel_dp_hdcp_read_ksv_fifo(struct intel_digital_port *intel_dig_port, ksv_fifo + i * DRM_HDCP_KSV_LEN, len); if (ret != len) { - DRM_DEBUG_KMS("Read ksv[%d] from DP/AUX failed (%zd)\n", - i, ret); + drm_dbg_kms(&i915->drm, + "Read ksv[%d] from DP/AUX failed (%zd)\n", + i, ret); return ret >= 0 ? -EIO : ret; } } @@ -6109,6 +6592,7 @@ static int intel_dp_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, int i, u32 *part) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; if (i >= DRM_HDCP_V_PRIME_NUM_PARTS) @@ -6118,7 +6602,8 @@ int intel_dp_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, DP_AUX_HDCP_V_PRIME(i), part, DRM_HDCP_V_PRIME_PART_LEN); if (ret != DRM_HDCP_V_PRIME_PART_LEN) { - DRM_DEBUG_KMS("Read v'[%d] from DP/AUX failed (%zd)\n", i, ret); + drm_dbg_kms(&i915->drm, + "Read v'[%d] from DP/AUX failed (%zd)\n", i, ret); return ret >= 0 ? -EIO : ret; } return 0; @@ -6135,13 +6620,15 @@ int intel_dp_hdcp_toggle_signalling(struct intel_digital_port *intel_dig_port, static bool intel_dp_hdcp_check_link(struct intel_digital_port *intel_dig_port) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; u8 bstatus; ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_AUX_HDCP_BSTATUS, &bstatus, 1); if (ret != 1) { - DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, + "Read bstatus from DP/AUX failed (%zd)\n", ret); return false; } @@ -6212,17 +6699,19 @@ static const struct hdcp2_dp_msg_data hdcp2_dp_msg_data[] = { 0, 0 }, }; -static inline -int intel_dp_hdcp2_read_rx_status(struct intel_digital_port *intel_dig_port, - u8 *rx_status) +static int +intel_dp_hdcp2_read_rx_status(struct intel_digital_port *intel_dig_port, + u8 *rx_status) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); ssize_t ret; ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, DP_HDCP_2_2_REG_RXSTATUS_OFFSET, rx_status, HDCP_2_2_DP_RXSTATUS_LEN); if (ret != HDCP_2_2_DP_RXSTATUS_LEN) { - DRM_DEBUG_KMS("Read bstatus from DP/AUX failed (%zd)\n", ret); + drm_dbg_kms(&i915->drm, + "Read bstatus from DP/AUX failed (%zd)\n", ret); return ret >= 0 ? -EIO : ret; } @@ -6266,6 +6755,7 @@ static ssize_t intel_dp_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port, const struct hdcp2_dp_msg_data *hdcp2_msg_data) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); struct intel_dp *dp = &intel_dig_port->dp; struct intel_hdcp *hdcp = &dp->attached_connector->hdcp; u8 msg_id = hdcp2_msg_data->msg_id; @@ -6297,8 +6787,9 @@ intel_dp_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port, } if (ret) - DRM_DEBUG_KMS("msg_id %d, ret %d, timeout(mSec): %d\n", - hdcp2_msg_data->msg_id, ret, timeout); + drm_dbg_kms(&i915->drm, + "msg_id %d, ret %d, timeout(mSec): %d\n", + hdcp2_msg_data->msg_id, ret, timeout); return ret; } @@ -6384,6 +6875,7 @@ static int intel_dp_hdcp2_read_msg(struct intel_digital_port *intel_dig_port, u8 msg_id, void *buf, size_t size) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); unsigned int offset; u8 *byte = buf; ssize_t ret, bytes_to_recv, len; @@ -6417,7 +6909,8 @@ int intel_dp_hdcp2_read_msg(struct intel_digital_port *intel_dig_port, ret = drm_dp_dpcd_read(&intel_dig_port->dp.aux, offset, (void *)byte, len); if (ret < 0) { - DRM_DEBUG_KMS("msg_id %d, ret %zd\n", msg_id, ret); + drm_dbg_kms(&i915->drm, "msg_id %d, ret %zd\n", + msg_id, ret); return ret; } @@ -6435,6 +6928,7 @@ static int intel_dp_hdcp2_config_stream_type(struct intel_digital_port *intel_dig_port, bool is_repeater, u8 content_type) { + int ret; struct hdcp2_dp_errata_stream_type stream_type_msg; if (is_repeater) @@ -6450,8 +6944,11 @@ int intel_dp_hdcp2_config_stream_type(struct intel_digital_port *intel_dig_port, stream_type_msg.msg_id = HDCP_2_2_ERRATA_DP_STREAM_TYPE; stream_type_msg.stream_type = content_type; - return intel_dp_hdcp2_write_msg(intel_dig_port, &stream_type_msg, + ret = intel_dp_hdcp2_write_msg(intel_dig_port, &stream_type_msg, sizeof(stream_type_msg)); + + return ret < 0 ? ret : 0; + } static @@ -6622,7 +7119,7 @@ static int intel_modeset_tile_group(struct intel_atomic_state *state, if (ret) break; } - drm_connector_list_iter_begin(&dev_priv->drm, &conn_iter); + drm_connector_list_iter_end(&conn_iter); return ret; } @@ -6662,7 +7159,7 @@ static int intel_modeset_affected_transcoders(struct intel_atomic_state *state, transcoders &= ~BIT(crtc_state->cpu_transcoder); } - WARN_ON(transcoders != 0); + drm_WARN_ON(&dev_priv->drm, transcoders != 0); return 0; } @@ -6704,7 +7201,11 @@ static int intel_dp_connector_atomic_check(struct drm_connector *conn, if (ret) return ret; - if (INTEL_GEN(dev_priv) < 11) + /* + * We don't enable port sync on BDW due to missing w/as and + * due to not having adjusted the modeset sequence appropriately. + */ + if (INTEL_GEN(dev_priv) < 9) return 0; if (!intel_connector_needs_modeset(state, conn)) @@ -6743,28 +7244,45 @@ static const struct drm_encoder_funcs intel_dp_enc_funcs = { .destroy = intel_dp_encoder_destroy, }; +static bool intel_edp_have_power(struct intel_dp *intel_dp) +{ + intel_wakeref_t wakeref; + bool have_power = false; + + with_pps_lock(intel_dp, wakeref) { + have_power = edp_have_panel_power(intel_dp) && + edp_have_panel_vdd(intel_dp); + } + + return have_power; +} + enum irqreturn intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); struct intel_dp *intel_dp = &intel_dig_port->dp; - if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) { + if (intel_dig_port->base.type == INTEL_OUTPUT_EDP && + (long_hpd || !intel_edp_have_power(intel_dp))) { /* - * vdd off can generate a long pulse on eDP which + * vdd off can generate a long/short pulse on eDP which * would require vdd on to handle it, and thus we * would end up in an endless cycle of - * "vdd off -> long hpd -> vdd on -> detect -> vdd off -> ..." + * "vdd off -> long/short hpd -> vdd on -> detect -> vdd off -> ..." */ - DRM_DEBUG_KMS("ignoring long hpd on eDP [ENCODER:%d:%s]\n", - intel_dig_port->base.base.base.id, - intel_dig_port->base.base.name); + drm_dbg_kms(&i915->drm, + "ignoring %s hpd on eDP [ENCODER:%d:%s]\n", + long_hpd ? "long" : "short", + intel_dig_port->base.base.base.id, + intel_dig_port->base.base.name); return IRQ_HANDLED; } - DRM_DEBUG_KMS("got hpd irq on [ENCODER:%d:%s] - %s\n", - intel_dig_port->base.base.base.id, - intel_dig_port->base.base.name, - long_hpd ? "long" : "short"); + drm_dbg_kms(&i915->drm, "got hpd irq on [ENCODER:%d:%s] - %s\n", + intel_dig_port->base.base.base.id, + intel_dig_port->base.base.name, + long_hpd ? "long" : "short"); if (long_hpd) { intel_dp->reset_link_params = true; @@ -6772,18 +7290,25 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd) } if (intel_dp->is_mst) { - if (intel_dp_check_mst_status(intel_dp) == -EINVAL) { + switch (intel_dp_check_mst_status(intel_dp)) { + case -EINVAL: /* * If we were in MST mode, and device is not * there, get out of MST mode */ - DRM_DEBUG_KMS("MST device may have disappeared %d vs %d\n", - intel_dp->is_mst, intel_dp->mst_mgr.mst_state); + drm_dbg_kms(&i915->drm, + "MST device may have disappeared %d vs %d\n", + intel_dp->is_mst, + intel_dp->mst_mgr.mst_state); intel_dp->is_mst = false; drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst); return IRQ_NONE; + case 1: + return IRQ_NONE; + default: + break; } } @@ -7043,7 +7568,8 @@ intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp, if (force_disable_vdd) { u32 pp = ilk_get_pp_control(intel_dp); - WARN(pp & PANEL_POWER_ON, "Panel power already on\n"); + drm_WARN(&dev_priv->drm, pp & PANEL_POWER_ON, + "Panel power already on\n"); if (pp & EDP_FORCE_VDD) drm_dbg_kms(&dev_priv->drm, @@ -7519,7 +8045,8 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, * with an already powered-on LVDS power sequencer. */ if (intel_get_lvds_encoder(dev_priv)) { - WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))); + drm_WARN_ON(dev, + !(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv))); drm_info(&dev_priv->drm, "LVDS was detected, not registering eDP\n"); @@ -7546,8 +8073,8 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, edid = drm_get_edid(connector, &intel_dp->aux.ddc); if (edid) { if (drm_add_edid_modes(connector, edid)) { - drm_connector_update_edid_property(connector, - edid); + drm_connector_update_edid_property(connector, edid); + intel_dp->edid_quirks = drm_dp_get_edid_quirks(edid); } else { kfree(edid); edid = ERR_PTR(-EINVAL); @@ -7592,9 +8119,11 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, intel_connector->panel.backlight.power = intel_edp_backlight_power; intel_panel_setup_backlight(connector, pipe); - if (fixed_mode) - drm_connector_init_panel_orientation_property( - connector, fixed_mode->hdisplay, fixed_mode->vdisplay); + if (fixed_mode) { + drm_connector_set_panel_orientation_with_quirk(connector, + dev_priv->vbt.orientation, + fixed_mode->hdisplay, fixed_mode->vdisplay); + } return true; @@ -7650,10 +8179,10 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, INIT_WORK(&intel_connector->modeset_retry_work, intel_dp_modeset_retry_work_fn); - if (WARN(intel_dig_port->max_lanes < 1, - "Not enough lanes (%d) for DP on [ENCODER:%d:%s]\n", - intel_dig_port->max_lanes, intel_encoder->base.base.id, - intel_encoder->base.name)) + if (drm_WARN(dev, intel_dig_port->max_lanes < 1, + "Not enough lanes (%d) for DP on [ENCODER:%d:%s]\n", + intel_dig_port->max_lanes, intel_encoder->base.base.id, + intel_encoder->base.name)) return false; intel_dp_set_source_rates(intel_dp); @@ -7671,7 +8200,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, * Currently we don't support eDP on TypeC ports, although in * theory it could work on TypeC legacy ports. */ - WARN_ON(intel_phy_is_tc(dev_priv, phy)); + drm_WARN_ON(dev, intel_phy_is_tc(dev_priv, phy)); type = DRM_MODE_CONNECTOR_eDP; } else { type = DRM_MODE_CONNECTOR_DisplayPort; @@ -7689,9 +8218,10 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, intel_encoder->type = INTEL_OUTPUT_EDP; /* eDP only on port B and/or C on vlv/chv */ - if (WARN_ON((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && - intel_dp_is_edp(intel_dp) && - port != PORT_B && port != PORT_C)) + if (drm_WARN_ON(dev, (IS_VALLEYVIEW(dev_priv) || + IS_CHERRYVIEW(dev_priv)) && + intel_dp_is_edp(intel_dp) && + port != PORT_B && port != PORT_C)) return false; drm_dbg_kms(&dev_priv->drm, @@ -7710,6 +8240,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port, connector->ycbcr_420_allowed = true; intel_encoder->hpd_pin = intel_hpd_pin_default(dev_priv, port); + intel_connector->polled = DRM_CONNECTOR_POLL_HPD; intel_dp_aux_init(intel_dp); @@ -7808,8 +8339,27 @@ bool intel_dp_init(struct drm_i915_private *dev_priv, intel_encoder->post_disable = g4x_post_disable_dp; } + if ((IS_IVYBRIDGE(dev_priv) && port == PORT_A) || + (HAS_PCH_CPT(dev_priv) && port != PORT_A)) + intel_dig_port->dp.set_link_train = cpt_set_link_train; + else + intel_dig_port->dp.set_link_train = g4x_set_link_train; + + if (IS_CHERRYVIEW(dev_priv)) + intel_dig_port->dp.set_signal_levels = chv_set_signal_levels; + else if (IS_VALLEYVIEW(dev_priv)) + intel_dig_port->dp.set_signal_levels = vlv_set_signal_levels; + else if (IS_IVYBRIDGE(dev_priv) && port == PORT_A) + intel_dig_port->dp.set_signal_levels = ivb_cpu_edp_set_signal_levels; + else if (IS_GEN(dev_priv, 6) && port == PORT_A) + intel_dig_port->dp.set_signal_levels = snb_cpu_edp_set_signal_levels; + else + intel_dig_port->dp.set_signal_levels = g4x_set_signal_levels; + intel_dig_port->dp.output_reg = output_reg; intel_dig_port->max_lanes = 4; + intel_dig_port->dp.regs.dp_tp_ctl = DP_TP_CTL(port); + intel_dig_port->dp.regs.dp_tp_status = DP_TP_STATUS(port); intel_encoder->type = INTEL_OUTPUT_DP; intel_encoder->power_domain = intel_port_to_power_domain(port); @@ -7826,6 +8376,18 @@ bool intel_dp_init(struct drm_i915_private *dev_priv, intel_dig_port->hpd_pulse = intel_dp_hpd_pulse; + if (HAS_GMCH(dev_priv)) { + if (IS_GM45(dev_priv)) + intel_dig_port->connected = gm45_digital_port_connected; + else + intel_dig_port->connected = g4x_digital_port_connected; + } else { + if (port == PORT_A) + intel_dig_port->connected = ilk_digital_port_connected; + else + intel_dig_port->connected = ibx_digital_port_connected; + } + if (port != PORT_A) intel_infoframe_init(intel_dig_port); diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h index 3da166054788..1702959ca079 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.h +++ b/drivers/gpu/drm/i915/display/intel_dp.h @@ -8,8 +8,6 @@ #include <linux/types.h> -#include <drm/i915_drm.h> - #include "i915_reg.h" enum pipe; @@ -18,6 +16,7 @@ struct drm_connector_state; struct drm_encoder; struct drm_i915_private; struct drm_modeset_acquire_ctx; +struct drm_dp_vsc_sdp; struct intel_connector; struct intel_crtc_state; struct intel_digital_port; @@ -110,13 +109,21 @@ int intel_dp_link_required(int pixel_clock, int bpp); int intel_dp_max_data_rate(int max_link_clock, int max_lanes); bool intel_dp_needs_vsc_sdp(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); -void intel_dp_vsc_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); -void intel_dp_hdr_metadata_enable(struct intel_dp *intel_dp, +void intel_dp_compute_psr_vsc_sdp(struct intel_dp *intel_dp, const struct intel_crtc_state *crtc_state, - const struct drm_connector_state *conn_state); + const struct drm_connector_state *conn_state, + struct drm_dp_vsc_sdp *vsc); +void intel_write_dp_vsc_sdp(struct intel_encoder *encoder, + const struct intel_crtc_state *crtc_state, + struct drm_dp_vsc_sdp *vsc); +void intel_dp_set_infoframes(struct intel_encoder *encoder, bool enable, + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); +void intel_read_dp_sdp(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state, + unsigned int type); bool intel_digital_port_connected(struct intel_encoder *encoder); +void intel_dp_process_phy_request(struct intel_dp *intel_dp); static inline unsigned int intel_dp_unused_lane_mask(int lane_count) { diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c index 48276237b362..0722540d64ad 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c @@ -27,6 +27,7 @@ static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 reg_val = 0; /* Early return when display use other mechanism to enable backlight. */ @@ -35,8 +36,8 @@ static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER, ®_val) < 0) { - DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", - DP_EDP_DISPLAY_CONTROL_REGISTER); + drm_dbg_kms(&i915->drm, "Failed to read DPCD register 0x%x\n", + DP_EDP_DISPLAY_CONTROL_REGISTER); return; } if (enable) @@ -46,8 +47,8 @@ static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER, reg_val) != 1) { - DRM_DEBUG_KMS("Failed to %s aux backlight\n", - enable ? "enable" : "disable"); + drm_dbg_kms(&i915->drm, "Failed to %s aux backlight\n", + enable ? "enable" : "disable"); } } @@ -58,6 +59,7 @@ static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable) static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 read_val[2] = { 0x0 }; u8 mode_reg; u16 level = 0; @@ -65,8 +67,9 @@ static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &mode_reg) != 1) { - DRM_DEBUG_KMS("Failed to read the DPCD register 0x%x\n", - DP_EDP_BACKLIGHT_MODE_SET_REGISTER); + drm_dbg_kms(&i915->drm, + "Failed to read the DPCD register 0x%x\n", + DP_EDP_BACKLIGHT_MODE_SET_REGISTER); return 0; } @@ -80,8 +83,8 @@ static u32 intel_dp_aux_get_backlight(struct intel_connector *connector) if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, &read_val, sizeof(read_val)) < 0) { - DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", - DP_EDP_BACKLIGHT_BRIGHTNESS_MSB); + drm_dbg_kms(&i915->drm, "Failed to read DPCD register 0x%x\n", + DP_EDP_BACKLIGHT_BRIGHTNESS_MSB); return 0; } level = read_val[0]; @@ -100,6 +103,7 @@ intel_dp_aux_set_backlight(const struct drm_connector_state *conn_state, u32 lev { struct intel_connector *connector = to_intel_connector(conn_state->connector); struct intel_dp *intel_dp = intel_attached_dp(connector); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 vals[2] = { 0x0 }; vals[0] = level; @@ -111,7 +115,8 @@ intel_dp_aux_set_backlight(const struct drm_connector_state *conn_state, u32 lev } if (drm_dp_dpcd_write(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB, vals, sizeof(vals)) < 0) { - DRM_DEBUG_KMS("Failed to write aux backlight level\n"); + drm_dbg_kms(&i915->drm, + "Failed to write aux backlight level\n"); return; } } @@ -133,7 +138,8 @@ static bool intel_dp_aux_set_pwm_freq(struct intel_connector *connector) freq = dev_priv->vbt.backlight.pwm_freq_hz; if (!freq) { - DRM_DEBUG_KMS("Use panel default backlight frequency\n"); + drm_dbg_kms(&dev_priv->drm, + "Use panel default backlight frequency\n"); return false; } @@ -146,13 +152,14 @@ static bool intel_dp_aux_set_pwm_freq(struct intel_connector *connector) fxp_max = DIV_ROUND_CLOSEST(fxp * 5, 4); if (fxp_min > fxp_actual || fxp_actual > fxp_max) { - DRM_DEBUG_KMS("Actual frequency out of range\n"); + drm_dbg_kms(&dev_priv->drm, "Actual frequency out of range\n"); return false; } if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_BACKLIGHT_FREQ_SET, (u8) f) < 0) { - DRM_DEBUG_KMS("Failed to write aux backlight freq\n"); + drm_dbg_kms(&dev_priv->drm, + "Failed to write aux backlight freq\n"); return false; } return true; @@ -163,13 +170,14 @@ static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_st { struct intel_connector *connector = to_intel_connector(conn_state->connector); struct intel_dp *intel_dp = intel_attached_dp(connector); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); struct intel_panel *panel = &connector->panel; u8 dpcd_buf, new_dpcd_buf, edp_backlight_mode; if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) != 1) { - DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n", - DP_EDP_BACKLIGHT_MODE_SET_REGISTER); + drm_dbg_kms(&i915->drm, "Failed to read DPCD register 0x%x\n", + DP_EDP_BACKLIGHT_MODE_SET_REGISTER); return; } @@ -186,7 +194,8 @@ static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_st if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_PWMGEN_BIT_COUNT, panel->backlight.pwmgen_bit_count) < 0) - DRM_DEBUG_KMS("Failed to write aux pwmgen bit count\n"); + drm_dbg_kms(&i915->drm, + "Failed to write aux pwmgen bit count\n"); break; @@ -203,7 +212,8 @@ static void intel_dp_aux_enable_backlight(const struct intel_crtc_state *crtc_st if (new_dpcd_buf != dpcd_buf) { if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER, new_dpcd_buf) < 0) { - DRM_DEBUG_KMS("Failed to write aux backlight mode\n"); + drm_dbg_kms(&i915->drm, + "Failed to write aux backlight mode\n"); } } @@ -237,9 +247,11 @@ static u32 intel_dp_aux_calc_max_backlight(struct intel_connector *connector) * minimum value will applied automatically. So no need to check that. */ freq = i915->vbt.backlight.pwm_freq_hz; - DRM_DEBUG_KMS("VBT defined backlight frequency %u Hz\n", freq); + drm_dbg_kms(&i915->drm, "VBT defined backlight frequency %u Hz\n", + freq); if (!freq) { - DRM_DEBUG_KMS("Use panel default backlight frequency\n"); + drm_dbg_kms(&i915->drm, + "Use panel default backlight frequency\n"); return max_backlight; } @@ -254,12 +266,14 @@ static u32 intel_dp_aux_calc_max_backlight(struct intel_connector *connector) */ if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_PWMGEN_BIT_COUNT_CAP_MIN, &pn_min) != 1) { - DRM_DEBUG_KMS("Failed to read pwmgen bit count cap min\n"); + drm_dbg_kms(&i915->drm, + "Failed to read pwmgen bit count cap min\n"); return max_backlight; } if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_PWMGEN_BIT_COUNT_CAP_MAX, &pn_max) != 1) { - DRM_DEBUG_KMS("Failed to read pwmgen bit count cap max\n"); + drm_dbg_kms(&i915->drm, + "Failed to read pwmgen bit count cap max\n"); return max_backlight; } pn_min &= DP_EDP_PWMGEN_BIT_COUNT_MASK; @@ -268,7 +282,8 @@ static u32 intel_dp_aux_calc_max_backlight(struct intel_connector *connector) fxp_min = DIV_ROUND_CLOSEST(fxp * 3, 4); fxp_max = DIV_ROUND_CLOSEST(fxp * 5, 4); if (fxp_min < (1 << pn_min) || (255 << pn_max) < fxp_max) { - DRM_DEBUG_KMS("VBT defined backlight frequency out of range\n"); + drm_dbg_kms(&i915->drm, + "VBT defined backlight frequency out of range\n"); return max_backlight; } @@ -279,10 +294,11 @@ static u32 intel_dp_aux_calc_max_backlight(struct intel_connector *connector) break; } - DRM_DEBUG_KMS("Using eDP pwmgen bit count of %d\n", pn); + drm_dbg_kms(&i915->drm, "Using eDP pwmgen bit count of %d\n", pn); if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_PWMGEN_BIT_COUNT, pn) < 0) { - DRM_DEBUG_KMS("Failed to write aux pwmgen bit count\n"); + drm_dbg_kms(&i915->drm, + "Failed to write aux pwmgen bit count\n"); return max_backlight; } panel->backlight.pwmgen_bit_count = pn; @@ -312,6 +328,7 @@ static bool intel_dp_aux_display_control_capable(struct intel_connector *connector) { struct intel_dp *intel_dp = intel_attached_dp(connector); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); /* Check the eDP Display control capabilities registers to determine if * the panel can support backlight control over the aux channel @@ -319,7 +336,7 @@ intel_dp_aux_display_control_capable(struct intel_connector *connector) if (intel_dp->edp_dpcd[1] & DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP && (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_AUX_SET_CAP) && !(intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_PWM_PIN_CAP)) { - DRM_DEBUG_KMS("AUX Backlight Control Supported!\n"); + drm_dbg_kms(&i915->drm, "AUX Backlight Control Supported!\n"); return true; } return false; @@ -328,15 +345,31 @@ intel_dp_aux_display_control_capable(struct intel_connector *connector) int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector) { struct intel_panel *panel = &intel_connector->panel; - struct drm_i915_private *dev_priv = to_i915(intel_connector->base.dev); + struct intel_dp *intel_dp = enc_to_intel_dp(intel_connector->encoder); + struct drm_i915_private *i915 = dp_to_i915(intel_dp); if (i915_modparams.enable_dpcd_backlight == 0 || - (i915_modparams.enable_dpcd_backlight == -1 && - dev_priv->vbt.backlight.type != INTEL_BACKLIGHT_VESA_EDP_AUX_INTERFACE)) + !intel_dp_aux_display_control_capable(intel_connector)) return -ENODEV; - if (!intel_dp_aux_display_control_capable(intel_connector)) + /* + * There are a lot of machines that don't advertise the backlight + * control interface to use properly in their VBIOS, :\ + */ + if (i915->vbt.backlight.type != + INTEL_BACKLIGHT_VESA_EDP_AUX_INTERFACE && + i915_modparams.enable_dpcd_backlight != 1 && + !drm_dp_has_quirk(&intel_dp->desc, intel_dp->edid_quirks, + DP_QUIRK_FORCE_DPCD_BACKLIGHT)) { + drm_info(&i915->drm, + "Panel advertises DPCD backlight support, but " + "VBT disagrees. If your backlight controls " + "don't work try booting with " + "i915.enable_dpcd_backlight=1. If your machine " + "needs this, please file a _new_ bug report on " + "drm/i915, see " FDO_BUG_URL " for details.\n"); return -ENODEV; + } panel->backlight.setup = intel_dp_aux_setup_backlight; panel->backlight.enable = intel_dp_aux_enable_backlight; diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index a7defb37ab00..e4f1843170b7 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -34,9 +34,8 @@ intel_dp_dump_link_status(const u8 link_status[DP_LINK_STATUS_SIZE]) link_status[3], link_status[4], link_status[5]); } -static void -intel_get_adjust_train(struct intel_dp *intel_dp, - const u8 link_status[DP_LINK_STATUS_SIZE]) +void intel_dp_get_adjust_train(struct intel_dp *intel_dp, + const u8 link_status[DP_LINK_STATUS_SIZE]) { u8 v = 0; u8 p = 0; @@ -219,7 +218,7 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp) voltage = intel_dp->train_set[0] & DP_TRAIN_VOLTAGE_SWING_MASK; /* Update training set as requested by target */ - intel_get_adjust_train(intel_dp, link_status); + intel_dp_get_adjust_train(intel_dp, link_status); if (!intel_dp_update_link_train(intel_dp)) { drm_err(&i915->drm, "failed to update link training\n"); @@ -338,7 +337,7 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp) } /* Update training set as requested by target */ - intel_get_adjust_train(intel_dp, link_status); + intel_dp_get_adjust_train(intel_dp, link_status); if (!intel_dp_update_link_train(intel_dp)) { drm_err(&i915->drm, "failed to update link training\n"); diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.h b/drivers/gpu/drm/i915/display/intel_dp_link_training.h index 174566adcc92..01f1dabbb060 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.h +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.h @@ -6,8 +6,12 @@ #ifndef __INTEL_DP_LINK_TRAINING_H__ #define __INTEL_DP_LINK_TRAINING_H__ +#include <drm/drm_dp_helper.h> + struct intel_dp; +void intel_dp_get_adjust_train(struct intel_dp *intel_dp, + const u8 link_status[DP_LINK_STATUS_SIZE]); void intel_dp_start_link_train(struct intel_dp *intel_dp); void intel_dp_stop_link_train(struct intel_dp *intel_dp); diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index d7bfa7c350e9..8c732418a33f 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -47,10 +47,10 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, struct intel_dp *intel_dp = &intel_mst->primary->dp; struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; - void *port = connector->port; - bool constant_n = drm_dp_has_quirk(&intel_dp->desc, + bool constant_n = drm_dp_has_quirk(&intel_dp->desc, 0, DP_DPCD_QUIRK_CONSTANT_N); int bpp, slots = -EINVAL; @@ -65,7 +65,8 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, false); slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr, - port, crtc_state->pbn, 0); + connector->port, + crtc_state->pbn, 0); if (slots == -EDEADLK) return slots; if (slots >= 0) @@ -73,7 +74,8 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, } if (slots < 0) { - DRM_DEBUG_KMS("failed finding vcpi slots:%d\n", slots); + drm_dbg_kms(&i915->drm, "failed finding vcpi slots:%d\n", + slots); return slots; } @@ -88,56 +90,10 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder, return 0; } -/* - * Iterate over all connectors and return the smallest transcoder in the MST - * stream - */ -static enum transcoder -intel_dp_mst_master_trans_compute(struct intel_atomic_state *state, - struct intel_dp *mst_port) -{ - struct drm_i915_private *dev_priv = to_i915(state->base.dev); - struct intel_digital_connector_state *conn_state; - struct intel_connector *connector; - enum pipe ret = I915_MAX_PIPES; - int i; - - if (INTEL_GEN(dev_priv) < 12) - return INVALID_TRANSCODER; - - for_each_new_intel_connector_in_state(state, connector, conn_state, i) { - struct intel_crtc_state *crtc_state; - struct intel_crtc *crtc; - - if (connector->mst_port != mst_port || !conn_state->base.crtc) - continue; - - crtc = to_intel_crtc(conn_state->base.crtc); - crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - if (!crtc_state->uapi.active) - continue; - - /* - * Using crtc->pipe because crtc_state->cpu_transcoder is - * computed, so others CRTCs could have non-computed - * cpu_transcoder - */ - if (crtc->pipe < ret) - ret = crtc->pipe; - } - - if (ret == I915_MAX_PIPES) - return INVALID_TRANSCODER; - - /* Simple cast works because TGL don't have a eDP transcoder */ - return (enum transcoder)ret; -} - static int intel_dp_mst_compute_config(struct intel_encoder *encoder, struct intel_crtc_state *pipe_config, struct drm_connector_state *conn_state) { - struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); struct intel_dp *intel_dp = &intel_mst->primary->dp; @@ -147,7 +103,6 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder, to_intel_digital_connector_state(conn_state); const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - void *port = connector->port; struct link_config_limits limits; int ret; @@ -159,7 +114,8 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder, if (intel_conn_state->force_audio == HDMI_AUDIO_AUTO) pipe_config->has_audio = - drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, port); + drm_dp_mst_port_has_audio(&intel_dp->mst_mgr, + connector->port); else pipe_config->has_audio = intel_conn_state->force_audio == HDMI_AUDIO_ON; @@ -201,7 +157,56 @@ static int intel_dp_mst_compute_config(struct intel_encoder *encoder, intel_ddi_compute_min_voltage_level(dev_priv, pipe_config); - pipe_config->mst_master_transcoder = intel_dp_mst_master_trans_compute(state, intel_dp); + return 0; +} + +/* + * Iterate over all connectors and return a mask of + * all CPU transcoders streaming over the same DP link. + */ +static unsigned int +intel_dp_mst_transcoder_mask(struct intel_atomic_state *state, + struct intel_dp *mst_port) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_digital_connector_state *conn_state; + struct intel_connector *connector; + u8 transcoders = 0; + int i; + + if (INTEL_GEN(dev_priv) < 12) + return 0; + + for_each_new_intel_connector_in_state(state, connector, conn_state, i) { + const struct intel_crtc_state *crtc_state; + struct intel_crtc *crtc; + + if (connector->mst_port != mst_port || !conn_state->base.crtc) + continue; + + crtc = to_intel_crtc(conn_state->base.crtc); + crtc_state = intel_atomic_get_new_crtc_state(state, crtc); + + if (!crtc_state->hw.active) + continue; + + transcoders |= BIT(crtc_state->cpu_transcoder); + } + + return transcoders; +} + +static int intel_dp_mst_compute_config_late(struct intel_encoder *encoder, + struct intel_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state); + struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder); + struct intel_dp *intel_dp = &intel_mst->primary->dp; + + /* lowest numbered transcoder will be designated master */ + crtc_state->mst_master_transcoder = + ffs(intel_dp_mst_transcoder_mask(state, intel_dp)) - 1; return 0; } @@ -313,7 +318,8 @@ intel_dp_mst_atomic_check(struct drm_connector *connector, return ret; } -static void intel_mst_disable_dp(struct intel_encoder *encoder, +static void intel_mst_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -322,22 +328,25 @@ static void intel_mst_disable_dp(struct intel_encoder *encoder, struct intel_dp *intel_dp = &intel_dig_port->dp; struct intel_connector *connector = to_intel_connector(old_conn_state->connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); int ret; - DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); + drm_dbg_kms(&i915->drm, "active links %d\n", + intel_dp->active_mst_links); drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port); ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr); if (ret) { - DRM_DEBUG_KMS("failed to update payload %d\n", ret); + drm_dbg_kms(&i915->drm, "failed to update payload %d\n", ret); } if (old_crtc_state->has_audio) intel_audio_codec_disable(encoder, old_crtc_state, old_conn_state); } -static void intel_mst_post_disable_dp(struct intel_encoder *encoder, +static void intel_mst_post_disable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -371,7 +380,8 @@ static void intel_mst_post_disable_dp(struct intel_encoder *encoder, if (intel_de_wait_for_set(dev_priv, intel_dp->regs.dp_tp_status, DP_TP_STATUS_ACT_SENT, 1)) - DRM_ERROR("Timed out waiting for ACT sent when disabling\n"); + drm_err(&dev_priv->drm, + "Timed out waiting for ACT sent when disabling\n"); drm_dp_check_act_status(&intel_dp->mst_mgr); drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, connector->port); @@ -402,13 +412,15 @@ static void intel_mst_post_disable_dp(struct intel_encoder *encoder, intel_mst->connector = NULL; if (last_mst_stream) - intel_dig_port->base.post_disable(&intel_dig_port->base, + intel_dig_port->base.post_disable(state, &intel_dig_port->base, old_crtc_state, NULL); - DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); + drm_dbg_kms(&dev_priv->drm, "active links %d\n", + intel_dp->active_mst_links); } -static void intel_mst_pre_pll_enable_dp(struct intel_encoder *encoder, +static void intel_mst_pre_pll_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -417,11 +429,12 @@ static void intel_mst_pre_pll_enable_dp(struct intel_encoder *encoder, struct intel_dp *intel_dp = &intel_dig_port->dp; if (intel_dp->active_mst_links == 0) - intel_dig_port->base.pre_pll_enable(&intel_dig_port->base, + intel_dig_port->base.pre_pll_enable(state, &intel_dig_port->base, pipe_config, NULL); } -static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, +static void intel_mst_pre_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -445,7 +458,8 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, INTEL_GEN(dev_priv) >= 12 && first_mst_stream && !intel_dp_mst_is_master_trans(pipe_config)); - DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); + drm_dbg_kms(&dev_priv->drm, "active links %d\n", + intel_dp->active_mst_links); if (first_mst_stream) intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON); @@ -453,7 +467,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, drm_dp_send_power_updown_phy(&intel_dp->mst_mgr, connector->port, true); if (first_mst_stream) - intel_dig_port->base.pre_enable(&intel_dig_port->base, + intel_dig_port->base.pre_enable(state, &intel_dig_port->base, pipe_config, NULL); ret = drm_dp_mst_allocate_vcpi(&intel_dp->mst_mgr, @@ -461,7 +475,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, pipe_config->pbn, pipe_config->dp_m_n.tu); if (!ret) - DRM_ERROR("failed to allocate vcpi\n"); + drm_err(&dev_priv->drm, "failed to allocate vcpi\n"); intel_dp->active_mst_links++; temp = intel_de_read(dev_priv, intel_dp->regs.dp_tp_status); @@ -477,14 +491,15 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder, * here for the following ones. */ if (INTEL_GEN(dev_priv) < 12 || !first_mst_stream) - intel_ddi_enable_pipe_clock(pipe_config); + intel_ddi_enable_pipe_clock(encoder, pipe_config); intel_ddi_set_dp_msa(pipe_config, conn_state); intel_dp_set_m_n(pipe_config, M1_N1); } -static void intel_mst_enable_dp(struct intel_encoder *encoder, +static void intel_mst_enable_dp(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -495,19 +510,23 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder, drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder); - intel_enable_pipe(pipe_config); - - intel_crtc_vblank_on(pipe_config); + intel_ddi_enable_transcoder_func(encoder, pipe_config); - DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links); + drm_dbg_kms(&dev_priv->drm, "active links %d\n", + intel_dp->active_mst_links); if (intel_de_wait_for_set(dev_priv, intel_dp->regs.dp_tp_status, DP_TP_STATUS_ACT_SENT, 1)) - DRM_ERROR("Timed out waiting for ACT sent\n"); + drm_err(&dev_priv->drm, "Timed out waiting for ACT sent\n"); drm_dp_check_act_status(&intel_dp->mst_mgr); drm_dp_update_payload_part2(&intel_dp->mst_mgr); + + intel_enable_pipe(pipe_config); + + intel_crtc_vblank_on(pipe_config); + if (pipe_config->has_audio) intel_audio_codec_enable(encoder, pipe_config, conn_state); } @@ -548,12 +567,41 @@ static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector) return ret; } +static int +intel_dp_mst_connector_late_register(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + int ret; + + ret = drm_dp_mst_connector_late_register(connector, + intel_connector->port); + if (ret < 0) + return ret; + + ret = intel_connector_register(connector); + if (ret < 0) + drm_dp_mst_connector_early_unregister(connector, + intel_connector->port); + + return ret; +} + +static void +intel_dp_mst_connector_early_unregister(struct drm_connector *connector) +{ + struct intel_connector *intel_connector = to_intel_connector(connector); + + intel_connector_unregister(connector); + drm_dp_mst_connector_early_unregister(connector, + intel_connector->port); +} + static const struct drm_connector_funcs intel_dp_mst_connector_funcs = { .fill_modes = drm_helper_probe_single_connector_modes, .atomic_get_property = intel_digital_connector_atomic_get_property, .atomic_set_property = intel_digital_connector_atomic_set_property, - .late_register = intel_connector_register, - .early_unregister = intel_connector_unregister, + .late_register = intel_dp_mst_connector_late_register, + .early_unregister = intel_dp_mst_connector_early_unregister, .destroy = intel_connector_destroy, .atomic_destroy_state = drm_atomic_helper_connector_destroy_state, .atomic_duplicate_state = intel_digital_connector_duplicate_state, @@ -719,36 +767,8 @@ err: return NULL; } -static void intel_dp_register_mst_connector(struct drm_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->dev); - - if (dev_priv->fbdev) - drm_fb_helper_add_one_connector(&dev_priv->fbdev->helper, - connector); - - drm_connector_register(connector); -} - -static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr, - struct drm_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->dev); - - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id, connector->name); - drm_connector_unregister(connector); - - if (dev_priv->fbdev) - drm_fb_helper_remove_one_connector(&dev_priv->fbdev->helper, - connector); - - drm_connector_put(connector); -} - static const struct drm_dp_mst_topology_cbs mst_cbs = { .add_connector = intel_dp_add_mst_connector, - .register_connector = intel_dp_register_mst_connector, - .destroy_connector = intel_dp_destroy_mst_connector, }; static struct intel_dp_mst_encoder * @@ -785,6 +805,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum intel_encoder->pipe_mask = ~0; intel_encoder->compute_config = intel_dp_mst_compute_config; + intel_encoder->compute_config_late = intel_dp_mst_compute_config_late; intel_encoder->disable = intel_mst_disable_dp; intel_encoder->post_disable = intel_mst_post_disable_dp; intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp; diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index e5bfe5245276..b45185b80bec 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -45,6 +45,22 @@ * commit phase. */ +struct intel_dpll_mgr { + const struct dpll_info *dpll_info; + + bool (*get_dplls)(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct intel_encoder *encoder); + void (*put_dplls)(struct intel_atomic_state *state, + struct intel_crtc *crtc); + void (*update_active_dpll)(struct intel_atomic_state *state, + struct intel_crtc *crtc, + struct intel_encoder *encoder); + void (*update_ref_clks)(struct drm_i915_private *i915); + void (*dump_hw_state)(struct drm_i915_private *dev_priv, + const struct intel_dpll_hw_state *hw_state); +}; + static void intel_atomic_duplicate_dpll_state(struct drm_i915_private *dev_priv, struct intel_shared_dpll_state *shared_dpll) @@ -52,8 +68,8 @@ intel_atomic_duplicate_dpll_state(struct drm_i915_private *dev_priv, enum intel_dpll_id i; /* Copy shared dpll state */ - for (i = 0; i < dev_priv->num_shared_dpll; i++) { - struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i]; + for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { + struct intel_shared_dpll *pll = &dev_priv->dpll.shared_dplls[i]; shared_dpll[i] = pll->state; } @@ -64,7 +80,7 @@ intel_atomic_get_shared_dpll_state(struct drm_atomic_state *s) { struct intel_atomic_state *state = to_intel_atomic_state(s); - WARN_ON(!drm_modeset_is_locked(&s->dev->mode_config.connection_mutex)); + drm_WARN_ON(s->dev, !drm_modeset_is_locked(&s->dev->mode_config.connection_mutex)); if (!state->dpll_set) { state->dpll_set = true; @@ -88,7 +104,7 @@ struct intel_shared_dpll * intel_get_shared_dpll_by_id(struct drm_i915_private *dev_priv, enum intel_dpll_id id) { - return &dev_priv->shared_dplls[id]; + return &dev_priv->dpll.shared_dplls[id]; } /** @@ -103,11 +119,14 @@ enum intel_dpll_id intel_get_shared_dpll_id(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { - if (drm_WARN_ON(&dev_priv->drm, pll < dev_priv->shared_dplls || - pll > &dev_priv->shared_dplls[dev_priv->num_shared_dpll])) + long pll_idx = pll - dev_priv->dpll.shared_dplls; + + if (drm_WARN_ON(&dev_priv->drm, + pll_idx < 0 || + pll_idx >= dev_priv->dpll.num_shared_dpll)) return -1; - return (enum intel_dpll_id) (pll - dev_priv->shared_dplls); + return pll_idx; } /* For ILK+ */ @@ -144,7 +163,7 @@ void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state) if (drm_WARN_ON(&dev_priv->drm, pll == NULL)) return; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); drm_WARN_ON(&dev_priv->drm, !pll->state.crtc_mask); if (!pll->active_mask) { drm_dbg(&dev_priv->drm, "setting up %s\n", pll->info->name); @@ -153,7 +172,7 @@ void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state) pll->info->funcs->prepare(dev_priv, pll); } - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } /** @@ -173,7 +192,7 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state) if (drm_WARN_ON(&dev_priv->drm, pll == NULL)) return; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); old_mask = pll->active_mask; if (drm_WARN_ON(&dev_priv->drm, !(pll->state.crtc_mask & crtc_mask)) || @@ -199,7 +218,7 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state) pll->on = true; out: - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } /** @@ -222,7 +241,7 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state) if (pll == NULL) return; - mutex_lock(&dev_priv->dpll_lock); + mutex_lock(&dev_priv->dpll.lock); if (drm_WARN_ON(&dev_priv->drm, !(pll->active_mask & crtc_mask))) goto out; @@ -243,7 +262,7 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state) pll->on = false; out: - mutex_unlock(&dev_priv->dpll_lock); + mutex_unlock(&dev_priv->dpll.lock); } static struct intel_shared_dpll * @@ -262,7 +281,7 @@ intel_find_shared_dpll(struct intel_atomic_state *state, drm_WARN_ON(&dev_priv->drm, dpll_mask & ~(BIT(I915_NUM_PLLS) - 1)); for_each_set_bit(i, &dpll_mask, I915_NUM_PLLS) { - pll = &dev_priv->shared_dplls[i]; + pll = &dev_priv->dpll.shared_dplls[i]; /* Only want to check enabled timings first */ if (shared_dpll[i].crtc_mask == 0) { @@ -362,9 +381,9 @@ void intel_shared_dpll_swap_state(struct intel_atomic_state *state) if (!state->dpll_set) return; - for (i = 0; i < dev_priv->num_shared_dpll; i++) { + for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) { struct intel_shared_dpll *pll = - &dev_priv->shared_dplls[i]; + &dev_priv->dpll.shared_dplls[i]; swap(pll->state, shared_dpll[i]); } @@ -462,7 +481,7 @@ static bool ibx_get_dpll(struct intel_atomic_state *state, if (HAS_PCH_IBX(dev_priv)) { /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */ i = (enum intel_dpll_id) crtc->pipe; - pll = &dev_priv->shared_dplls[i]; + pll = &dev_priv->dpll.shared_dplls[i]; drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] using pre-allocated %s\n", @@ -506,6 +525,19 @@ static const struct intel_shared_dpll_funcs ibx_pch_dpll_funcs = { .get_hw_state = ibx_pch_dpll_get_hw_state, }; +static const struct dpll_info pch_plls[] = { + { "PCH DPLL A", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_A, 0 }, + { "PCH DPLL B", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_B, 0 }, + { }, +}; + +static const struct intel_dpll_mgr pch_pll_mgr = { + .dpll_info = pch_plls, + .get_dplls = ibx_get_dpll, + .put_dplls = intel_put_dpll, + .dump_hw_state = ibx_dump_hw_state, +}; + static void hsw_ddi_wrpll_enable(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll) { @@ -818,8 +850,8 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */, } static struct intel_shared_dpll * -hsw_ddi_hdmi_get_dpll(struct intel_atomic_state *state, - struct intel_crtc *crtc) +hsw_ddi_wrpll_get_dpll(struct intel_atomic_state *state, + struct intel_crtc *crtc) { struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); @@ -846,8 +878,47 @@ hsw_ddi_hdmi_get_dpll(struct intel_atomic_state *state, return pll; } +static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, + const struct intel_shared_dpll *pll) +{ + int refclk; + int n, p, r; + u32 wrpll = pll->state.hw_state.wrpll; + + switch (wrpll & WRPLL_REF_MASK) { + case WRPLL_REF_SPECIAL_HSW: + /* Muxed-SSC for BDW, non-SSC for non-ULT HSW. */ + if (IS_HASWELL(dev_priv) && !IS_HSW_ULT(dev_priv)) { + refclk = dev_priv->dpll.ref_clks.nssc; + break; + } + /* fall through */ + case WRPLL_REF_PCH_SSC: + /* + * We could calculate spread here, but our checking + * code only cares about 5% accuracy, and spread is a max of + * 0.5% downspread. + */ + refclk = dev_priv->dpll.ref_clks.ssc; + break; + case WRPLL_REF_LCPLL: + refclk = 2700000; + break; + default: + MISSING_CASE(wrpll); + return 0; + } + + r = wrpll & WRPLL_DIVIDER_REF_MASK; + p = (wrpll & WRPLL_DIVIDER_POST_MASK) >> WRPLL_DIVIDER_POST_SHIFT; + n = (wrpll & WRPLL_DIVIDER_FB_MASK) >> WRPLL_DIVIDER_FB_SHIFT; + + /* Convert to KHz, p & r have a fixed point portion */ + return (refclk * n / 10) / (p * r) * 2; +} + static struct intel_shared_dpll * -hsw_ddi_dp_get_dpll(struct intel_crtc_state *crtc_state) +hsw_ddi_lcpll_get_dpll(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); struct intel_shared_dpll *pll; @@ -878,6 +949,69 @@ hsw_ddi_dp_get_dpll(struct intel_crtc_state *crtc_state) return pll; } +static int hsw_ddi_lcpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + int link_clock = 0; + + switch (pll->info->id) { + case DPLL_ID_LCPLL_810: + link_clock = 81000; + break; + case DPLL_ID_LCPLL_1350: + link_clock = 135000; + break; + case DPLL_ID_LCPLL_2700: + link_clock = 270000; + break; + default: + drm_WARN(&i915->drm, 1, "bad port clock sel\n"); + break; + } + + return link_clock * 2; +} + +static struct intel_shared_dpll * +hsw_ddi_spll_get_dpll(struct intel_atomic_state *state, + struct intel_crtc *crtc) +{ + struct intel_crtc_state *crtc_state = + intel_atomic_get_new_crtc_state(state, crtc); + + if (drm_WARN_ON(crtc->base.dev, crtc_state->port_clock / 2 != 135000)) + return NULL; + + crtc_state->dpll_hw_state.spll = SPLL_PLL_ENABLE | SPLL_FREQ_1350MHz | + SPLL_REF_MUXED_SSC; + + return intel_find_shared_dpll(state, crtc, &crtc_state->dpll_hw_state, + BIT(DPLL_ID_SPLL)); +} + +static int hsw_ddi_spll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + int link_clock = 0; + + switch (pll->state.hw_state.spll & SPLL_FREQ_MASK) { + case SPLL_FREQ_810MHz: + link_clock = 81000; + break; + case SPLL_FREQ_1350MHz: + link_clock = 135000; + break; + case SPLL_FREQ_2700MHz: + link_clock = 270000; + break; + default: + drm_WARN(&i915->drm, 1, "bad spll freq\n"); + break; + } + + return link_clock * 2; +} + static bool hsw_get_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder) @@ -889,23 +1023,14 @@ static bool hsw_get_dpll(struct intel_atomic_state *state, memset(&crtc_state->dpll_hw_state, 0, sizeof(crtc_state->dpll_hw_state)); - if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) { - pll = hsw_ddi_hdmi_get_dpll(state, crtc); - } else if (intel_crtc_has_dp_encoder(crtc_state)) { - pll = hsw_ddi_dp_get_dpll(crtc_state); - } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) { - if (WARN_ON(crtc_state->port_clock / 2 != 135000)) - return false; - - crtc_state->dpll_hw_state.spll = - SPLL_PLL_ENABLE | SPLL_FREQ_1350MHz | SPLL_REF_MUXED_SSC; - - pll = intel_find_shared_dpll(state, crtc, - &crtc_state->dpll_hw_state, - BIT(DPLL_ID_SPLL)); - } else { + if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) + pll = hsw_ddi_wrpll_get_dpll(state, crtc); + else if (intel_crtc_has_dp_encoder(crtc_state)) + pll = hsw_ddi_lcpll_get_dpll(crtc_state); + else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) + pll = hsw_ddi_spll_get_dpll(state, crtc); + else return false; - } if (!pll) return false; @@ -918,6 +1043,16 @@ static bool hsw_get_dpll(struct intel_atomic_state *state, return true; } +static void hsw_update_dpll_ref_clks(struct drm_i915_private *i915) +{ + i915->dpll.ref_clks.ssc = 135000; + /* Non-SSC is only used on non-ULT HSW. */ + if (intel_de_read(i915, FUSE_STRAP3) & HSW_REF_CLK_SELECT) + i915->dpll.ref_clks.nssc = 24000; + else + i915->dpll.ref_clks.nssc = 135000; +} + static void hsw_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { @@ -929,12 +1064,14 @@ static const struct intel_shared_dpll_funcs hsw_ddi_wrpll_funcs = { .enable = hsw_ddi_wrpll_enable, .disable = hsw_ddi_wrpll_disable, .get_hw_state = hsw_ddi_wrpll_get_hw_state, + .get_freq = hsw_ddi_wrpll_get_freq, }; static const struct intel_shared_dpll_funcs hsw_ddi_spll_funcs = { .enable = hsw_ddi_spll_enable, .disable = hsw_ddi_spll_disable, .get_hw_state = hsw_ddi_spll_get_hw_state, + .get_freq = hsw_ddi_spll_get_freq, }; static void hsw_ddi_lcpll_enable(struct drm_i915_private *dev_priv, @@ -958,6 +1095,25 @@ static const struct intel_shared_dpll_funcs hsw_ddi_lcpll_funcs = { .enable = hsw_ddi_lcpll_enable, .disable = hsw_ddi_lcpll_disable, .get_hw_state = hsw_ddi_lcpll_get_hw_state, + .get_freq = hsw_ddi_lcpll_get_freq, +}; + +static const struct dpll_info hsw_plls[] = { + { "WRPLL 1", &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL1, 0 }, + { "WRPLL 2", &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL2, 0 }, + { "SPLL", &hsw_ddi_spll_funcs, DPLL_ID_SPLL, 0 }, + { "LCPLL 810", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_810, INTEL_DPLL_ALWAYS_ON }, + { "LCPLL 1350", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_1350, INTEL_DPLL_ALWAYS_ON }, + { "LCPLL 2700", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_2700, INTEL_DPLL_ALWAYS_ON }, + { }, +}; + +static const struct intel_dpll_mgr hsw_pll_mgr = { + .dpll_info = hsw_plls, + .get_dplls = hsw_get_dpll, + .put_dplls = intel_put_dpll, + .update_ref_clks = hsw_update_dpll_ref_clks, + .dump_hw_state = hsw_dump_hw_state, }; struct skl_dpll_regs { @@ -1230,6 +1386,7 @@ struct skl_wrpll_params { static void skl_wrpll_params_populate(struct skl_wrpll_params *params, u64 afe_clock, + int ref_clock, u64 central_freq, u32 p0, u32 p1, u32 p2) { @@ -1289,14 +1446,15 @@ static void skl_wrpll_params_populate(struct skl_wrpll_params *params, * Intermediate values are in Hz. * Divide by MHz to match bsepc */ - params->dco_integer = div_u64(dco_freq, 24 * MHz(1)); + params->dco_integer = div_u64(dco_freq, ref_clock * KHz(1)); params->dco_fraction = - div_u64((div_u64(dco_freq, 24) - + div_u64((div_u64(dco_freq, ref_clock / KHz(1)) - params->dco_integer * MHz(1)) * 0x8000, MHz(1)); } static bool skl_ddi_calculate_wrpll(int clock /* in Hz */, + int ref_clock, struct skl_wrpll_params *wrpll_params) { u64 afe_clock = clock * 5; /* AFE Clock is 5x Pixel clock */ @@ -1362,14 +1520,15 @@ skip_remaining_dividers: */ p0 = p1 = p2 = 0; skl_wrpll_get_multipliers(ctx.p, &p0, &p1, &p2); - skl_wrpll_params_populate(wrpll_params, afe_clock, ctx.central_freq, - p0, p1, p2); + skl_wrpll_params_populate(wrpll_params, afe_clock, ref_clock, + ctx.central_freq, p0, p1, p2); return true; } static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); u32 ctrl1, cfgcr1, cfgcr2; struct skl_wrpll_params wrpll_params = { 0, }; @@ -1382,6 +1541,7 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) ctrl1 |= DPLL_CTRL1_HDMI_MODE(0); if (!skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000, + i915->dpll.ref_clks.nssc, &wrpll_params)) return false; @@ -1404,6 +1564,64 @@ static bool skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) return true; } +static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; + int ref_clock = i915->dpll.ref_clks.nssc; + u32 p0, p1, p2, dco_freq; + + p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK; + p2 = pll_state->cfgcr2 & DPLL_CFGCR2_KDIV_MASK; + + if (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_MODE(1)) + p1 = (pll_state->cfgcr2 & DPLL_CFGCR2_QDIV_RATIO_MASK) >> 8; + else + p1 = 1; + + + switch (p0) { + case DPLL_CFGCR2_PDIV_1: + p0 = 1; + break; + case DPLL_CFGCR2_PDIV_2: + p0 = 2; + break; + case DPLL_CFGCR2_PDIV_3: + p0 = 3; + break; + case DPLL_CFGCR2_PDIV_7: + p0 = 7; + break; + } + + switch (p2) { + case DPLL_CFGCR2_KDIV_5: + p2 = 5; + break; + case DPLL_CFGCR2_KDIV_2: + p2 = 2; + break; + case DPLL_CFGCR2_KDIV_3: + p2 = 3; + break; + case DPLL_CFGCR2_KDIV_1: + p2 = 1; + break; + } + + dco_freq = (pll_state->cfgcr1 & DPLL_CFGCR1_DCO_INTEGER_MASK) * + ref_clock; + + dco_freq += ((pll_state->cfgcr1 & DPLL_CFGCR1_DCO_FRACTION_MASK) >> 9) * + ref_clock / 0x8000; + + if (drm_WARN_ON(&i915->drm, p0 == 0 || p1 == 0 || p2 == 0)) + return 0; + + return dco_freq / (p0 * p1 * p2 * 5); +} + static bool skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) { @@ -1444,6 +1662,40 @@ skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) return true; } +static int skl_ddi_lcpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + int link_clock = 0; + + switch ((pll->state.hw_state.ctrl1 & + DPLL_CTRL1_LINK_RATE_MASK(0)) >> + DPLL_CTRL1_LINK_RATE_SHIFT(0)) { + case DPLL_CTRL1_LINK_RATE_810: + link_clock = 81000; + break; + case DPLL_CTRL1_LINK_RATE_1080: + link_clock = 108000; + break; + case DPLL_CTRL1_LINK_RATE_1350: + link_clock = 135000; + break; + case DPLL_CTRL1_LINK_RATE_1620: + link_clock = 162000; + break; + case DPLL_CTRL1_LINK_RATE_2160: + link_clock = 216000; + break; + case DPLL_CTRL1_LINK_RATE_2700: + link_clock = 270000; + break; + default: + drm_WARN(&i915->drm, 1, "Unsupported link rate\n"); + break; + } + + return link_clock * 2; +} + static bool skl_get_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder) @@ -1493,6 +1745,25 @@ static bool skl_get_dpll(struct intel_atomic_state *state, return true; } +static int skl_ddi_pll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + /* + * ctrl1 register is already shifted for each pll, just use 0 to get + * the internal shift for each field + */ + if (pll->state.hw_state.ctrl1 & DPLL_CTRL1_HDMI_MODE(0)) + return skl_ddi_wrpll_get_freq(i915, pll); + else + return skl_ddi_lcpll_get_freq(i915, pll); +} + +static void skl_update_dpll_ref_clks(struct drm_i915_private *i915) +{ + /* No SSC ref */ + i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref; +} + static void skl_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { @@ -1507,12 +1778,30 @@ static const struct intel_shared_dpll_funcs skl_ddi_pll_funcs = { .enable = skl_ddi_pll_enable, .disable = skl_ddi_pll_disable, .get_hw_state = skl_ddi_pll_get_hw_state, + .get_freq = skl_ddi_pll_get_freq, }; static const struct intel_shared_dpll_funcs skl_ddi_dpll0_funcs = { .enable = skl_ddi_dpll0_enable, .disable = skl_ddi_dpll0_disable, .get_hw_state = skl_ddi_dpll0_get_hw_state, + .get_freq = skl_ddi_pll_get_freq, +}; + +static const struct dpll_info skl_plls[] = { + { "DPLL 0", &skl_ddi_dpll0_funcs, DPLL_ID_SKL_DPLL0, INTEL_DPLL_ALWAYS_ON }, + { "DPLL 1", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 }, + { "DPLL 2", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 }, + { "DPLL 3", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL3, 0 }, + { }, +}; + +static const struct intel_dpll_mgr skl_pll_mgr = { + .dpll_info = skl_plls, + .get_dplls = skl_get_dpll, + .put_dplls = intel_put_dpll, + .update_ref_clks = skl_update_dpll_ref_clks, + .dump_hw_state = skl_dump_hw_state, }; static void bxt_ddi_pll_enable(struct drm_i915_private *dev_priv, @@ -1785,7 +2074,7 @@ bxt_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state, clk_div->p1 = best_clock.p1; clk_div->p2 = best_clock.p2; - WARN_ON(best_clock.m1 != 2); + drm_WARN_ON(&i915->drm, best_clock.m1 != 2); clk_div->n = best_clock.n; clk_div->m2_int = best_clock.m2 >> 22; clk_div->m2_frac = best_clock.m2 & ((1 << 22) - 1); @@ -1903,6 +2192,23 @@ bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc_state *crtc_state) return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div); } +static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; + struct dpll clock; + + clock.m1 = 2; + clock.m2 = (pll_state->pll0 & PORT_PLL_M2_MASK) << 22; + if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE) + clock.m2 |= pll_state->pll2 & PORT_PLL_M2_FRAC_MASK; + clock.n = (pll_state->pll1 & PORT_PLL_N_MASK) >> PORT_PLL_N_SHIFT; + clock.p1 = (pll_state->ebb0 & PORT_PLL_P1_MASK) >> PORT_PLL_P1_SHIFT; + clock.p2 = (pll_state->ebb0 & PORT_PLL_P2_MASK) >> PORT_PLL_P2_SHIFT; + + return chv_calc_dpll_params(i915->dpll.ref_clks.nssc, &clock); +} + static bool bxt_get_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder) @@ -1936,6 +2242,13 @@ static bool bxt_get_dpll(struct intel_atomic_state *state, return true; } +static void bxt_update_dpll_ref_clks(struct drm_i915_private *i915) +{ + i915->dpll.ref_clks.ssc = 100000; + i915->dpll.ref_clks.nssc = 100000; + /* DSI non-SSC ref 19.2MHz */ +} + static void bxt_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { @@ -1959,66 +2272,7 @@ static const struct intel_shared_dpll_funcs bxt_ddi_pll_funcs = { .enable = bxt_ddi_pll_enable, .disable = bxt_ddi_pll_disable, .get_hw_state = bxt_ddi_pll_get_hw_state, -}; - -struct intel_dpll_mgr { - const struct dpll_info *dpll_info; - - bool (*get_dplls)(struct intel_atomic_state *state, - struct intel_crtc *crtc, - struct intel_encoder *encoder); - void (*put_dplls)(struct intel_atomic_state *state, - struct intel_crtc *crtc); - void (*update_active_dpll)(struct intel_atomic_state *state, - struct intel_crtc *crtc, - struct intel_encoder *encoder); - void (*dump_hw_state)(struct drm_i915_private *dev_priv, - const struct intel_dpll_hw_state *hw_state); -}; - -static const struct dpll_info pch_plls[] = { - { "PCH DPLL A", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_A, 0 }, - { "PCH DPLL B", &ibx_pch_dpll_funcs, DPLL_ID_PCH_PLL_B, 0 }, - { }, -}; - -static const struct intel_dpll_mgr pch_pll_mgr = { - .dpll_info = pch_plls, - .get_dplls = ibx_get_dpll, - .put_dplls = intel_put_dpll, - .dump_hw_state = ibx_dump_hw_state, -}; - -static const struct dpll_info hsw_plls[] = { - { "WRPLL 1", &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL1, 0 }, - { "WRPLL 2", &hsw_ddi_wrpll_funcs, DPLL_ID_WRPLL2, 0 }, - { "SPLL", &hsw_ddi_spll_funcs, DPLL_ID_SPLL, 0 }, - { "LCPLL 810", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_810, INTEL_DPLL_ALWAYS_ON }, - { "LCPLL 1350", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_1350, INTEL_DPLL_ALWAYS_ON }, - { "LCPLL 2700", &hsw_ddi_lcpll_funcs, DPLL_ID_LCPLL_2700, INTEL_DPLL_ALWAYS_ON }, - { }, -}; - -static const struct intel_dpll_mgr hsw_pll_mgr = { - .dpll_info = hsw_plls, - .get_dplls = hsw_get_dpll, - .put_dplls = intel_put_dpll, - .dump_hw_state = hsw_dump_hw_state, -}; - -static const struct dpll_info skl_plls[] = { - { "DPLL 0", &skl_ddi_dpll0_funcs, DPLL_ID_SKL_DPLL0, INTEL_DPLL_ALWAYS_ON }, - { "DPLL 1", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL1, 0 }, - { "DPLL 2", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL2, 0 }, - { "DPLL 3", &skl_ddi_pll_funcs, DPLL_ID_SKL_DPLL3, 0 }, - { }, -}; - -static const struct intel_dpll_mgr skl_pll_mgr = { - .dpll_info = skl_plls, - .get_dplls = skl_get_dpll, - .put_dplls = intel_put_dpll, - .dump_hw_state = skl_dump_hw_state, + .get_freq = bxt_ddi_pll_get_freq, }; static const struct dpll_info bxt_plls[] = { @@ -2032,6 +2286,7 @@ static const struct intel_dpll_mgr bxt_pll_mgr = { .dpll_info = bxt_plls, .get_dplls = bxt_get_dpll, .put_dplls = intel_put_dpll, + .update_ref_clks = bxt_update_dpll_ref_clks, .dump_hw_state = bxt_dump_hw_state, }; @@ -2275,27 +2530,12 @@ static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, params->dco_fraction = dco & 0x7fff; } -int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv) -{ - int ref_clock = dev_priv->cdclk.hw.ref; - - /* - * For ICL+, the spec states: if reference frequency is 38.4, - * use 19.2 because the DPLL automatically divides that by 2. - */ - if (INTEL_GEN(dev_priv) >= 11 && ref_clock == 38400) - ref_clock = 19200; - - return ref_clock; -} - static bool -cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, - struct skl_wrpll_params *wrpll_params) +__cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, + struct skl_wrpll_params *wrpll_params, + int ref_clock) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); u32 afe_clock = crtc_state->port_clock * 5; - u32 ref_clock; u32 dco_min = 7998000; u32 dco_max = 10000000; u32 dco_mid = (dco_min + dco_max) / 2; @@ -2327,15 +2567,22 @@ cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, return false; cnl_wrpll_get_multipliers(best_div, &pdiv, &qdiv, &kdiv); - - ref_clock = cnl_hdmi_pll_ref_clock(dev_priv); - cnl_wrpll_params_populate(wrpll_params, best_dco, ref_clock, pdiv, qdiv, kdiv); return true; } +static bool +cnl_ddi_calculate_wrpll(struct intel_crtc_state *crtc_state, + struct skl_wrpll_params *wrpll_params) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + + return __cnl_ddi_calculate_wrpll(crtc_state, wrpll_params, + i915->dpll.ref_clks.nssc); +} + static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) { u32 cfgcr0, cfgcr1; @@ -2363,6 +2610,68 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state) return true; } +static int __cnl_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv, + const struct intel_shared_dpll *pll, + int ref_clock) +{ + const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; + u32 p0, p1, p2, dco_freq; + + p0 = pll_state->cfgcr1 & DPLL_CFGCR1_PDIV_MASK; + p2 = pll_state->cfgcr1 & DPLL_CFGCR1_KDIV_MASK; + + if (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_MODE(1)) + p1 = (pll_state->cfgcr1 & DPLL_CFGCR1_QDIV_RATIO_MASK) >> + DPLL_CFGCR1_QDIV_RATIO_SHIFT; + else + p1 = 1; + + + switch (p0) { + case DPLL_CFGCR1_PDIV_2: + p0 = 2; + break; + case DPLL_CFGCR1_PDIV_3: + p0 = 3; + break; + case DPLL_CFGCR1_PDIV_5: + p0 = 5; + break; + case DPLL_CFGCR1_PDIV_7: + p0 = 7; + break; + } + + switch (p2) { + case DPLL_CFGCR1_KDIV_1: + p2 = 1; + break; + case DPLL_CFGCR1_KDIV_2: + p2 = 2; + break; + case DPLL_CFGCR1_KDIV_3: + p2 = 3; + break; + } + + dco_freq = (pll_state->cfgcr0 & DPLL_CFGCR0_DCO_INTEGER_MASK) * + ref_clock; + + dco_freq += (((pll_state->cfgcr0 & DPLL_CFGCR0_DCO_FRACTION_MASK) >> + DPLL_CFGCR0_DCO_FRACTION_SHIFT) * ref_clock) / 0x8000; + + if (drm_WARN_ON(&dev_priv->drm, p0 == 0 || p1 == 0 || p2 == 0)) + return 0; + + return dco_freq / (p0 * p1 * p2 * 5); +} + +static int cnl_ddi_wrpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + return __cnl_ddi_wrpll_get_freq(i915, pll, i915->dpll.ref_clks.nssc); +} + static bool cnl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) { @@ -2408,6 +2717,44 @@ cnl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state) return true; } +static int cnl_ddi_lcpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + int link_clock = 0; + + switch (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_LINK_RATE_MASK) { + case DPLL_CFGCR0_LINK_RATE_810: + link_clock = 81000; + break; + case DPLL_CFGCR0_LINK_RATE_1080: + link_clock = 108000; + break; + case DPLL_CFGCR0_LINK_RATE_1350: + link_clock = 135000; + break; + case DPLL_CFGCR0_LINK_RATE_1620: + link_clock = 162000; + break; + case DPLL_CFGCR0_LINK_RATE_2160: + link_clock = 216000; + break; + case DPLL_CFGCR0_LINK_RATE_2700: + link_clock = 270000; + break; + case DPLL_CFGCR0_LINK_RATE_3240: + link_clock = 324000; + break; + case DPLL_CFGCR0_LINK_RATE_4050: + link_clock = 405000; + break; + default: + drm_WARN(&i915->drm, 1, "Unsupported link rate\n"); + break; + } + + return link_clock * 2; +} + static bool cnl_get_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder) @@ -2457,6 +2804,21 @@ static bool cnl_get_dpll(struct intel_atomic_state *state, return true; } +static int cnl_ddi_pll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + if (pll->state.hw_state.cfgcr0 & DPLL_CFGCR0_HDMI_MODE) + return cnl_ddi_wrpll_get_freq(i915, pll); + else + return cnl_ddi_lcpll_get_freq(i915, pll); +} + +static void cnl_update_dpll_ref_clks(struct drm_i915_private *i915) +{ + /* No SSC reference */ + i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref; +} + static void cnl_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { @@ -2470,6 +2832,7 @@ static const struct intel_shared_dpll_funcs cnl_ddi_pll_funcs = { .enable = cnl_ddi_pll_enable, .disable = cnl_ddi_pll_disable, .get_hw_state = cnl_ddi_pll_get_hw_state, + .get_freq = cnl_ddi_pll_get_freq, }; static const struct dpll_info cnl_plls[] = { @@ -2483,6 +2846,7 @@ static const struct intel_dpll_mgr cnl_pll_mgr = { .dpll_info = cnl_plls, .get_dplls = cnl_get_dpll, .put_dplls = intel_put_dpll, + .update_ref_clks = cnl_update_dpll_ref_clks, .dump_hw_state = cnl_dump_hw_state, }; @@ -2578,7 +2942,7 @@ static bool icl_calc_dp_combo_pll(struct intel_crtc_state *crtc_state, { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); const struct icl_combo_pll_params *params = - dev_priv->cdclk.hw.ref == 24000 ? + dev_priv->dpll.ref_clks.nssc == 24000 ? icl_dp_combo_pll_24MHz_values : icl_dp_combo_pll_19_2MHz_values; int clock = crtc_state->port_clock; @@ -2601,9 +2965,9 @@ static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); if (INTEL_GEN(dev_priv) >= 12) { - switch (dev_priv->cdclk.hw.ref) { + switch (dev_priv->dpll.ref_clks.nssc) { default: - MISSING_CASE(dev_priv->cdclk.hw.ref); + MISSING_CASE(dev_priv->dpll.ref_clks.nssc); /* fall-through */ case 19200: case 38400: @@ -2614,9 +2978,9 @@ static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, break; } } else { - switch (dev_priv->cdclk.hw.ref) { + switch (dev_priv->dpll.ref_clks.nssc) { default: - MISSING_CASE(dev_priv->cdclk.hw.ref); + MISSING_CASE(dev_priv->dpll.ref_clks.nssc); /* fall-through */ case 19200: case 38400: @@ -2631,6 +2995,49 @@ static bool icl_calc_tbt_pll(struct intel_crtc_state *crtc_state, return true; } +static int icl_ddi_tbt_pll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + /* + * The PLL outputs multiple frequencies at the same time, selection is + * made at DDI clock mux level. + */ + drm_WARN_ON(&i915->drm, 1); + + return 0; +} + +static int icl_wrpll_ref_clock(struct drm_i915_private *i915) +{ + int ref_clock = i915->dpll.ref_clks.nssc; + + /* + * For ICL+, the spec states: if reference frequency is 38.4, + * use 19.2 because the DPLL automatically divides that by 2. + */ + if (ref_clock == 38400) + ref_clock = 19200; + + return ref_clock; +} + +static bool +icl_calc_wrpll(struct intel_crtc_state *crtc_state, + struct skl_wrpll_params *wrpll_params) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + + return __cnl_ddi_calculate_wrpll(crtc_state, wrpll_params, + icl_wrpll_ref_clock(i915)); +} + +static int icl_ddi_combo_pll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + return __cnl_ddi_wrpll_get_freq(i915, pll, + icl_wrpll_ref_clock(i915)); +} + static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state, struct intel_encoder *encoder, struct intel_dpll_hw_state *pll_state) @@ -2645,7 +3052,7 @@ static bool icl_calc_dpll_state(struct intel_crtc_state *crtc_state, ret = icl_calc_tbt_pll(crtc_state, &pll_params); else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) || intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) - ret = cnl_ddi_calculate_wrpll(crtc_state, &pll_params); + ret = icl_calc_wrpll(crtc_state, &pll_params); else ret = icl_calc_dp_combo_pll(crtc_state, &pll_params); @@ -2768,7 +3175,7 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, struct intel_dpll_hw_state *pll_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - int refclk_khz = dev_priv->cdclk.hw.ref; + int refclk_khz = dev_priv->dpll.ref_clks.nssc; int clock = crtc_state->port_clock; u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac; u32 iref_ndiv, iref_trim, iref_pulse_w; @@ -2969,6 +3376,78 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state, return true; } +static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *dev_priv, + const struct intel_shared_dpll *pll) +{ + const struct intel_dpll_hw_state *pll_state = &pll->state.hw_state; + u32 m1, m2_int, m2_frac, div1, div2, ref_clock; + u64 tmp; + + ref_clock = dev_priv->dpll.ref_clks.nssc; + + if (INTEL_GEN(dev_priv) >= 12) { + m1 = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBPREDIV_MASK; + m1 = m1 >> DKL_PLL_DIV0_FBPREDIV_SHIFT; + m2_int = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBDIV_INT_MASK; + + if (pll_state->mg_pll_bias & DKL_PLL_BIAS_FRAC_EN_H) { + m2_frac = pll_state->mg_pll_bias & + DKL_PLL_BIAS_FBDIV_FRAC_MASK; + m2_frac = m2_frac >> DKL_PLL_BIAS_FBDIV_SHIFT; + } else { + m2_frac = 0; + } + } else { + m1 = pll_state->mg_pll_div1 & MG_PLL_DIV1_FBPREDIV_MASK; + m2_int = pll_state->mg_pll_div0 & MG_PLL_DIV0_FBDIV_INT_MASK; + + if (pll_state->mg_pll_div0 & MG_PLL_DIV0_FRACNEN_H) { + m2_frac = pll_state->mg_pll_div0 & + MG_PLL_DIV0_FBDIV_FRAC_MASK; + m2_frac = m2_frac >> MG_PLL_DIV0_FBDIV_FRAC_SHIFT; + } else { + m2_frac = 0; + } + } + + switch (pll_state->mg_clktop2_hsclkctl & + MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK) { + case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_2: + div1 = 2; + break; + case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_3: + div1 = 3; + break; + case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_5: + div1 = 5; + break; + case MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_7: + div1 = 7; + break; + default: + MISSING_CASE(pll_state->mg_clktop2_hsclkctl); + return 0; + } + + div2 = (pll_state->mg_clktop2_hsclkctl & + MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK) >> + MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_SHIFT; + + /* div2 value of 0 is same as 1 means no div */ + if (div2 == 0) + div2 = 1; + + /* + * Adjust the original formula to delay the division by 2^22 in order to + * minimize possible rounding errors. + */ + tmp = (u64)m1 * m2_int * ref_clock + + (((u64)m1 * m2_frac * ref_clock) >> 22); + tmp = div_u64(tmp, 5 * div1 * div2); + + return tmp; +} + /** * icl_set_active_port_dpll - select the active port DPLL for a given CRTC * @crtc_state: state for the CRTC to select the DPLL for @@ -3201,7 +3680,7 @@ static bool mg_pll_get_hw_state(struct drm_i915_private *dev_priv, hw_state->mg_pll_tdc_coldst_bias = intel_de_read(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port)); - if (dev_priv->cdclk.hw.ref == 38400) { + if (dev_priv->dpll.ref_clks.nssc == 38400) { hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART; hw_state->mg_pll_bias_mask = 0; } else { @@ -3682,6 +4161,12 @@ static void mg_pll_disable(struct drm_i915_private *dev_priv, icl_pll_disable(dev_priv, pll, enable_reg); } +static void icl_update_dpll_ref_clks(struct drm_i915_private *i915) +{ + /* No SSC ref */ + i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref; +} + static void icl_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { @@ -3709,18 +4194,21 @@ static const struct intel_shared_dpll_funcs combo_pll_funcs = { .enable = combo_pll_enable, .disable = combo_pll_disable, .get_hw_state = combo_pll_get_hw_state, + .get_freq = icl_ddi_combo_pll_get_freq, }; static const struct intel_shared_dpll_funcs tbt_pll_funcs = { .enable = tbt_pll_enable, .disable = tbt_pll_disable, .get_hw_state = tbt_pll_get_hw_state, + .get_freq = icl_ddi_tbt_pll_get_freq, }; static const struct intel_shared_dpll_funcs mg_pll_funcs = { .enable = mg_pll_enable, .disable = mg_pll_disable, .get_hw_state = mg_pll_get_hw_state, + .get_freq = icl_ddi_mg_pll_get_freq, }; static const struct dpll_info icl_plls[] = { @@ -3739,6 +4227,7 @@ static const struct intel_dpll_mgr icl_pll_mgr = { .get_dplls = icl_get_dplls, .put_dplls = icl_put_dplls, .update_active_dpll = icl_update_active_dpll, + .update_ref_clks = icl_update_dpll_ref_clks, .dump_hw_state = icl_dump_hw_state, }; @@ -3753,6 +4242,7 @@ static const struct intel_dpll_mgr ehl_pll_mgr = { .dpll_info = ehl_plls, .get_dplls = icl_get_dplls, .put_dplls = icl_put_dplls, + .update_ref_clks = icl_update_dpll_ref_clks, .dump_hw_state = icl_dump_hw_state, }; @@ -3760,6 +4250,7 @@ static const struct intel_shared_dpll_funcs dkl_pll_funcs = { .enable = mg_pll_enable, .disable = mg_pll_disable, .get_hw_state = dkl_pll_get_hw_state, + .get_freq = icl_ddi_mg_pll_get_freq, }; static const struct dpll_info tgl_plls[] = { @@ -3780,6 +4271,7 @@ static const struct intel_dpll_mgr tgl_pll_mgr = { .get_dplls = icl_get_dplls, .put_dplls = icl_put_dplls, .update_active_dpll = icl_update_active_dpll, + .update_ref_clks = icl_update_dpll_ref_clks, .dump_hw_state = icl_dump_hw_state, }; @@ -3814,7 +4306,7 @@ void intel_shared_dpll_init(struct drm_device *dev) dpll_mgr = &pch_pll_mgr; if (!dpll_mgr) { - dev_priv->num_shared_dpll = 0; + dev_priv->dpll.num_shared_dpll = 0; return; } @@ -3822,14 +4314,14 @@ void intel_shared_dpll_init(struct drm_device *dev) for (i = 0; dpll_info[i].name; i++) { drm_WARN_ON(dev, i != dpll_info[i].id); - dev_priv->shared_dplls[i].info = &dpll_info[i]; + dev_priv->dpll.shared_dplls[i].info = &dpll_info[i]; } - dev_priv->dpll_mgr = dpll_mgr; - dev_priv->num_shared_dpll = i; - mutex_init(&dev_priv->dpll_lock); + dev_priv->dpll.mgr = dpll_mgr; + dev_priv->dpll.num_shared_dpll = i; + mutex_init(&dev_priv->dpll.lock); - BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS); + BUG_ON(dev_priv->dpll.num_shared_dpll > I915_NUM_PLLS); } /** @@ -3856,7 +4348,7 @@ bool intel_reserve_shared_dplls(struct intel_atomic_state *state, struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll_mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr)) return false; @@ -3879,7 +4371,7 @@ void intel_release_shared_dplls(struct intel_atomic_state *state, struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(state->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll_mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; /* * FIXME: this function is called for every platform having a @@ -3908,7 +4400,7 @@ void intel_update_active_dpll(struct intel_atomic_state *state, struct intel_encoder *encoder) { struct drm_i915_private *dev_priv = to_i915(encoder->base.dev); - const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll_mgr; + const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr; if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr)) return; @@ -3917,6 +4409,84 @@ void intel_update_active_dpll(struct intel_atomic_state *state, } /** + * intel_dpll_get_freq - calculate the DPLL's output frequency + * @i915: i915 device + * @pll: DPLL for which to calculate the output frequency + * + * Return the output frequency corresponding to @pll's current state. + */ +int intel_dpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll) +{ + if (drm_WARN_ON(&i915->drm, !pll->info->funcs->get_freq)) + return 0; + + return pll->info->funcs->get_freq(i915, pll); +} + +static void readout_dpll_hw_state(struct drm_i915_private *i915, + struct intel_shared_dpll *pll) +{ + struct intel_crtc *crtc; + + pll->on = pll->info->funcs->get_hw_state(i915, pll, + &pll->state.hw_state); + + if (IS_ELKHARTLAKE(i915) && pll->on && + pll->info->id == DPLL_ID_EHL_DPLL4) { + pll->wakeref = intel_display_power_get(i915, + POWER_DOMAIN_DPLL_DC_OFF); + } + + pll->state.crtc_mask = 0; + for_each_intel_crtc(&i915->drm, crtc) { + struct intel_crtc_state *crtc_state = + to_intel_crtc_state(crtc->base.state); + + if (crtc_state->hw.active && crtc_state->shared_dpll == pll) + pll->state.crtc_mask |= 1 << crtc->pipe; + } + pll->active_mask = pll->state.crtc_mask; + + drm_dbg_kms(&i915->drm, + "%s hw state readout: crtc_mask 0x%08x, on %i\n", + pll->info->name, pll->state.crtc_mask, pll->on); +} + +void intel_dpll_readout_hw_state(struct drm_i915_private *i915) +{ + int i; + + if (i915->dpll.mgr && i915->dpll.mgr->update_ref_clks) + i915->dpll.mgr->update_ref_clks(i915); + + for (i = 0; i < i915->dpll.num_shared_dpll; i++) + readout_dpll_hw_state(i915, &i915->dpll.shared_dplls[i]); +} + +static void sanitize_dpll_state(struct drm_i915_private *i915, + struct intel_shared_dpll *pll) +{ + if (!pll->on || pll->active_mask) + return; + + drm_dbg_kms(&i915->drm, + "%s enabled but not in use, disabling\n", + pll->info->name); + + pll->info->funcs->disable(i915, pll); + pll->on = false; +} + +void intel_dpll_sanitize_state(struct drm_i915_private *i915) +{ + int i; + + for (i = 0; i < i915->dpll.num_shared_dpll; i++) + sanitize_dpll_state(i915, &i915->dpll.shared_dplls[i]); +} + +/** * intel_shared_dpll_dump_hw_state - write hw_state to dmesg * @dev_priv: i915 drm device * @hw_state: hw state to be written to the log @@ -3926,8 +4496,8 @@ void intel_update_active_dpll(struct intel_atomic_state *state, void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state) { - if (dev_priv->dpll_mgr) { - dev_priv->dpll_mgr->dump_hw_state(dev_priv, hw_state); + if (dev_priv->dpll.mgr) { + dev_priv->dpll.mgr->dump_hw_state(dev_priv, hw_state); } else { /* fallback for platforms that don't use the shared dpll * infrastructure diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h index 2a104c64291d..5d9a2bc371e7 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.h +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.h @@ -278,6 +278,15 @@ struct intel_shared_dpll_funcs { bool (*get_hw_state)(struct drm_i915_private *dev_priv, struct intel_shared_dpll *pll, struct intel_dpll_hw_state *hw_state); + + /** + * @get_freq: + * + * Hook for calculating the pll's output frequency based on its + * current state. + */ + int (*get_freq)(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll); }; /** @@ -372,15 +381,18 @@ void icl_set_active_port_dpll(struct intel_crtc_state *crtc_state, void intel_update_active_dpll(struct intel_atomic_state *state, struct intel_crtc *crtc, struct intel_encoder *encoder); +int intel_dpll_get_freq(struct drm_i915_private *i915, + const struct intel_shared_dpll *pll); void intel_prepare_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state); void intel_shared_dpll_swap_state(struct intel_atomic_state *state); void intel_shared_dpll_init(struct drm_device *dev); +void intel_dpll_readout_hw_state(struct drm_i915_private *dev_priv); +void intel_dpll_sanitize_state(struct drm_i915_private *dev_priv); void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv, const struct intel_dpll_hw_state *hw_state); -int cnl_hdmi_pll_ref_clock(struct drm_i915_private *dev_priv); enum intel_dpll_id icl_tc_port_to_pll_id(enum tc_port tc_port); bool intel_dpll_is_combophy(enum intel_dpll_id id); diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c index 76ae01277fd6..29fec6a92d17 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.c +++ b/drivers/gpu/drm/i915/display/intel_dsb.c @@ -34,7 +34,7 @@ #define DSB_BYTE_EN_SHIFT 20 #define DSB_REG_VALUE_MASK 0xfffff -static inline bool is_dsb_busy(struct intel_dsb *dsb) +static bool is_dsb_busy(struct intel_dsb *dsb) { struct intel_crtc *crtc = container_of(dsb, typeof(*crtc), dsb); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -43,7 +43,7 @@ static inline bool is_dsb_busy(struct intel_dsb *dsb) return DSB_STATUS & intel_de_read(dev_priv, DSB_CTRL(pipe, dsb->id)); } -static inline bool intel_dsb_enable_engine(struct intel_dsb *dsb) +static bool intel_dsb_enable_engine(struct intel_dsb *dsb) { struct intel_crtc *crtc = container_of(dsb, typeof(*crtc), dsb); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -52,7 +52,7 @@ static inline bool intel_dsb_enable_engine(struct intel_dsb *dsb) dsb_ctrl = intel_de_read(dev_priv, DSB_CTRL(pipe, dsb->id)); if (DSB_STATUS & dsb_ctrl) { - DRM_DEBUG_KMS("DSB engine is busy.\n"); + drm_dbg_kms(&dev_priv->drm, "DSB engine is busy.\n"); return false; } @@ -63,7 +63,7 @@ static inline bool intel_dsb_enable_engine(struct intel_dsb *dsb) return true; } -static inline bool intel_dsb_disable_engine(struct intel_dsb *dsb) +static bool intel_dsb_disable_engine(struct intel_dsb *dsb) { struct intel_crtc *crtc = container_of(dsb, typeof(*crtc), dsb); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -72,7 +72,7 @@ static inline bool intel_dsb_disable_engine(struct intel_dsb *dsb) dsb_ctrl = intel_de_read(dev_priv, DSB_CTRL(pipe, dsb->id)); if (DSB_STATUS & dsb_ctrl) { - DRM_DEBUG_KMS("DSB engine is busy.\n"); + drm_dbg_kms(&dev_priv->drm, "DSB engine is busy.\n"); return false; } @@ -115,20 +115,20 @@ intel_dsb_get(struct intel_crtc *crtc) obj = i915_gem_object_create_internal(i915, DSB_BUF_SIZE); if (IS_ERR(obj)) { - DRM_ERROR("Gem object creation failed\n"); + drm_err(&i915->drm, "Gem object creation failed\n"); goto out; } vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); if (IS_ERR(vma)) { - DRM_ERROR("Vma creation failed\n"); + drm_err(&i915->drm, "Vma creation failed\n"); i915_gem_object_put(obj); goto out; } buf = i915_gem_object_pin_map(vma->obj, I915_MAP_WC); if (IS_ERR(buf)) { - DRM_ERROR("Command buffer creation failed\n"); + drm_err(&i915->drm, "Command buffer creation failed\n"); goto out; } @@ -203,7 +203,7 @@ void intel_dsb_indexed_reg_write(struct intel_dsb *dsb, i915_reg_t reg, } if (drm_WARN_ON(&dev_priv->drm, dsb->free_pos >= DSB_BUF_SIZE)) { - DRM_DEBUG_KMS("DSB buffer overflow\n"); + drm_dbg_kms(&dev_priv->drm, "DSB buffer overflow\n"); return; } @@ -277,7 +277,7 @@ void intel_dsb_reg_write(struct intel_dsb *dsb, i915_reg_t reg, u32 val) } if (drm_WARN_ON(&dev_priv->drm, dsb->free_pos >= DSB_BUF_SIZE)) { - DRM_DEBUG_KMS("DSB buffer overflow\n"); + drm_dbg_kms(&dev_priv->drm, "DSB buffer overflow\n"); return; } @@ -310,7 +310,8 @@ void intel_dsb_commit(struct intel_dsb *dsb) goto reset; if (is_dsb_busy(dsb)) { - DRM_ERROR("HEAD_PTR write failed - dsb engine is busy.\n"); + drm_err(&dev_priv->drm, + "HEAD_PTR write failed - dsb engine is busy.\n"); goto reset; } intel_de_write(dev_priv, DSB_HEAD(pipe, dsb->id), @@ -322,15 +323,18 @@ void intel_dsb_commit(struct intel_dsb *dsb) (tail - dsb->free_pos * 4)); if (is_dsb_busy(dsb)) { - DRM_ERROR("TAIL_PTR write failed - dsb engine is busy.\n"); + drm_err(&dev_priv->drm, + "TAIL_PTR write failed - dsb engine is busy.\n"); goto reset; } - DRM_DEBUG_KMS("DSB execution started - head 0x%x, tail 0x%x\n", - i915_ggtt_offset(dsb->vma), tail); + drm_dbg_kms(&dev_priv->drm, + "DSB execution started - head 0x%x, tail 0x%x\n", + i915_ggtt_offset(dsb->vma), tail); intel_de_write(dev_priv, DSB_TAIL(pipe, dsb->id), i915_ggtt_offset(dsb->vma) + tail); if (wait_for(!is_dsb_busy(dsb), 1)) { - DRM_ERROR("Timed out waiting for DSB workload completion.\n"); + drm_err(&dev_priv->drm, + "Timed out waiting for DSB workload completion.\n"); goto reset; } diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c index a2a937109a5a..afa4e6817e8c 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi.c +++ b/drivers/gpu/drm/i915/display/intel_dsi.c @@ -31,20 +31,21 @@ int intel_dsi_tlpx_ns(const struct intel_dsi *intel_dsi) int intel_dsi_get_modes(struct drm_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->dev); struct intel_connector *intel_connector = to_intel_connector(connector); struct drm_display_mode *mode; - DRM_DEBUG_KMS("\n"); + drm_dbg_kms(&i915->drm, "\n"); if (!intel_connector->panel.fixed_mode) { - DRM_DEBUG_KMS("no fixed mode\n"); + drm_dbg_kms(&i915->drm, "no fixed mode\n"); return 0; } mode = drm_mode_duplicate(connector->dev, intel_connector->panel.fixed_mode); if (!mode) { - DRM_DEBUG_KMS("drm_mode_duplicate failed\n"); + drm_dbg_kms(&i915->drm, "drm_mode_duplicate failed\n"); return 0; } @@ -60,7 +61,7 @@ enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector, const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; int max_dotclk = to_i915(connector->dev)->max_dotclk_freq; - DRM_DEBUG_KMS("\n"); + drm_dbg_kms(&dev_priv->drm, "\n"); if (mode->flags & DRM_MODE_FLAG_DBLSCAN) return MODE_NO_DBLESCAN; diff --git a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c index 694498f4b719..eed037ec0b29 100644 --- a/drivers/gpu/drm/i915/display/intel_dsi_vbt.c +++ b/drivers/gpu/drm/i915/display/intel_dsi_vbt.c @@ -36,7 +36,6 @@ #include <drm/drm_crtc.h> #include <drm/drm_edid.h> -#include <drm/i915_drm.h> #include <video/mipi_display.h> @@ -122,7 +121,7 @@ struct i2c_adapter_lookup { #define ICL_GPIO_DDPA_CTRLCLK_2 8 #define ICL_GPIO_DDPA_CTRLDATA_2 9 -static inline enum port intel_dsi_seq_port_to_port(u8 port) +static enum port intel_dsi_seq_port_to_port(u8 port) { return port ? PORT_C : PORT_A; } @@ -454,8 +453,7 @@ static inline void i2c_acpi_find_adapter(struct intel_dsi *intel_dsi, static const u8 *mipi_exec_i2c(struct intel_dsi *intel_dsi, const u8 *data) { - struct drm_device *drm_dev = intel_dsi->base.base.dev; - struct device *dev = &drm_dev->pdev->dev; + struct drm_i915_private *i915 = to_i915(intel_dsi->base.base.dev); struct i2c_adapter *adapter; struct i2c_msg msg; int ret; @@ -472,7 +470,7 @@ static const u8 *mipi_exec_i2c(struct intel_dsi *intel_dsi, const u8 *data) adapter = i2c_get_adapter(intel_dsi->i2c_bus_num); if (!adapter) { - DRM_DEV_ERROR(dev, "Cannot find a valid i2c bus for xfer\n"); + drm_err(&i915->drm, "Cannot find a valid i2c bus for xfer\n"); goto err_bus; } @@ -490,9 +488,9 @@ static const u8 *mipi_exec_i2c(struct intel_dsi *intel_dsi, const u8 *data) ret = i2c_transfer(adapter, &msg, 1); if (ret < 0) - DRM_DEV_ERROR(dev, - "Failed to xfer payload of size (%u) to reg (%u)\n", - payload_size, reg_offset); + drm_err(&i915->drm, + "Failed to xfer payload of size (%u) to reg (%u)\n", + payload_size, reg_offset); kfree(payload_data); err_alloc: diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c index 77f3d083b7a1..5cd09034519b 100644 --- a/drivers/gpu/drm/i915/display/intel_dvo.c +++ b/drivers/gpu/drm/i915/display/intel_dvo.c @@ -30,7 +30,6 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_connector.h" @@ -184,7 +183,8 @@ static void intel_dvo_get_config(struct intel_encoder *encoder, pipe_config->hw.adjusted_mode.crtc_clock = pipe_config->port_clock; } -static void intel_disable_dvo(struct intel_encoder *encoder, +static void intel_disable_dvo(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -198,7 +198,8 @@ static void intel_disable_dvo(struct intel_encoder *encoder, intel_de_read(dev_priv, dvo_reg); } -static void intel_enable_dvo(struct intel_encoder *encoder, +static void intel_enable_dvo(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -273,7 +274,8 @@ static int intel_dvo_compute_config(struct intel_encoder *encoder, return 0; } -static void intel_dvo_pre_enable(struct intel_encoder *encoder, +static void intel_dvo_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index ddf8d3bb7a7d..1c26673acb2d 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -42,6 +42,7 @@ #include "i915_drv.h" #include "i915_trace.h" +#include "i915_vgpu.h" #include "intel_display_types.h" #include "intel_fbc.h" #include "intel_frontbuffer.h" @@ -103,7 +104,7 @@ static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv) /* Wait for compressing bit to clear */ if (intel_de_wait_for_clear(dev_priv, FBC_STATUS, FBC_STAT_COMPRESSING, 10)) { - DRM_DEBUG_KMS("FBC idle timed out\n"); + drm_dbg_kms(&dev_priv->drm, "FBC idle timed out\n"); return; } } @@ -320,7 +321,7 @@ static void gen7_fbc_activate(struct drm_i915_private *dev_priv) SNB_CPU_FENCE_ENABLE | params->fence_id); intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, params->crtc.fence_y_offset); - } else { + } else if (dev_priv->ggtt.num_fences) { intel_de_write(dev_priv, SNB_DPFC_CTL_SA, 0); intel_de_write(dev_priv, DPFC_CPU_FENCE_OFFSET, 0); } @@ -484,8 +485,8 @@ static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, if (!ret) goto err_llb; else if (ret > 1) { - DRM_INFO("Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n"); - + drm_info_once(&dev_priv->drm, + "Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n"); } fbc->threshold = ret; @@ -508,20 +509,21 @@ static int intel_fbc_alloc_cfb(struct drm_i915_private *dev_priv, fbc->compressed_llb = compressed_llb; - GEM_BUG_ON(range_overflows_t(u64, dev_priv->dsm.start, - fbc->compressed_fb.start, - U32_MAX)); - GEM_BUG_ON(range_overflows_t(u64, dev_priv->dsm.start, - fbc->compressed_llb->start, - U32_MAX)); + GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start, + fbc->compressed_fb.start, + U32_MAX)); + GEM_BUG_ON(range_overflows_end_t(u64, dev_priv->dsm.start, + fbc->compressed_llb->start, + U32_MAX)); intel_de_write(dev_priv, FBC_CFB_BASE, dev_priv->dsm.start + fbc->compressed_fb.start); intel_de_write(dev_priv, FBC_LL_BASE, dev_priv->dsm.start + compressed_llb->start); } - DRM_DEBUG_KMS("reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n", - fbc->compressed_fb.size, fbc->threshold); + drm_dbg_kms(&dev_priv->drm, + "reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n", + fbc->compressed_fb.size, fbc->threshold); return 0; @@ -530,7 +532,7 @@ err_fb: i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb); err_llb: if (drm_mm_initialized(&dev_priv->mm.stolen)) - pr_info_once("drm: not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size); + drm_info_once(&dev_priv->drm, "not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size); return -ENOSPC; } @@ -538,6 +540,9 @@ static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv) { struct intel_fbc *fbc = &dev_priv->fbc; + if (WARN_ON(intel_fbc_hw_is_active(dev_priv))) + return; + if (!drm_mm_node_allocated(&fbc->compressed_fb)) return; @@ -562,7 +567,7 @@ void intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv) } static bool stride_is_valid(struct drm_i915_private *dev_priv, - unsigned int stride) + u64 modifier, unsigned int stride) { /* This should have been caught earlier. */ if (drm_WARN_ON_ONCE(&dev_priv->drm, (stride & (64 - 1)) != 0)) @@ -578,6 +583,11 @@ static bool stride_is_valid(struct drm_i915_private *dev_priv, if (IS_GEN(dev_priv, 4) && !IS_G4X(dev_priv) && stride < 2048) return false; + /* Display WA #1105: skl,bxt,kbl,cfl,glk */ + if (IS_GEN(dev_priv, 9) && + modifier == DRM_FORMAT_MOD_LINEAR && stride & 511) + return false; + if (stride > 16384) return false; @@ -605,6 +615,19 @@ static bool pixel_format_is_valid(struct drm_i915_private *dev_priv, } } +static bool rotation_is_valid(struct drm_i915_private *dev_priv, + u32 pixel_format, unsigned int rotation) +{ + if (INTEL_GEN(dev_priv) >= 9 && pixel_format == DRM_FORMAT_RGB565 && + drm_rotation_90_or_270(rotation)) + return false; + else if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) && + rotation != DRM_MODE_ROTATE_0) + return false; + + return true; +} + /* * For some reason, the hardware tracking starts looking at whatever we * programmed as the display plane base address register. It does not look at @@ -639,6 +662,22 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc) return effective_w <= max_w && effective_h <= max_h; } +static bool tiling_is_valid(struct drm_i915_private *dev_priv, + uint64_t modifier) +{ + switch (modifier) { + case DRM_FORMAT_MOD_LINEAR: + if (INTEL_GEN(dev_priv) >= 9) + return true; + return false; + case I915_FORMAT_MOD_X_TILED: + case I915_FORMAT_MOD_Y_TILED: + return true; + default: + return false; + } +} + static void intel_fbc_update_state_cache(struct intel_crtc *crtc, const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) @@ -672,6 +711,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc, cache->fb.format = fb->format; cache->fb.stride = fb->pitches[0]; + cache->fb.modifier = fb->modifier; drm_WARN_ON(&dev_priv->drm, plane_state->flags & PLANE_HAS_FENCE && !plane_state->vma->fence); @@ -691,12 +731,37 @@ static bool intel_fbc_cfb_size_changed(struct drm_i915_private *dev_priv) fbc->compressed_fb.size * fbc->threshold; } +static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv) +{ + struct intel_fbc *fbc = &dev_priv->fbc; + + if (intel_vgpu_active(dev_priv)) { + fbc->no_fbc_reason = "VGPU is active"; + return false; + } + + if (!i915_modparams.enable_fbc) { + fbc->no_fbc_reason = "disabled per module param or by default"; + return false; + } + + if (fbc->underrun_detected) { + fbc->no_fbc_reason = "underrun detected"; + return false; + } + + return true; +} + static bool intel_fbc_can_activate(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct intel_fbc *fbc = &dev_priv->fbc; struct intel_fbc_state_cache *cache = &fbc->state_cache; + if (!intel_fbc_can_enable(dev_priv)) + return false; + if (!cache->plane.visible) { fbc->no_fbc_reason = "primary plane not visible"; return false; @@ -720,30 +785,40 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) return false; } - /* The use of a CPU fence is mandatory in order to detect writes - * by the CPU to the scanout and trigger updates to the FBC. + /* The use of a CPU fence is one of two ways to detect writes by the + * CPU to the scanout and trigger updates to the FBC. + * + * The other method is by software tracking (see + * intel_fbc_invalidate/flush()), it will manually notify FBC and nuke + * the current compressed buffer and recompress it. * * Note that is possible for a tiled surface to be unmappable (and - * so have no fence associated with it) due to aperture constaints + * so have no fence associated with it) due to aperture constraints * at the time of pinning. * * FIXME with 90/270 degree rotation we should use the fence on * the normal GTT view (the rotated view doesn't even have a * fence). Would need changes to the FBC fence Y offset as well. - * For now this will effecively disable FBC with 90/270 degree + * For now this will effectively disable FBC with 90/270 degree * rotation. */ - if (cache->fence_id < 0) { + if (INTEL_GEN(dev_priv) < 9 && cache->fence_id < 0) { fbc->no_fbc_reason = "framebuffer not tiled or fenced"; return false; } - if (INTEL_GEN(dev_priv) <= 4 && !IS_G4X(dev_priv) && - cache->plane.rotation != DRM_MODE_ROTATE_0) { + + if (!rotation_is_valid(dev_priv, cache->fb.format->format, + cache->plane.rotation)) { fbc->no_fbc_reason = "rotation unsupported"; return false; } - if (!stride_is_valid(dev_priv, cache->fb.stride)) { + if (!tiling_is_valid(dev_priv, cache->fb.modifier)) { + fbc->no_fbc_reason = "tiling unsupported"; + return false; + } + + if (!stride_is_valid(dev_priv, cache->fb.modifier, cache->fb.stride)) { fbc->no_fbc_reason = "framebuffer stride not supported"; return false; } @@ -795,28 +870,6 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc) return true; } -static bool intel_fbc_can_enable(struct drm_i915_private *dev_priv) -{ - struct intel_fbc *fbc = &dev_priv->fbc; - - if (intel_vgpu_active(dev_priv)) { - fbc->no_fbc_reason = "VGPU is active"; - return false; - } - - if (!i915_modparams.enable_fbc) { - fbc->no_fbc_reason = "disabled per module param or by default"; - return false; - } - - if (fbc->underrun_detected) { - fbc->no_fbc_reason = "underrun detected"; - return false; - } - - return true; -} - static void intel_fbc_get_reg_params(struct intel_crtc *crtc, struct intel_fbc_reg_params *params) { @@ -944,7 +997,8 @@ static void __intel_fbc_disable(struct drm_i915_private *dev_priv) drm_WARN_ON(&dev_priv->drm, !fbc->crtc); drm_WARN_ON(&dev_priv->drm, fbc->active); - DRM_DEBUG_KMS("Disabling FBC on pipe %c\n", pipe_name(crtc->pipe)); + drm_dbg_kms(&dev_priv->drm, "Disabling FBC on pipe %c\n", + pipe_name(crtc->pipe)); __intel_fbc_cleanup_cfb(dev_priv); @@ -1172,7 +1226,8 @@ void intel_fbc_enable(struct intel_atomic_state *state, else cache->gen9_wa_cfb_stride = 0; - DRM_DEBUG_KMS("Enabling FBC on pipe %c\n", pipe_name(crtc->pipe)); + drm_dbg_kms(&dev_priv->drm, "Enabling FBC on pipe %c\n", + pipe_name(crtc->pipe)); fbc->no_fbc_reason = "FBC enabled but not active yet\n"; fbc->crtc = crtc; @@ -1234,7 +1289,7 @@ static void intel_fbc_underrun_work_fn(struct work_struct *work) if (fbc->underrun_detected || !fbc->crtc) goto out; - DRM_DEBUG_KMS("Disabling FBC due to FIFO underrun.\n"); + drm_dbg_kms(&dev_priv->drm, "Disabling FBC due to FIFO underrun.\n"); fbc->underrun_detected = true; intel_fbc_deactivate(dev_priv, "FIFO underrun"); @@ -1260,7 +1315,8 @@ int intel_fbc_reset_underrun(struct drm_i915_private *dev_priv) return ret; if (dev_priv->fbc.underrun_detected) { - DRM_DEBUG_KMS("Re-allowing FBC after fifo underrun\n"); + drm_dbg_kms(&dev_priv->drm, + "Re-allowing FBC after fifo underrun\n"); dev_priv->fbc.no_fbc_reason = "FIFO underrun cleared"; } @@ -1331,7 +1387,8 @@ static bool need_fbc_vtd_wa(struct drm_i915_private *dev_priv) /* WaFbcTurnOffFbcWhenHyperVisorIsUsed:skl,bxt */ if (intel_vtd_active() && (IS_SKYLAKE(dev_priv) || IS_BROXTON(dev_priv))) { - DRM_INFO("Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n"); + drm_info(&dev_priv->drm, + "Disabling framebuffer compression (FBC) to prevent screen flicker with VT-d enabled\n"); return true; } @@ -1359,8 +1416,8 @@ void intel_fbc_init(struct drm_i915_private *dev_priv) mkwrite_device_info(dev_priv)->display.has_fbc = false; i915_modparams.enable_fbc = intel_sanitize_fbc_option(dev_priv); - DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n", - i915_modparams.enable_fbc); + drm_dbg_kms(&dev_priv->drm, "Sanitized enable_fbc value: %d\n", + i915_modparams.enable_fbc); if (!HAS_FBC(dev_priv)) { fbc->no_fbc_reason = "unsupported by this chipset"; diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index b4ff77225236..bd39eb6a21b8 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -40,7 +40,6 @@ #include <drm/drm_crtc.h> #include <drm/drm_fb_helper.h> #include <drm/drm_fourcc.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_display_types.h" @@ -147,7 +146,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper, if (IS_ERR(obj)) obj = i915_gem_object_create_shmem(dev_priv, size); if (IS_ERR(obj)) { - DRM_ERROR("failed to allocate framebuffer\n"); + drm_err(&dev_priv->drm, "failed to allocate framebuffer\n"); return PTR_ERR(obj); } @@ -184,21 +183,23 @@ static int intelfb_create(struct drm_fb_helper *helper, if (intel_fb && (sizes->fb_width > intel_fb->base.width || sizes->fb_height > intel_fb->base.height)) { - DRM_DEBUG_KMS("BIOS fb too small (%dx%d), we require (%dx%d)," - " releasing it\n", - intel_fb->base.width, intel_fb->base.height, - sizes->fb_width, sizes->fb_height); + drm_dbg_kms(&dev_priv->drm, + "BIOS fb too small (%dx%d), we require (%dx%d)," + " releasing it\n", + intel_fb->base.width, intel_fb->base.height, + sizes->fb_width, sizes->fb_height); drm_framebuffer_put(&intel_fb->base); intel_fb = ifbdev->fb = NULL; } if (!intel_fb || drm_WARN_ON(dev, !intel_fb_obj(&intel_fb->base))) { - DRM_DEBUG_KMS("no BIOS fb, allocating a new one\n"); + drm_dbg_kms(&dev_priv->drm, + "no BIOS fb, allocating a new one\n"); ret = intelfb_alloc(helper, sizes); if (ret) return ret; intel_fb = ifbdev->fb; } else { - DRM_DEBUG_KMS("re-using BIOS fb\n"); + drm_dbg_kms(&dev_priv->drm, "re-using BIOS fb\n"); prealloc = true; sizes->fb_width = intel_fb->base.width; sizes->fb_height = intel_fb->base.height; @@ -221,7 +222,7 @@ static int intelfb_create(struct drm_fb_helper *helper, info = drm_fb_helper_alloc_fbi(helper); if (IS_ERR(info)) { - DRM_ERROR("Failed to allocate fb_info\n"); + drm_err(&dev_priv->drm, "Failed to allocate fb_info\n"); ret = PTR_ERR(info); goto out_unpin; } @@ -241,7 +242,8 @@ static int intelfb_create(struct drm_fb_helper *helper, vaddr = i915_vma_pin_iomap(vma); if (IS_ERR(vaddr)) { - DRM_ERROR("Failed to remap framebuffer into virtual memory\n"); + drm_err(&dev_priv->drm, + "Failed to remap framebuffer into virtual memory\n"); ret = PTR_ERR(vaddr); goto out_unpin; } @@ -259,9 +261,9 @@ static int intelfb_create(struct drm_fb_helper *helper, /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ - DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x\n", - ifbdev->fb->base.width, ifbdev->fb->base.height, - i915_ggtt_offset(vma)); + drm_dbg_kms(&dev_priv->drm, "allocated %dx%d fb: 0x%08x\n", + ifbdev->fb->base.width, ifbdev->fb->base.height, + i915_ggtt_offset(vma)); ifbdev->vma = vma; ifbdev->vma_flags = flags; @@ -310,6 +312,7 @@ static void intel_fbdev_destroy(struct intel_fbdev *ifbdev) static bool intel_fbdev_init_bios(struct drm_device *dev, struct intel_fbdev *ifbdev) { + struct drm_i915_private *i915 = to_i915(dev); struct intel_framebuffer *fb = NULL; struct drm_crtc *crtc; struct intel_crtc *intel_crtc; @@ -322,21 +325,24 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, intel_crtc = to_intel_crtc(crtc); if (!crtc->state->active || !obj) { - DRM_DEBUG_KMS("pipe %c not active or no fb, skipping\n", - pipe_name(intel_crtc->pipe)); + drm_dbg_kms(&i915->drm, + "pipe %c not active or no fb, skipping\n", + pipe_name(intel_crtc->pipe)); continue; } if (obj->base.size > max_size) { - DRM_DEBUG_KMS("found possible fb from plane %c\n", - pipe_name(intel_crtc->pipe)); + drm_dbg_kms(&i915->drm, + "found possible fb from plane %c\n", + pipe_name(intel_crtc->pipe)); fb = to_intel_framebuffer(crtc->primary->state->fb); max_size = obj->base.size; } } if (!fb) { - DRM_DEBUG_KMS("no active fbs found, not using BIOS config\n"); + drm_dbg_kms(&i915->drm, + "no active fbs found, not using BIOS config\n"); goto out; } @@ -347,13 +353,14 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, intel_crtc = to_intel_crtc(crtc); if (!crtc->state->active) { - DRM_DEBUG_KMS("pipe %c not active, skipping\n", - pipe_name(intel_crtc->pipe)); + drm_dbg_kms(&i915->drm, + "pipe %c not active, skipping\n", + pipe_name(intel_crtc->pipe)); continue; } - DRM_DEBUG_KMS("checking plane %c for BIOS fb\n", - pipe_name(intel_crtc->pipe)); + drm_dbg_kms(&i915->drm, "checking plane %c for BIOS fb\n", + pipe_name(intel_crtc->pipe)); /* * See if the plane fb we found above will fit on this @@ -363,9 +370,10 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, cur_size = crtc->state->adjusted_mode.crtc_hdisplay; cur_size = cur_size * fb->base.format->cpp[0]; if (fb->base.pitches[0] < cur_size) { - DRM_DEBUG_KMS("fb not wide enough for plane %c (%d vs %d)\n", - pipe_name(intel_crtc->pipe), - cur_size, fb->base.pitches[0]); + drm_dbg_kms(&i915->drm, + "fb not wide enough for plane %c (%d vs %d)\n", + pipe_name(intel_crtc->pipe), + cur_size, fb->base.pitches[0]); fb = NULL; break; } @@ -373,28 +381,32 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, cur_size = crtc->state->adjusted_mode.crtc_vdisplay; cur_size = intel_fb_align_height(&fb->base, 0, cur_size); cur_size *= fb->base.pitches[0]; - DRM_DEBUG_KMS("pipe %c area: %dx%d, bpp: %d, size: %d\n", - pipe_name(intel_crtc->pipe), - crtc->state->adjusted_mode.crtc_hdisplay, - crtc->state->adjusted_mode.crtc_vdisplay, - fb->base.format->cpp[0] * 8, - cur_size); + drm_dbg_kms(&i915->drm, + "pipe %c area: %dx%d, bpp: %d, size: %d\n", + pipe_name(intel_crtc->pipe), + crtc->state->adjusted_mode.crtc_hdisplay, + crtc->state->adjusted_mode.crtc_vdisplay, + fb->base.format->cpp[0] * 8, + cur_size); if (cur_size > max_size) { - DRM_DEBUG_KMS("fb not big enough for plane %c (%d vs %d)\n", - pipe_name(intel_crtc->pipe), - cur_size, max_size); + drm_dbg_kms(&i915->drm, + "fb not big enough for plane %c (%d vs %d)\n", + pipe_name(intel_crtc->pipe), + cur_size, max_size); fb = NULL; break; } - DRM_DEBUG_KMS("fb big enough for plane %c (%d >= %d)\n", - pipe_name(intel_crtc->pipe), - max_size, cur_size); + drm_dbg_kms(&i915->drm, + "fb big enough for plane %c (%d >= %d)\n", + pipe_name(intel_crtc->pipe), + max_size, cur_size); } if (!fb) { - DRM_DEBUG_KMS("BIOS fb not suitable for all pipes, not using\n"); + drm_dbg_kms(&i915->drm, + "BIOS fb not suitable for all pipes, not using\n"); goto out; } @@ -416,7 +428,7 @@ static bool intel_fbdev_init_bios(struct drm_device *dev, } - DRM_DEBUG_KMS("using BIOS fb for initial console\n"); + drm_dbg_kms(&i915->drm, "using BIOS fb for initial console\n"); return true; out: @@ -453,7 +465,7 @@ int intel_fbdev_init(struct drm_device *dev) if (!intel_fbdev_init_bios(dev, ifbdev)) ifbdev->preferred_bpp = 32; - ret = drm_fb_helper_init(dev, &ifbdev->helper, 4); + ret = drm_fb_helper_init(dev, &ifbdev->helper); if (ret) { kfree(ifbdev); return ret; @@ -462,8 +474,6 @@ int intel_fbdev_init(struct drm_device *dev) dev_priv->fbdev = ifbdev; INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker); - drm_fb_helper_single_add_all_connectors(&ifbdev->helper); - return 0; } @@ -525,8 +535,9 @@ void intel_fbdev_fini(struct drm_i915_private *dev_priv) * processing, fbdev will perform a full connector reprobe if a hotplug event * was received while HPD was suspended. */ -static void intel_fbdev_hpd_set_suspend(struct intel_fbdev *ifbdev, int state) +static void intel_fbdev_hpd_set_suspend(struct drm_i915_private *i915, int state) { + struct intel_fbdev *ifbdev = i915->fbdev; bool send_hpd = false; mutex_lock(&ifbdev->hpd_lock); @@ -536,7 +547,7 @@ static void intel_fbdev_hpd_set_suspend(struct intel_fbdev *ifbdev, int state) mutex_unlock(&ifbdev->hpd_lock); if (send_hpd) { - DRM_DEBUG_KMS("Handling delayed fbcon HPD event\n"); + drm_dbg_kms(&i915->drm, "Handling delayed fbcon HPD event\n"); drm_fb_helper_hotplug_event(&ifbdev->helper); } } @@ -591,7 +602,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous drm_fb_helper_set_suspend(&ifbdev->helper, state); console_unlock(); - intel_fbdev_hpd_set_suspend(ifbdev, state); + intel_fbdev_hpd_set_suspend(dev_priv, state); } void intel_fbdev_output_poll_changed(struct drm_device *dev) diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c index 470b3b0b9bdb..813a4f7033e1 100644 --- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c +++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c @@ -103,7 +103,7 @@ static void i9xx_check_fifo_underruns(struct intel_crtc *crtc) intel_de_posting_read(dev_priv, reg); trace_intel_cpu_fifo_underrun(dev_priv, crtc->pipe); - DRM_ERROR("pipe %c underrun\n", pipe_name(crtc->pipe)); + drm_err(&dev_priv->drm, "pipe %c underrun\n", pipe_name(crtc->pipe)); } static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, @@ -123,7 +123,8 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev, intel_de_posting_read(dev_priv, reg); } else { if (old && intel_de_read(dev_priv, reg) & PIPE_FIFO_UNDERRUN_STATUS) - DRM_ERROR("pipe %c underrun\n", pipe_name(pipe)); + drm_err(&dev_priv->drm, "pipe %c underrun\n", + pipe_name(pipe)); } } @@ -155,7 +156,7 @@ static void ivb_check_fifo_underruns(struct intel_crtc *crtc) intel_de_posting_read(dev_priv, GEN7_ERR_INT); trace_intel_cpu_fifo_underrun(dev_priv, pipe); - DRM_ERROR("fifo underrun on pipe %c\n", pipe_name(pipe)); + drm_err(&dev_priv->drm, "fifo underrun on pipe %c\n", pipe_name(pipe)); } static void ivb_set_fifo_underrun_reporting(struct drm_device *dev, @@ -176,8 +177,9 @@ static void ivb_set_fifo_underrun_reporting(struct drm_device *dev, if (old && intel_de_read(dev_priv, GEN7_ERR_INT) & ERR_INT_FIFO_UNDERRUN(pipe)) { - DRM_ERROR("uncleared fifo underrun on pipe %c\n", - pipe_name(pipe)); + drm_err(&dev_priv->drm, + "uncleared fifo underrun on pipe %c\n", + pipe_name(pipe)); } } } @@ -223,8 +225,8 @@ static void cpt_check_pch_fifo_underruns(struct intel_crtc *crtc) intel_de_posting_read(dev_priv, SERR_INT); trace_intel_pch_fifo_underrun(dev_priv, pch_transcoder); - DRM_ERROR("pch fifo underrun on pch transcoder %c\n", - pipe_name(pch_transcoder)); + drm_err(&dev_priv->drm, "pch fifo underrun on pch transcoder %c\n", + pipe_name(pch_transcoder)); } static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, @@ -246,8 +248,9 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev, if (old && intel_de_read(dev_priv, SERR_INT) & SERR_INT_TRANS_FIFO_UNDERRUN(pch_transcoder)) { - DRM_ERROR("uncleared pch fifo underrun on pch transcoder %c\n", - pipe_name(pch_transcoder)); + drm_err(&dev_priv->drm, + "uncleared pch fifo underrun on pch transcoder %c\n", + pipe_name(pch_transcoder)); } } } @@ -381,8 +384,8 @@ void intel_cpu_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, if (intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false)) { trace_intel_cpu_fifo_underrun(dev_priv, pipe); - DRM_ERROR("CPU pipe %c FIFO underrun\n", - pipe_name(pipe)); + drm_err(&dev_priv->drm, "CPU pipe %c FIFO underrun\n", + pipe_name(pipe)); } intel_fbc_handle_fifo_underrun_irq(dev_priv); @@ -403,8 +406,8 @@ void intel_pch_fifo_underrun_irq_handler(struct drm_i915_private *dev_priv, if (intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, false)) { trace_intel_pch_fifo_underrun(dev_priv, pch_transcoder); - DRM_ERROR("PCH transcoder %c FIFO underrun\n", - pipe_name(pch_transcoder)); + drm_err(&dev_priv->drm, "PCH transcoder %c FIFO underrun\n", + pipe_name(pch_transcoder)); } } diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c index 6cb02c912acc..2979ed2588eb 100644 --- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c +++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c @@ -302,12 +302,14 @@ void intel_frontbuffer_track(struct intel_frontbuffer *old, BITS_PER_TYPE(atomic_t)); if (old) { - WARN_ON(!(atomic_read(&old->bits) & frontbuffer_bits)); + drm_WARN_ON(old->obj->base.dev, + !(atomic_read(&old->bits) & frontbuffer_bits)); atomic_andnot(frontbuffer_bits, &old->bits); } if (new) { - WARN_ON(atomic_read(&new->bits) & frontbuffer_bits); + drm_WARN_ON(new->obj->base.dev, + atomic_read(&new->bits) & frontbuffer_bits); atomic_or(frontbuffer_bits, &new->bits); } } diff --git a/drivers/gpu/drm/i915/display/intel_global_state.c b/drivers/gpu/drm/i915/display/intel_global_state.c index a0cc894c3868..212d4ee68205 100644 --- a/drivers/gpu/drm/i915/display/intel_global_state.c +++ b/drivers/gpu/drm/i915/display/intel_global_state.c @@ -64,13 +64,14 @@ static void assert_global_state_read_locked(struct intel_atomic_state *state) return; } - WARN(1, "Global state not read locked\n"); + drm_WARN(&dev_priv->drm, 1, "Global state not read locked\n"); } struct intel_global_state * intel_atomic_get_global_obj_state(struct intel_atomic_state *state, struct intel_global_obj *obj) { + struct drm_i915_private *i915 = to_i915(state->base.dev); int index, num_objs, i; size_t size; struct __intel_global_objs_state *arr; @@ -106,8 +107,8 @@ intel_atomic_get_global_obj_state(struct intel_atomic_state *state, state->num_global_objs = num_objs; - DRM_DEBUG_ATOMIC("Added new global object %p state %p to %p\n", - obj, obj_state, state); + drm_dbg_atomic(&i915->drm, "Added new global object %p state %p to %p\n", + obj, obj_state, state); return obj_state; } @@ -147,7 +148,7 @@ void intel_atomic_swap_global_state(struct intel_atomic_state *state) for_each_oldnew_global_obj_in_state(state, obj, old_obj_state, new_obj_state, i) { - WARN_ON(obj->state != old_obj_state); + drm_WARN_ON(&dev_priv->drm, obj->state != old_obj_state); /* * If the new state wasn't modified (and properly diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index 4ef8a81ae0ad..a8d119b6b45c 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -32,7 +32,6 @@ #include <linux/i2c.h> #include <drm/drm_hdcp.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_display_types.h" @@ -380,8 +379,7 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv) return ret; } -static inline -unsigned int gmbus_max_xfer_size(struct drm_i915_private *dev_priv) +static unsigned int gmbus_max_xfer_size(struct drm_i915_private *dev_priv) { return INTEL_GEN(dev_priv) >= 9 ? GEN9_GMBUS_BYTE_COUNT_MAX : GMBUS_BYTE_COUNT_MAX; @@ -632,8 +630,9 @@ retry: * till then let it sleep. */ if (gmbus_wait_idle(dev_priv)) { - DRM_DEBUG_KMS("GMBUS [%s] timed out waiting for idle\n", - adapter->name); + drm_dbg_kms(&dev_priv->drm, + "GMBUS [%s] timed out waiting for idle\n", + adapter->name); ret = -ETIMEDOUT; } intel_de_write_fw(dev_priv, GMBUS0, 0); @@ -656,8 +655,9 @@ clear_err: */ ret = -ENXIO; if (gmbus_wait_idle(dev_priv)) { - DRM_DEBUG_KMS("GMBUS [%s] timed out after NAK\n", - adapter->name); + drm_dbg_kms(&dev_priv->drm, + "GMBUS [%s] timed out after NAK\n", + adapter->name); ret = -ETIMEDOUT; } @@ -669,9 +669,9 @@ clear_err: intel_de_write_fw(dev_priv, GMBUS1, 0); intel_de_write_fw(dev_priv, GMBUS0, 0); - DRM_DEBUG_KMS("GMBUS [%s] NAK for addr: %04x %c(%d)\n", - adapter->name, msgs[i].addr, - (msgs[i].flags & I2C_M_RD) ? 'r' : 'w', msgs[i].len); + drm_dbg_kms(&dev_priv->drm, "GMBUS [%s] NAK for addr: %04x %c(%d)\n", + adapter->name, msgs[i].addr, + (msgs[i].flags & I2C_M_RD) ? 'r' : 'w', msgs[i].len); /* * Passive adapters sometimes NAK the first probe. Retry the first @@ -680,16 +680,18 @@ clear_err: * drm_do_probe_ddc_edid, which bails out on the first -ENXIO. */ if (ret == -ENXIO && i == 0 && try++ == 0) { - DRM_DEBUG_KMS("GMBUS [%s] NAK on first message, retry\n", - adapter->name); + drm_dbg_kms(&dev_priv->drm, + "GMBUS [%s] NAK on first message, retry\n", + adapter->name); goto retry; } goto out; timeout: - DRM_DEBUG_KMS("GMBUS [%s] timed out, falling back to bit banging on pin %d\n", - bus->adapter.name, bus->reg0 & 0xff); + drm_dbg_kms(&dev_priv->drm, + "GMBUS [%s] timed out, falling back to bit banging on pin %d\n", + bus->adapter.name, bus->reg0 & 0xff); intel_de_write_fw(dev_priv, GMBUS0, 0); /* @@ -926,9 +928,10 @@ void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit) mutex_lock(&dev_priv->gmbus_mutex); bus->force_bit += force_bit ? 1 : -1; - DRM_DEBUG_KMS("%sabling bit-banging on %s. force bit now %d\n", - force_bit ? "en" : "dis", adapter->name, - bus->force_bit); + drm_dbg_kms(&dev_priv->drm, + "%sabling bit-banging on %s. force bit now %d\n", + force_bit ? "en" : "dis", adapter->name, + bus->force_bit); mutex_unlock(&dev_priv->gmbus_mutex); } diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c index 30e0a3aa9d57..2cbc4619b4ce 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_hdcp.c @@ -43,6 +43,7 @@ static int intel_hdcp_read_valid_bksv(struct intel_digital_port *intel_dig_port, const struct intel_hdcp_shim *shim, u8 *bksv) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); int ret, i, tries = 2; /* HDCP spec states that we must retry the bksv if it is invalid */ @@ -54,7 +55,7 @@ int intel_hdcp_read_valid_bksv(struct intel_digital_port *intel_dig_port, break; } if (i == tries) { - DRM_DEBUG_KMS("Bksv is invalid\n"); + drm_dbg_kms(&i915->drm, "Bksv is invalid\n"); return -ENODEV; } @@ -108,18 +109,16 @@ bool intel_hdcp2_capable(struct intel_connector *connector) return capable; } -static inline -bool intel_hdcp_in_use(struct drm_i915_private *dev_priv, - enum transcoder cpu_transcoder, enum port port) +static bool intel_hdcp_in_use(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder, enum port port) { return intel_de_read(dev_priv, HDCP_STATUS(dev_priv, cpu_transcoder, port)) & HDCP_STATUS_ENC; } -static inline -bool intel_hdcp2_in_use(struct drm_i915_private *dev_priv, - enum transcoder cpu_transcoder, enum port port) +static bool intel_hdcp2_in_use(struct drm_i915_private *dev_priv, + enum transcoder cpu_transcoder, enum port port) { return intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) & @@ -485,8 +484,8 @@ int intel_hdcp_validate_v_prime(struct intel_connector *connector, return ret; sha_idx += sizeof(sha_text); } else { - DRM_DEBUG_KMS("Invalid number of leftovers %d\n", - sha_leftovers); + drm_dbg_kms(&dev_priv->drm, "Invalid number of leftovers %d\n", + sha_leftovers); return -EINVAL; } @@ -514,11 +513,11 @@ int intel_hdcp_validate_v_prime(struct intel_connector *connector, rep_ctl | HDCP_SHA1_COMPLETE_HASH); if (intel_de_wait_for_set(dev_priv, HDCP_REP_CTL, HDCP_SHA1_COMPLETE, 1)) { - DRM_ERROR("Timed out waiting for SHA1 complete\n"); + drm_err(&dev_priv->drm, "Timed out waiting for SHA1 complete\n"); return -ETIMEDOUT; } if (!(intel_de_read(dev_priv, HDCP_REP_CTL) & HDCP_SHA1_V_MATCH)) { - DRM_DEBUG_KMS("SHA-1 mismatch, HDCP failed\n"); + drm_dbg_kms(&dev_priv->drm, "SHA-1 mismatch, HDCP failed\n"); return -ENXIO; } @@ -537,7 +536,8 @@ int intel_hdcp_auth_downstream(struct intel_connector *connector) ret = intel_hdcp_poll_ksv_fifo(intel_dig_port, shim); if (ret) { - DRM_DEBUG_KMS("KSV list failed to become ready (%d)\n", ret); + drm_dbg_kms(&dev_priv->drm, + "KSV list failed to become ready (%d)\n", ret); return ret; } @@ -547,7 +547,7 @@ int intel_hdcp_auth_downstream(struct intel_connector *connector) if (DRM_HDCP_MAX_DEVICE_EXCEEDED(bstatus[0]) || DRM_HDCP_MAX_CASCADE_EXCEEDED(bstatus[1])) { - DRM_DEBUG_KMS("Max Topology Limit Exceeded\n"); + drm_dbg_kms(&dev_priv->drm, "Max Topology Limit Exceeded\n"); return -EPERM; } @@ -560,13 +560,14 @@ int intel_hdcp_auth_downstream(struct intel_connector *connector) */ num_downstream = DRM_HDCP_NUM_DOWNSTREAM(bstatus[0]); if (num_downstream == 0) { - DRM_DEBUG_KMS("Repeater with zero downstream devices\n"); + drm_dbg_kms(&dev_priv->drm, + "Repeater with zero downstream devices\n"); return -EINVAL; } ksv_fifo = kcalloc(DRM_HDCP_KSV_LEN, num_downstream, GFP_KERNEL); if (!ksv_fifo) { - DRM_DEBUG_KMS("Out of mem: ksv_fifo\n"); + drm_dbg_kms(&dev_priv->drm, "Out of mem: ksv_fifo\n"); return -ENOMEM; } @@ -576,7 +577,7 @@ int intel_hdcp_auth_downstream(struct intel_connector *connector) if (drm_hdcp_check_ksvs_revoked(&dev_priv->drm, ksv_fifo, num_downstream)) { - DRM_ERROR("Revoked Ksv(s) in ksv_fifo\n"); + drm_err(&dev_priv->drm, "Revoked Ksv(s) in ksv_fifo\n"); ret = -EPERM; goto err; } @@ -594,12 +595,13 @@ int intel_hdcp_auth_downstream(struct intel_connector *connector) } if (i == tries) { - DRM_DEBUG_KMS("V Prime validation failed.(%d)\n", ret); + drm_dbg_kms(&dev_priv->drm, + "V Prime validation failed.(%d)\n", ret); goto err; } - DRM_DEBUG_KMS("HDCP is enabled (%d downstream devices)\n", - num_downstream); + drm_dbg_kms(&dev_priv->drm, "HDCP is enabled (%d downstream devices)\n", + num_downstream); ret = 0; err: kfree(ksv_fifo); @@ -642,7 +644,8 @@ static int intel_hdcp_auth(struct intel_connector *connector) if (ret) return ret; if (!hdcp_capable) { - DRM_DEBUG_KMS("Panel is not HDCP capable\n"); + drm_dbg_kms(&dev_priv->drm, + "Panel is not HDCP capable\n"); return -EINVAL; } } @@ -659,7 +662,7 @@ static int intel_hdcp_auth(struct intel_connector *connector) if (intel_de_wait_for_set(dev_priv, HDCP_STATUS(dev_priv, cpu_transcoder, port), HDCP_STATUS_AN_READY, 1)) { - DRM_ERROR("Timed out waiting for An\n"); + drm_err(&dev_priv->drm, "Timed out waiting for An\n"); return -ETIMEDOUT; } @@ -680,7 +683,7 @@ static int intel_hdcp_auth(struct intel_connector *connector) return ret; if (drm_hdcp_check_ksvs_revoked(&dev_priv->drm, bksv.shim, 1)) { - DRM_ERROR("BKSV is revoked\n"); + drm_err(&dev_priv->drm, "BKSV is revoked\n"); return -EPERM; } @@ -706,7 +709,7 @@ static int intel_hdcp_auth(struct intel_connector *connector) /* Wait for R0 ready */ if (wait_for(intel_de_read(dev_priv, HDCP_STATUS(dev_priv, cpu_transcoder, port)) & (HDCP_STATUS_R0_READY | HDCP_STATUS_ENC), 1)) { - DRM_ERROR("Timed out waiting for R0 ready\n"); + drm_err(&dev_priv->drm, "Timed out waiting for R0 ready\n"); return -ETIMEDOUT; } @@ -743,8 +746,10 @@ static int intel_hdcp_auth(struct intel_connector *connector) } if (i == tries) { - DRM_DEBUG_KMS("Timed out waiting for Ri prime match (%x)\n", - intel_de_read(dev_priv, HDCP_STATUS(dev_priv, cpu_transcoder, port))); + drm_dbg_kms(&dev_priv->drm, + "Timed out waiting for Ri prime match (%x)\n", + intel_de_read(dev_priv, HDCP_STATUS(dev_priv, + cpu_transcoder, port))); return -ETIMEDOUT; } @@ -753,7 +758,7 @@ static int intel_hdcp_auth(struct intel_connector *connector) HDCP_STATUS(dev_priv, cpu_transcoder, port), HDCP_STATUS_ENC, ENCRYPT_STATUS_CHANGE_TIMEOUT_MS)) { - DRM_ERROR("Timed out waiting for encryption\n"); + drm_err(&dev_priv->drm, "Timed out waiting for encryption\n"); return -ETIMEDOUT; } @@ -765,7 +770,7 @@ static int intel_hdcp_auth(struct intel_connector *connector) if (repeater_present) return intel_hdcp_auth_downstream(connector); - DRM_DEBUG_KMS("HDCP is enabled (no repeater present)\n"); + drm_dbg_kms(&dev_priv->drm, "HDCP is enabled (no repeater present)\n"); return 0; } @@ -846,8 +851,7 @@ static int _intel_hdcp_enable(struct intel_connector *connector) return ret; } -static inline -struct intel_connector *intel_hdcp_to_connector(struct intel_hdcp *hdcp) +static struct intel_connector *intel_hdcp_to_connector(struct intel_hdcp *hdcp) { return container_of(hdcp, struct intel_connector, hdcp); } @@ -872,7 +876,8 @@ static int intel_hdcp_check_link(struct intel_connector *connector) goto out; } - if (WARN_ON(!intel_hdcp_in_use(dev_priv, cpu_transcoder, port))) { + if (drm_WARN_ON(&dev_priv->drm, + !intel_hdcp_in_use(dev_priv, cpu_transcoder, port))) { drm_err(&dev_priv->drm, "%s:%d HDCP link stopped encryption,%x\n", connector->base.name, connector->base.base.id, @@ -1270,7 +1275,7 @@ static int hdcp2_authentication_key_exchange(struct intel_connector *connector) return ret; if (msgs.send_cert.rx_caps[0] != HDCP_2_2_RX_CAPS_VERSION_VAL) { - DRM_DEBUG_KMS("cert.rx_caps dont claim HDCP2.2\n"); + drm_dbg_kms(&dev_priv->drm, "cert.rx_caps dont claim HDCP2.2\n"); return -EINVAL; } @@ -1279,7 +1284,7 @@ static int hdcp2_authentication_key_exchange(struct intel_connector *connector) if (drm_hdcp_check_ksvs_revoked(&dev_priv->drm, msgs.send_cert.cert_rx.receiver_id, 1)) { - DRM_ERROR("Receiver ID is revoked\n"); + drm_err(&dev_priv->drm, "Receiver ID is revoked\n"); return -EPERM; } @@ -1383,6 +1388,7 @@ static int hdcp2_propagate_stream_management_info(struct intel_connector *connector) { struct intel_digital_port *intel_dig_port = intel_attached_dig_port(connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_hdcp *hdcp = &connector->hdcp; union { struct hdcp2_rep_stream_manage stream_manage; @@ -1423,7 +1429,7 @@ int hdcp2_propagate_stream_management_info(struct intel_connector *connector) hdcp->seq_num_m++; if (hdcp->seq_num_m > HDCP_2_2_SEQ_NUM_MAX) { - DRM_DEBUG_KMS("seq_num_m roll over.\n"); + drm_dbg_kms(&i915->drm, "seq_num_m roll over.\n"); return -1; } @@ -1454,7 +1460,7 @@ int hdcp2_authenticate_repeater_topology(struct intel_connector *connector) if (HDCP_2_2_MAX_CASCADE_EXCEEDED(rx_info[1]) || HDCP_2_2_MAX_DEVS_EXCEEDED(rx_info[1])) { - DRM_DEBUG_KMS("Topology Max Size Exceeded\n"); + drm_dbg_kms(&dev_priv->drm, "Topology Max Size Exceeded\n"); return -EINVAL; } @@ -1462,9 +1468,15 @@ int hdcp2_authenticate_repeater_topology(struct intel_connector *connector) seq_num_v = drm_hdcp_be24_to_cpu((const u8 *)msgs.recvid_list.seq_num_v); + if (!hdcp->hdcp2_encrypted && seq_num_v) { + drm_dbg_kms(&dev_priv->drm, + "Non zero Seq_num_v at first RecvId_List msg\n"); + return -EINVAL; + } + if (seq_num_v < hdcp->seq_num_v) { /* Roll over of the seq_num_v from repeater. Reauthenticate. */ - DRM_DEBUG_KMS("Seq_num_v roll over.\n"); + drm_dbg_kms(&dev_priv->drm, "Seq_num_v roll over.\n"); return -EINVAL; } @@ -1473,7 +1485,7 @@ int hdcp2_authenticate_repeater_topology(struct intel_connector *connector) if (drm_hdcp_check_ksvs_revoked(&dev_priv->drm, msgs.recvid_list.receiver_ids, device_cnt)) { - DRM_ERROR("Revoked receiver ID(s) is in list\n"); + drm_err(&dev_priv->drm, "Revoked receiver ID(s) is in list\n"); return -EPERM; } @@ -1506,25 +1518,27 @@ static int hdcp2_authenticate_repeater(struct intel_connector *connector) static int hdcp2_authenticate_sink(struct intel_connector *connector) { struct intel_digital_port *intel_dig_port = intel_attached_dig_port(connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_hdcp *hdcp = &connector->hdcp; const struct intel_hdcp_shim *shim = hdcp->shim; int ret; ret = hdcp2_authentication_key_exchange(connector); if (ret < 0) { - DRM_DEBUG_KMS("AKE Failed. Err : %d\n", ret); + drm_dbg_kms(&i915->drm, "AKE Failed. Err : %d\n", ret); return ret; } ret = hdcp2_locality_check(connector); if (ret < 0) { - DRM_DEBUG_KMS("Locality Check failed. Err : %d\n", ret); + drm_dbg_kms(&i915->drm, + "Locality Check failed. Err : %d\n", ret); return ret; } ret = hdcp2_session_key_exchange(connector); if (ret < 0) { - DRM_DEBUG_KMS("SKE Failed. Err : %d\n", ret); + drm_dbg_kms(&i915->drm, "SKE Failed. Err : %d\n", ret); return ret; } @@ -1539,7 +1553,8 @@ static int hdcp2_authenticate_sink(struct intel_connector *connector) if (hdcp->is_repeater) { ret = hdcp2_authenticate_repeater(connector); if (ret < 0) { - DRM_DEBUG_KMS("Repeater Auth Failed. Err: %d\n", ret); + drm_dbg_kms(&i915->drm, + "Repeater Auth Failed. Err: %d\n", ret); return ret; } } @@ -1561,8 +1576,9 @@ static int hdcp2_enable_encryption(struct intel_connector *connector) enum transcoder cpu_transcoder = hdcp->cpu_transcoder; int ret; - WARN_ON(intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) & - LINK_ENCRYPTION_STATUS); + drm_WARN_ON(&dev_priv->drm, + intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) & + LINK_ENCRYPTION_STATUS); if (hdcp->shim->toggle_signalling) { ret = hdcp->shim->toggle_signalling(intel_dig_port, true); if (ret) { @@ -1599,8 +1615,8 @@ static int hdcp2_disable_encryption(struct intel_connector *connector) enum transcoder cpu_transcoder = hdcp->cpu_transcoder; int ret; - WARN_ON(!(intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) & - LINK_ENCRYPTION_STATUS)); + drm_WARN_ON(&dev_priv->drm, !(intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port)) & + LINK_ENCRYPTION_STATUS)); intel_de_write(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port), intel_de_read(dev_priv, HDCP2_CTL(dev_priv, cpu_transcoder, port)) & ~CTL_LINK_ENCRYPTION_REQ); @@ -1628,6 +1644,7 @@ static int hdcp2_disable_encryption(struct intel_connector *connector) static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->base.dev); int ret, i, tries = 3; for (i = 0; i < tries; i++) { @@ -1636,10 +1653,10 @@ static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector) break; /* Clearing the mei hdcp session */ - DRM_DEBUG_KMS("HDCP2.2 Auth %d of %d Failed.(%d)\n", - i + 1, tries, ret); + drm_dbg_kms(&i915->drm, "HDCP2.2 Auth %d of %d Failed.(%d)\n", + i + 1, tries, ret); if (hdcp2_deauthenticate_port(connector) < 0) - DRM_DEBUG_KMS("Port deauth failed.\n"); + drm_dbg_kms(&i915->drm, "Port deauth failed.\n"); } if (i != tries) { @@ -1650,9 +1667,10 @@ static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector) msleep(HDCP_2_2_DELAY_BEFORE_ENCRYPTION_EN); ret = hdcp2_enable_encryption(connector); if (ret < 0) { - DRM_DEBUG_KMS("Encryption Enable Failed.(%d)\n", ret); + drm_dbg_kms(&i915->drm, + "Encryption Enable Failed.(%d)\n", ret); if (hdcp2_deauthenticate_port(connector) < 0) - DRM_DEBUG_KMS("Port deauth failed.\n"); + drm_dbg_kms(&i915->drm, "Port deauth failed.\n"); } } @@ -1661,23 +1679,24 @@ static int hdcp2_authenticate_and_encrypt(struct intel_connector *connector) static int _intel_hdcp2_enable(struct intel_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_hdcp *hdcp = &connector->hdcp; int ret; - DRM_DEBUG_KMS("[%s:%d] HDCP2.2 is being enabled. Type: %d\n", - connector->base.name, connector->base.base.id, - hdcp->content_type); + drm_dbg_kms(&i915->drm, "[%s:%d] HDCP2.2 is being enabled. Type: %d\n", + connector->base.name, connector->base.base.id, + hdcp->content_type); ret = hdcp2_authenticate_and_encrypt(connector); if (ret) { - DRM_DEBUG_KMS("HDCP2 Type%d Enabling Failed. (%d)\n", - hdcp->content_type, ret); + drm_dbg_kms(&i915->drm, "HDCP2 Type%d Enabling Failed. (%d)\n", + hdcp->content_type, ret); return ret; } - DRM_DEBUG_KMS("[%s:%d] HDCP2.2 is enabled. Type %d\n", - connector->base.name, connector->base.base.id, - hdcp->content_type); + drm_dbg_kms(&i915->drm, "[%s:%d] HDCP2.2 is enabled. Type %d\n", + connector->base.name, connector->base.base.id, + hdcp->content_type); hdcp->hdcp2_encrypted = true; return 0; @@ -1685,15 +1704,16 @@ static int _intel_hdcp2_enable(struct intel_connector *connector) static int _intel_hdcp2_disable(struct intel_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->base.dev); int ret; - DRM_DEBUG_KMS("[%s:%d] HDCP2.2 is being Disabled\n", - connector->base.name, connector->base.base.id); + drm_dbg_kms(&i915->drm, "[%s:%d] HDCP2.2 is being Disabled\n", + connector->base.name, connector->base.base.id); ret = hdcp2_disable_encryption(connector); if (hdcp2_deauthenticate_port(connector) < 0) - DRM_DEBUG_KMS("Port deauth failed.\n"); + drm_dbg_kms(&i915->drm, "Port deauth failed.\n"); connector->hdcp.hdcp2_encrypted = false; @@ -1720,7 +1740,8 @@ static int intel_hdcp2_check_link(struct intel_connector *connector) goto out; } - if (WARN_ON(!intel_hdcp2_in_use(dev_priv, cpu_transcoder, port))) { + if (drm_WARN_ON(&dev_priv->drm, + !intel_hdcp2_in_use(dev_priv, cpu_transcoder, port))) { drm_err(&dev_priv->drm, "HDCP2.2 link stopped the encryption, %x\n", intel_de_read(dev_priv, HDCP2_STATUS(dev_priv, cpu_transcoder, port))); @@ -1832,8 +1853,7 @@ static const struct component_ops i915_hdcp_component_ops = { .unbind = i915_hdcp_component_unbind, }; -static inline -enum mei_fw_ddi intel_get_mei_fw_ddi_index(enum port port) +static enum mei_fw_ddi intel_get_mei_fw_ddi_index(enum port port) { switch (port) { case PORT_A: @@ -1845,8 +1865,7 @@ enum mei_fw_ddi intel_get_mei_fw_ddi_index(enum port port) } } -static inline -enum mei_fw_tc intel_get_mei_fw_tc(enum transcoder cpu_transcoder) +static enum mei_fw_tc intel_get_mei_fw_tc(enum transcoder cpu_transcoder) { switch (cpu_transcoder) { case TRANSCODER_A ... TRANSCODER_D: @@ -1856,8 +1875,8 @@ enum mei_fw_tc intel_get_mei_fw_tc(enum transcoder cpu_transcoder) } } -static inline int initialize_hdcp_port_data(struct intel_connector *connector, - const struct intel_hdcp_shim *shim) +static int initialize_hdcp_port_data(struct intel_connector *connector, + const struct intel_hdcp_shim *shim) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); struct intel_hdcp *hdcp = &connector->hdcp; @@ -1916,7 +1935,7 @@ void intel_hdcp_component_init(struct drm_i915_private *dev_priv) return; mutex_lock(&dev_priv->hdcp_comp_mutex); - WARN_ON(dev_priv->hdcp_comp_added); + drm_WARN_ON(&dev_priv->drm, dev_priv->hdcp_comp_added); dev_priv->hdcp_comp_added = true; mutex_unlock(&dev_priv->hdcp_comp_mutex); @@ -1935,12 +1954,13 @@ void intel_hdcp_component_init(struct drm_i915_private *dev_priv) static void intel_hdcp2_init(struct intel_connector *connector, const struct intel_hdcp_shim *shim) { + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_hdcp *hdcp = &connector->hdcp; int ret; ret = initialize_hdcp_port_data(connector, shim); if (ret) { - DRM_DEBUG_KMS("Mei hdcp data init failed\n"); + drm_dbg_kms(&i915->drm, "Mei hdcp data init failed\n"); return; } @@ -1990,7 +2010,8 @@ int intel_hdcp_enable(struct intel_connector *connector, return -ENOENT; mutex_lock(&hdcp->mutex); - WARN_ON(hdcp->value == DRM_MODE_CONTENT_PROTECTION_ENABLED); + drm_WARN_ON(&dev_priv->drm, + hdcp->value == DRM_MODE_CONTENT_PROTECTION_ENABLED); hdcp->content_type = content_type; if (INTEL_GEN(dev_priv) >= 12) { @@ -2050,7 +2071,8 @@ int intel_hdcp_disable(struct intel_connector *connector) return ret; } -void intel_hdcp_update_pipe(struct intel_encoder *encoder, +void intel_hdcp_update_pipe(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.h b/drivers/gpu/drm/i915/display/intel_hdcp.h index 1b2eacaf8949..86bbaec120cc 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.h +++ b/drivers/gpu/drm/i915/display/intel_hdcp.h @@ -8,11 +8,10 @@ #include <linux/types.h> -#include <drm/i915_drm.h> - struct drm_connector; struct drm_connector_state; struct drm_i915_private; +struct intel_atomic_state; struct intel_connector; struct intel_crtc_state; struct intel_encoder; @@ -28,7 +27,8 @@ int intel_hdcp_init(struct intel_connector *connector, int intel_hdcp_enable(struct intel_connector *connector, enum transcoder cpu_transcoder, u8 content_type); int intel_hdcp_disable(struct intel_connector *connector); -void intel_hdcp_update_pipe(struct intel_encoder *encoder, +void intel_hdcp_update_pipe(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port); diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index 476ce1d9c557..010f37240710 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -36,7 +36,6 @@ #include <drm/drm_edid.h> #include <drm/drm_hdcp.h> #include <drm/drm_scdc_helper.h> -#include <drm/i915_drm.h> #include <drm/intel_lpe_audio.h> #include "i915_debugfs.h" @@ -45,7 +44,6 @@ #include "intel_audio.h" #include "intel_connector.h" #include "intel_ddi.h" -#include "intel_display_debugfs.h" #include "intel_display_types.h" #include "intel_dp.h" #include "intel_dpio_phy.h" @@ -708,13 +706,15 @@ void intel_read_infoframe(struct intel_encoder *encoder, /* see comment above for the reason for this offset */ ret = hdmi_infoframe_unpack(frame, buffer + 1, sizeof(buffer) - 1); if (ret) { - DRM_DEBUG_KMS("Failed to unpack infoframe type 0x%02x\n", type); + drm_dbg_kms(encoder->base.dev, + "Failed to unpack infoframe type 0x%02x\n", type); return; } if (frame->any.type != type) - DRM_DEBUG_KMS("Found the wrong infoframe type 0x%x (expected 0x%02x)\n", - frame->any.type, type); + drm_dbg_kms(encoder->base.dev, + "Found the wrong infoframe type 0x%x (expected 0x%02x)\n", + frame->any.type, type); } static bool @@ -854,7 +854,8 @@ intel_hdmi_compute_drm_infoframe(struct intel_encoder *encoder, ret = drm_hdmi_infoframe_set_hdr_metadata(frame, conn_state); if (ret < 0) { - DRM_DEBUG_KMS("couldn't set HDR metadata in infoframe\n"); + drm_dbg_kms(&dev_priv->drm, + "couldn't set HDR metadata in infoframe\n"); return false; } @@ -894,8 +895,9 @@ static void g4x_set_infoframes(struct intel_encoder *encoder, if (!(val & VIDEO_DIP_ENABLE)) return; if (port != (val & VIDEO_DIP_PORT_MASK)) { - DRM_DEBUG_KMS("video DIP still enabled on port %c\n", - (val & VIDEO_DIP_PORT_MASK) >> 29); + drm_dbg_kms(&dev_priv->drm, + "video DIP still enabled on port %c\n", + (val & VIDEO_DIP_PORT_MASK) >> 29); return; } val &= ~(VIDEO_DIP_ENABLE | VIDEO_DIP_ENABLE_AVI | @@ -907,8 +909,9 @@ static void g4x_set_infoframes(struct intel_encoder *encoder, if (port != (val & VIDEO_DIP_PORT_MASK)) { if (val & VIDEO_DIP_ENABLE) { - DRM_DEBUG_KMS("video DIP already enabled on port %c\n", - (val & VIDEO_DIP_PORT_MASK) >> 29); + drm_dbg_kms(&dev_priv->drm, + "video DIP already enabled on port %c\n", + (val & VIDEO_DIP_PORT_MASK) >> 29); return; } val &= ~VIDEO_DIP_PORT_MASK; @@ -1265,8 +1268,8 @@ void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable) if (hdmi->dp_dual_mode.type < DRM_DP_DUAL_MODE_TYPE2_DVI) return; - DRM_DEBUG_KMS("%s DP dual mode adaptor TMDS output\n", - enable ? "Enabling" : "Disabling"); + drm_dbg_kms(&dev_priv->drm, "%s DP dual mode adaptor TMDS output\n", + enable ? "Enabling" : "Disabling"); drm_dp_dual_mode_set_tmds_output(hdmi->dp_dual_mode.type, adapter, enable); @@ -1347,13 +1350,14 @@ int intel_hdmi_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_write(intel_dig_port, DRM_HDCP_DDC_AN, an, DRM_HDCP_AN_LEN); if (ret) { - DRM_DEBUG_KMS("Write An over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Write An over DDC failed (%d)\n", + ret); return ret; } ret = intel_gmbus_output_aksv(adapter); if (ret < 0) { - DRM_DEBUG_KMS("Failed to output aksv (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Failed to output aksv (%d)\n", ret); return ret; } return 0; @@ -1362,11 +1366,14 @@ int intel_hdmi_hdcp_write_an_aksv(struct intel_digital_port *intel_dig_port, static int intel_hdmi_hdcp_read_bksv(struct intel_digital_port *intel_dig_port, u8 *bksv) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); + int ret; ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BKSV, bksv, DRM_HDCP_KSV_LEN); if (ret) - DRM_DEBUG_KMS("Read Bksv over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Read Bksv over DDC failed (%d)\n", + ret); return ret; } @@ -1374,11 +1381,14 @@ static int intel_hdmi_hdcp_read_bstatus(struct intel_digital_port *intel_dig_port, u8 *bstatus) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); + int ret; ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BSTATUS, bstatus, DRM_HDCP_BSTATUS_LEN); if (ret) - DRM_DEBUG_KMS("Read bstatus over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Read bstatus over DDC failed (%d)\n", + ret); return ret; } @@ -1386,12 +1396,14 @@ static int intel_hdmi_hdcp_repeater_present(struct intel_digital_port *intel_dig_port, bool *repeater_present) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); int ret; u8 val; ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BCAPS, &val, 1); if (ret) { - DRM_DEBUG_KMS("Read bcaps over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Read bcaps over DDC failed (%d)\n", + ret); return ret; } *repeater_present = val & DRM_HDCP_DDC_BCAPS_REPEATER_PRESENT; @@ -1402,11 +1414,14 @@ static int intel_hdmi_hdcp_read_ri_prime(struct intel_digital_port *intel_dig_port, u8 *ri_prime) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); + int ret; ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_RI_PRIME, ri_prime, DRM_HDCP_RI_LEN); if (ret) - DRM_DEBUG_KMS("Read Ri' over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Read Ri' over DDC failed (%d)\n", + ret); return ret; } @@ -1414,12 +1429,14 @@ static int intel_hdmi_hdcp_read_ksv_ready(struct intel_digital_port *intel_dig_port, bool *ksv_ready) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); int ret; u8 val; ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_BCAPS, &val, 1); if (ret) { - DRM_DEBUG_KMS("Read bcaps over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, "Read bcaps over DDC failed (%d)\n", + ret); return ret; } *ksv_ready = val & DRM_HDCP_DDC_BCAPS_KSV_FIFO_READY; @@ -1430,11 +1447,13 @@ static int intel_hdmi_hdcp_read_ksv_fifo(struct intel_digital_port *intel_dig_port, int num_downstream, u8 *ksv_fifo) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); int ret; ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_KSV_FIFO, ksv_fifo, num_downstream * DRM_HDCP_KSV_LEN); if (ret) { - DRM_DEBUG_KMS("Read ksv fifo over DDC failed (%d)\n", ret); + drm_dbg_kms(&i915->drm, + "Read ksv fifo over DDC failed (%d)\n", ret); return ret; } return 0; @@ -1444,6 +1463,7 @@ static int intel_hdmi_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, int i, u32 *part) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); int ret; if (i >= DRM_HDCP_V_PRIME_NUM_PARTS) @@ -1452,7 +1472,8 @@ int intel_hdmi_hdcp_read_v_prime_part(struct intel_digital_port *intel_dig_port, ret = intel_hdmi_hdcp_read(intel_dig_port, DRM_HDCP_DDC_V_PRIME(i), part, DRM_HDCP_V_PRIME_PART_LEN); if (ret) - DRM_DEBUG_KMS("Read V'[%d] over DDC failed (%d)\n", i, ret); + drm_dbg_kms(&i915->drm, "Read V'[%d] over DDC failed (%d)\n", + i, ret); return ret; } @@ -1475,12 +1496,14 @@ static int kbl_repositioning_enc_en_signal(struct intel_connector *connector) ret = intel_ddi_toggle_hdcp_signalling(&intel_dig_port->base, false); if (ret) { - DRM_ERROR("Disable HDCP signalling failed (%d)\n", ret); + drm_err(&dev_priv->drm, + "Disable HDCP signalling failed (%d)\n", ret); return ret; } ret = intel_ddi_toggle_hdcp_signalling(&intel_dig_port->base, true); if (ret) { - DRM_ERROR("Enable HDCP signalling failed (%d)\n", ret); + drm_err(&dev_priv->drm, + "Enable HDCP signalling failed (%d)\n", ret); return ret; } @@ -1501,8 +1524,8 @@ int intel_hdmi_hdcp_toggle_signalling(struct intel_digital_port *intel_dig_port, ret = intel_ddi_toggle_hdcp_signalling(&intel_dig_port->base, enable); if (ret) { - DRM_ERROR("%s HDCP signalling failed (%d)\n", - enable ? "Enable" : "Disable", ret); + drm_err(&dev_priv->drm, "%s HDCP signalling failed (%d)\n", + enable ? "Enable" : "Disable", ret); return ret; } @@ -1537,10 +1560,13 @@ bool intel_hdmi_hdcp_check_link(struct intel_digital_port *intel_dig_port) intel_de_write(i915, HDCP_RPRIME(i915, cpu_transcoder, port), ri.reg); /* Wait for Ri prime match */ - if (wait_for(intel_de_read(i915, HDCP_STATUS(i915, cpu_transcoder, port)) & + if (wait_for((intel_de_read(i915, HDCP_STATUS(i915, cpu_transcoder, port)) & + (HDCP_STATUS_RI_MATCH | HDCP_STATUS_ENC)) == (HDCP_STATUS_RI_MATCH | HDCP_STATUS_ENC), 1)) { - DRM_ERROR("Ri' mismatch detected, link check failed (%x)\n", - intel_de_read(i915, HDCP_STATUS(i915, cpu_transcoder, port))); + drm_err(&i915->drm, + "Ri' mismatch detected, link check failed (%x)\n", + intel_de_read(i915, HDCP_STATUS(i915, cpu_transcoder, + port))); return false; } return true; @@ -1588,17 +1614,19 @@ static int get_hdcp2_msg_timeout(u8 msg_id, bool is_paired) return -EINVAL; } -static inline -int hdcp2_detect_msg_availability(struct intel_digital_port *intel_digital_port, - u8 msg_id, bool *msg_ready, - ssize_t *msg_sz) +static int +hdcp2_detect_msg_availability(struct intel_digital_port *intel_dig_port, + u8 msg_id, bool *msg_ready, + ssize_t *msg_sz) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); u8 rx_status[HDCP_2_2_HDMI_RXSTATUS_LEN]; int ret; - ret = intel_hdmi_hdcp2_read_rx_status(intel_digital_port, rx_status); + ret = intel_hdmi_hdcp2_read_rx_status(intel_dig_port, rx_status); if (ret < 0) { - DRM_DEBUG_KMS("rx_status read failed. Err %d\n", ret); + drm_dbg_kms(&i915->drm, "rx_status read failed. Err %d\n", + ret); return ret; } @@ -1618,6 +1646,7 @@ static ssize_t intel_hdmi_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port, u8 msg_id, bool paired) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); bool msg_ready = false; int timeout, ret; ssize_t msg_sz = 0; @@ -1632,8 +1661,8 @@ intel_hdmi_hdcp2_wait_for_msg(struct intel_digital_port *intel_dig_port, !ret && msg_ready && msg_sz, timeout * 1000, 1000, 5 * 1000); if (ret) - DRM_DEBUG_KMS("msg_id: %d, ret: %d, timeout: %d\n", - msg_id, ret, timeout); + drm_dbg_kms(&i915->drm, "msg_id: %d, ret: %d, timeout: %d\n", + msg_id, ret, timeout); return ret ? ret : msg_sz; } @@ -1652,6 +1681,7 @@ static int intel_hdmi_hdcp2_read_msg(struct intel_digital_port *intel_dig_port, u8 msg_id, void *buf, size_t size) { + struct drm_i915_private *i915 = to_i915(intel_dig_port->base.base.dev); struct intel_hdmi *hdmi = &intel_dig_port->hdmi; struct intel_hdcp *hdcp = &hdmi->attached_connector->hdcp; unsigned int offset; @@ -1667,15 +1697,17 @@ int intel_hdmi_hdcp2_read_msg(struct intel_digital_port *intel_dig_port, * available buffer. */ if (ret > size) { - DRM_DEBUG_KMS("msg_sz(%zd) is more than exp size(%zu)\n", - ret, size); + drm_dbg_kms(&i915->drm, + "msg_sz(%zd) is more than exp size(%zu)\n", + ret, size); return -1; } offset = HDCP_2_2_HDMI_REG_RD_MSG_OFFSET; ret = intel_hdmi_hdcp_read(intel_dig_port, offset, buf, ret); if (ret) - DRM_DEBUG_KMS("Failed to read msg_id: %d(%zd)\n", msg_id, ret); + drm_dbg_kms(&i915->drm, "Failed to read msg_id: %d(%zd)\n", + msg_id, ret); return ret; } @@ -1718,12 +1750,6 @@ int intel_hdmi_hdcp2_capable(struct intel_digital_port *intel_dig_port, return ret; } -static inline -enum hdcp_wired_protocol intel_hdmi_hdcp2_protocol(void) -{ - return HDCP_PROTOCOL_HDMI; -} - static const struct intel_hdcp_shim intel_hdmi_hdcp_shim = { .write_an_aksv = intel_hdmi_hdcp_write_an_aksv, .read_bksv = intel_hdmi_hdcp_read_bksv, @@ -1871,15 +1897,17 @@ static void intel_enable_hdmi_audio(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); - drm_WARN_ON(encoder->base.dev, !pipe_config->has_hdmi_sink); - DRM_DEBUG_DRIVER("Enabling HDMI audio on pipe %c\n", - pipe_name(crtc->pipe)); + drm_WARN_ON(&i915->drm, !pipe_config->has_hdmi_sink); + drm_dbg_kms(&i915->drm, "Enabling HDMI audio on pipe %c\n", + pipe_name(crtc->pipe)); intel_audio_codec_enable(encoder, pipe_config, conn_state); } -static void g4x_enable_hdmi(struct intel_encoder *encoder, +static void g4x_enable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -1901,7 +1929,8 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder, intel_enable_hdmi_audio(encoder, pipe_config, conn_state); } -static void ibx_enable_hdmi(struct intel_encoder *encoder, +static void ibx_enable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -1952,7 +1981,8 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder, intel_enable_hdmi_audio(encoder, pipe_config, conn_state); } -static void cpt_enable_hdmi(struct intel_encoder *encoder, +static void cpt_enable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -2005,13 +2035,15 @@ static void cpt_enable_hdmi(struct intel_encoder *encoder, intel_enable_hdmi_audio(encoder, pipe_config, conn_state); } -static void vlv_enable_hdmi(struct intel_encoder *encoder, +static void vlv_enable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { } -static void intel_disable_hdmi(struct intel_encoder *encoder, +static void intel_disable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -2069,7 +2101,8 @@ static void intel_disable_hdmi(struct intel_encoder *encoder, intel_dp_dual_mode_set_tmds_output(intel_hdmi, false); } -static void g4x_disable_hdmi(struct intel_encoder *encoder, +static void g4x_disable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -2077,10 +2110,11 @@ static void g4x_disable_hdmi(struct intel_encoder *encoder, intel_audio_codec_disable(encoder, old_crtc_state, old_conn_state); - intel_disable_hdmi(encoder, old_crtc_state, old_conn_state); + intel_disable_hdmi(state, encoder, old_crtc_state, old_conn_state); } -static void pch_disable_hdmi(struct intel_encoder *encoder, +static void pch_disable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -2089,11 +2123,12 @@ static void pch_disable_hdmi(struct intel_encoder *encoder, old_crtc_state, old_conn_state); } -static void pch_post_disable_hdmi(struct intel_encoder *encoder, +static void pch_post_disable_hdmi(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_disable_hdmi(encoder, old_crtc_state, old_conn_state); + intel_disable_hdmi(state, encoder, old_crtc_state, old_conn_state); } static int intel_hdmi_source_max_tmds_clock(struct intel_encoder *encoder) @@ -2276,14 +2311,9 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, } } - /* Display WA #1139: glk */ - if (bpc == 12 && IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1) && - adjusted_mode->htotal > 5460) - return false; - - /* Display Wa_1405510057:icl */ + /* Display Wa_1405510057:icl,ehl */ if (crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 && - bpc == 10 && INTEL_GEN(dev_priv) >= 11 && + bpc == 10 && IS_GEN(dev_priv, 11) && (adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start) % 8 == 2) return false; @@ -2291,29 +2321,27 @@ static bool hdmi_deep_color_possible(const struct intel_crtc_state *crtc_state, return true; } -static bool -intel_hdmi_ycbcr420_config(struct drm_connector *connector, - struct intel_crtc_state *config) +static int +intel_hdmi_ycbcr420_config(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { - struct intel_crtc *intel_crtc = to_intel_crtc(config->uapi.crtc); - - if (!connector->ycbcr_420_allowed) { - DRM_ERROR("Platform doesn't support YCBCR420 output\n"); - return false; - } + struct drm_connector *connector = conn_state->connector; + struct drm_i915_private *i915 = to_i915(connector->dev); + const struct drm_display_mode *adjusted_mode = + &crtc_state->hw.adjusted_mode; - config->output_format = INTEL_OUTPUT_FORMAT_YCBCR420; + if (!drm_mode_is_420_only(&connector->display_info, adjusted_mode)) + return 0; - /* YCBCR 420 output conversion needs a scaler */ - if (skl_update_scaler_crtc(config)) { - DRM_DEBUG_KMS("Scaler allocation for output failed\n"); - return false; + if (!connector->ycbcr_420_allowed) { + drm_err(&i915->drm, + "Platform doesn't support YCBCR420 output\n"); + return -EINVAL; } - intel_pch_panel_fitting(intel_crtc, config, - DRM_MODE_SCALE_FULLSCREEN); + crtc_state->output_format = INTEL_OUTPUT_FORMAT_YCBCR420; - return true; + return intel_pch_panel_fitting(crtc_state, conn_state); } static int intel_hdmi_port_clock(int clock, int bpc) @@ -2347,6 +2375,7 @@ static int intel_hdmi_compute_bpc(struct intel_encoder *encoder, static int intel_hdmi_compute_clock(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder); const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; @@ -2371,13 +2400,15 @@ static int intel_hdmi_compute_clock(struct intel_encoder *encoder, if (crtc_state->pipe_bpp > bpc * 3) crtc_state->pipe_bpp = bpc * 3; - DRM_DEBUG_KMS("picking %d bpc for HDMI output (pipe bpp: %d)\n", - bpc, crtc_state->pipe_bpp); + drm_dbg_kms(&i915->drm, + "picking %d bpc for HDMI output (pipe bpp: %d)\n", + bpc, crtc_state->pipe_bpp); if (hdmi_port_clock_valid(intel_hdmi, crtc_state->port_clock, false, crtc_state->has_hdmi_sink) != MODE_OK) { - DRM_DEBUG_KMS("unsupported HDMI clock (%d kHz), rejecting mode\n", - crtc_state->port_clock); + drm_dbg_kms(&i915->drm, + "unsupported HDMI clock (%d kHz), rejecting mode\n", + crtc_state->port_clock); return -EINVAL; } @@ -2438,12 +2469,9 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder, if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) pipe_config->pixel_multiplier = 2; - if (drm_mode_is_420_only(&connector->display_info, adjusted_mode)) { - if (!intel_hdmi_ycbcr420_config(connector, pipe_config)) { - DRM_ERROR("Can't support YCBCR420 output\n"); - return -EINVAL; - } - } + ret = intel_hdmi_ycbcr420_config(pipe_config, conn_state); + if (ret) + return ret; pipe_config->limited_color_range = intel_hdmi_limited_color_range(pipe_config, conn_state); @@ -2480,25 +2508,26 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder, } } - intel_hdmi_compute_gcp_infoframe(encoder, pipe_config, conn_state); + intel_hdmi_compute_gcp_infoframe(encoder, pipe_config, + conn_state); if (!intel_hdmi_compute_avi_infoframe(encoder, pipe_config, conn_state)) { - DRM_DEBUG_KMS("bad AVI infoframe\n"); + drm_dbg_kms(&dev_priv->drm, "bad AVI infoframe\n"); return -EINVAL; } if (!intel_hdmi_compute_spd_infoframe(encoder, pipe_config, conn_state)) { - DRM_DEBUG_KMS("bad SPD infoframe\n"); + drm_dbg_kms(&dev_priv->drm, "bad SPD infoframe\n"); return -EINVAL; } if (!intel_hdmi_compute_hdmi_infoframe(encoder, pipe_config, conn_state)) { - DRM_DEBUG_KMS("bad HDMI infoframe\n"); + drm_dbg_kms(&dev_priv->drm, "bad HDMI infoframe\n"); return -EINVAL; } if (!intel_hdmi_compute_drm_infoframe(encoder, pipe_config, conn_state)) { - DRM_DEBUG_KMS("bad DRM infoframe\n"); + drm_dbg_kms(&dev_priv->drm, "bad DRM infoframe\n"); return -EINVAL; } @@ -2548,7 +2577,8 @@ intel_hdmi_dp_dual_mode_detect(struct drm_connector *connector, bool has_edid) */ if (has_edid && !connector->override_edid && intel_bios_is_port_dp_dual_mode(dev_priv, port)) { - DRM_DEBUG_KMS("Assuming DP dual mode adaptor presence based on VBT\n"); + drm_dbg_kms(&dev_priv->drm, + "Assuming DP dual mode adaptor presence based on VBT\n"); type = DRM_DP_DUAL_MODE_TYPE1_DVI; } else { type = DRM_DP_DUAL_MODE_NONE; @@ -2562,9 +2592,10 @@ intel_hdmi_dp_dual_mode_detect(struct drm_connector *connector, bool has_edid) hdmi->dp_dual_mode.max_tmds_clock = drm_dp_dual_mode_max_tmds_clock(type, adapter); - DRM_DEBUG_KMS("DP dual mode adaptor (%s) detected (max TMDS clock: %d kHz)\n", - drm_dp_get_dual_mode_type_name(type), - hdmi->dp_dual_mode.max_tmds_clock); + drm_dbg_kms(&dev_priv->drm, + "DP dual mode adaptor (%s) detected (max TMDS clock: %d kHz)\n", + drm_dp_get_dual_mode_type_name(type), + hdmi->dp_dual_mode.max_tmds_clock); } static bool @@ -2584,7 +2615,8 @@ intel_hdmi_set_edid(struct drm_connector *connector) edid = drm_get_edid(connector, i2c); if (!edid && !intel_gmbus_is_forced_bit(i2c)) { - DRM_DEBUG_KMS("HDMI GMBUS EDID read failed, retry using GPIO bit-banging\n"); + drm_dbg_kms(&dev_priv->drm, + "HDMI GMBUS EDID read failed, retry using GPIO bit-banging\n"); intel_gmbus_force_bit(i2c, true); edid = drm_get_edid(connector, i2c); intel_gmbus_force_bit(i2c, false); @@ -2616,8 +2648,8 @@ intel_hdmi_detect(struct drm_connector *connector, bool force) struct intel_encoder *encoder = &hdmi_to_dig_port(intel_hdmi)->base; intel_wakeref_t wakeref; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, connector->name); + drm_dbg_kms(&dev_priv->drm, "[CONNECTOR:%d:%s]\n", + connector->base.id, connector->name); wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS); @@ -2648,8 +2680,10 @@ out: static void intel_hdmi_force(struct drm_connector *connector) { - DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", - connector->base.id, connector->name); + struct drm_i915_private *i915 = to_i915(connector->dev); + + drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s]\n", + connector->base.id, connector->name); intel_hdmi_unset_edid(connector); @@ -2670,7 +2704,8 @@ static int intel_hdmi_get_modes(struct drm_connector *connector) return intel_connector_update_modes(connector, edid); } -static void intel_hdmi_pre_enable(struct intel_encoder *encoder, +static void intel_hdmi_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -2684,7 +2719,8 @@ static void intel_hdmi_pre_enable(struct intel_encoder *encoder, pipe_config, conn_state); } -static void vlv_hdmi_pre_enable(struct intel_encoder *encoder, +static void vlv_hdmi_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -2701,12 +2737,13 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder, pipe_config->has_infoframe, pipe_config, conn_state); - g4x_enable_hdmi(encoder, pipe_config, conn_state); + g4x_enable_hdmi(state, encoder, pipe_config, conn_state); vlv_wait_port_ready(dev_priv, dport, 0x0); } -static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder, +static void vlv_hdmi_pre_pll_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -2715,7 +2752,8 @@ static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder, vlv_phy_pre_pll_enable(encoder, pipe_config); } -static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder, +static void chv_hdmi_pre_pll_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -2724,14 +2762,16 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder, chv_phy_pre_pll_enable(encoder, pipe_config); } -static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder, +static void chv_hdmi_post_pll_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { chv_phy_post_pll_disable(encoder, old_crtc_state); } -static void vlv_hdmi_post_disable(struct intel_encoder *encoder, +static void vlv_hdmi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -2739,7 +2779,8 @@ static void vlv_hdmi_post_disable(struct intel_encoder *encoder, vlv_phy_reset_lanes(encoder, old_crtc_state); } -static void chv_hdmi_post_disable(struct intel_encoder *encoder, +static void chv_hdmi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -2754,7 +2795,8 @@ static void chv_hdmi_post_disable(struct intel_encoder *encoder, vlv_dpio_put(dev_priv); } -static void chv_hdmi_pre_enable(struct intel_encoder *encoder, +static void chv_hdmi_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -2772,7 +2814,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder, pipe_config->has_infoframe, pipe_config, conn_state); - g4x_enable_hdmi(encoder, pipe_config, conn_state); + g4x_enable_hdmi(state, encoder, pipe_config, conn_state); vlv_wait_port_ready(dev_priv, dport, 0x0); @@ -2791,6 +2833,7 @@ intel_hdmi_get_i2c_adapter(struct drm_connector *connector) static void intel_hdmi_create_i2c_symlink(struct drm_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->dev); struct i2c_adapter *adapter = intel_hdmi_get_i2c_adapter(connector); struct kobject *i2c_kobj = &adapter->dev.kobj; struct kobject *connector_kobj = &connector->kdev->kobj; @@ -2798,7 +2841,7 @@ static void intel_hdmi_create_i2c_symlink(struct drm_connector *connector) ret = sysfs_create_link(connector_kobj, i2c_kobj, i2c_kobj->name); if (ret) - DRM_ERROR("Failed to create i2c symlink (%d)\n", ret); + drm_err(&i915->drm, "Failed to create i2c symlink (%d)\n", ret); } static void intel_hdmi_remove_i2c_symlink(struct drm_connector *connector) @@ -2819,8 +2862,6 @@ intel_hdmi_connector_register(struct drm_connector *connector) if (ret) return ret; - intel_connector_debugfs_add(connector); - intel_hdmi_create_i2c_symlink(connector); return ret; @@ -2927,9 +2968,10 @@ bool intel_hdmi_handle_sink_scrambling(struct intel_encoder *encoder, if (!sink_scrambling->supported) return true; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] scrambling=%s, TMDS bit clock ratio=1/%d\n", - connector->base.id, connector->name, - yesno(scrambling), high_tmds_clock_ratio ? 40 : 10); + drm_dbg_kms(&dev_priv->drm, + "[CONNECTOR:%d:%s] scrambling=%s, TMDS bit clock ratio=1/%d\n", + connector->base.id, connector->name, + yesno(scrambling), high_tmds_clock_ratio ? 40 : 10); /* Set TMDS bit clock ratio to 1/40 or 1/10, and enable/disable scrambling */ return drm_scdc_set_high_tmds_clock_ratio(adapter, @@ -3071,8 +3113,9 @@ static u8 intel_hdmi_ddc_pin(struct intel_encoder *encoder) ddc_pin = intel_bios_alternate_ddc_pin(encoder); if (ddc_pin) { - DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (VBT)\n", - ddc_pin, port_name(port)); + drm_dbg_kms(&dev_priv->drm, + "Using DDC pin 0x%x for port %c (VBT)\n", + ddc_pin, port_name(port)); return ddc_pin; } @@ -3089,8 +3132,9 @@ static u8 intel_hdmi_ddc_pin(struct intel_encoder *encoder) else ddc_pin = g4x_port_to_ddc_pin(dev_priv, port); - DRM_DEBUG_KMS("Using DDC pin 0x%x for port %c (platform default)\n", - ddc_pin, port_name(port)); + drm_dbg_kms(&dev_priv->drm, + "Using DDC pin 0x%x for port %c (platform default)\n", + ddc_pin, port_name(port)); return ddc_pin; } @@ -3147,8 +3191,9 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, enum port port = intel_encoder->port; struct cec_connector_info conn_info; - DRM_DEBUG_KMS("Adding HDMI connector on [ENCODER:%d:%s]\n", - intel_encoder->base.base.id, intel_encoder->base.name); + drm_dbg_kms(&dev_priv->drm, + "Adding HDMI connector on [ENCODER:%d:%s]\n", + intel_encoder->base.base.id, intel_encoder->base.name); if (INTEL_GEN(dev_priv) < 12 && drm_WARN_ON(dev, port == PORT_A)) return; @@ -3176,6 +3221,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, connector->ycbcr_420_allowed = true; intel_encoder->hpd_pin = intel_hpd_pin_default(dev_priv, port); + intel_connector->polled = DRM_CONNECTOR_POLL_HPD; if (HAS_DDI(dev_priv)) intel_connector->get_hw_state = intel_ddi_connector_get_hw_state; @@ -3191,7 +3237,8 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, int ret = intel_hdcp_init(intel_connector, &intel_hdmi_hdcp_shim); if (ret) - DRM_DEBUG_KMS("HDCP init failed, skipping.\n"); + drm_dbg_kms(&dev_priv->drm, + "HDCP init failed, skipping.\n"); } /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written @@ -3210,16 +3257,16 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port, cec_notifier_conn_register(dev->dev, port_identifier(port), &conn_info); if (!intel_hdmi->cec_notifier) - DRM_DEBUG_KMS("CEC notifier get failed\n"); + drm_dbg_kms(&dev_priv->drm, "CEC notifier get failed\n"); } static enum intel_hotplug_state intel_hdmi_hotplug(struct intel_encoder *encoder, - struct intel_connector *connector, bool irq_received) + struct intel_connector *connector) { enum intel_hotplug_state state; - state = intel_encoder_hotplug(encoder, connector, irq_received); + state = intel_encoder_hotplug(encoder, connector); /* * On many platforms the HDMI live state signal is known to be @@ -3233,7 +3280,7 @@ intel_hdmi_hotplug(struct intel_encoder *encoder, * time around we didn't detect any change in the sink's connection * status. */ - if (state == INTEL_HOTPLUG_UNCHANGED && irq_received) + if (state == INTEL_HOTPLUG_UNCHANGED && !connector->hotplug_retries) state = INTEL_HOTPLUG_RETRY; return state; diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h index d3659d0b408b..8ff1f76a63df 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.h +++ b/drivers/gpu/drm/i915/display/intel_hdmi.h @@ -9,8 +9,6 @@ #include <linux/hdmi.h> #include <linux/types.h> -#include <drm/i915_drm.h> - #include "i915_reg.h" struct drm_connector; diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c index 127a2f28c1ac..4f6f560e093e 100644 --- a/drivers/gpu/drm/i915/display/intel_hotplug.c +++ b/drivers/gpu/drm/i915/display/intel_hotplug.c @@ -23,8 +23,6 @@ #include <linux/kernel.h> -#include <drm/i915_drm.h> - #include "i915_drv.h" #include "intel_display_types.h" #include "intel_hotplug.h" @@ -89,29 +87,16 @@ enum hpd_pin intel_hpd_pin_default(struct drm_i915_private *dev_priv, enum port port) { - switch (port) { - case PORT_A: - return HPD_PORT_A; - case PORT_B: - return HPD_PORT_B; - case PORT_C: - return HPD_PORT_C; - case PORT_D: - return HPD_PORT_D; - case PORT_E: - return HPD_PORT_E; - case PORT_F: - if (IS_CNL_WITH_PORT_F(dev_priv)) - return HPD_PORT_E; - return HPD_PORT_F; - case PORT_G: - return HPD_PORT_G; - case PORT_H: - return HPD_PORT_H; - case PORT_I: - return HPD_PORT_I; + enum phy phy = intel_port_to_phy(dev_priv, port); + + switch (phy) { + case PHY_F: + return IS_CNL_WITH_PORT_F(dev_priv) ? HPD_PORT_E : HPD_PORT_F; + case PHY_A ... PHY_E: + case PHY_G ... PHY_I: + return HPD_PORT_A + phy - PHY_A; default: - MISSING_CASE(port); + MISSING_CASE(phy); return HPD_NONE; } } @@ -185,10 +170,13 @@ static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv, hpd->stats[pin].count += increment; if (hpd->stats[pin].count > threshold) { hpd->stats[pin].state = HPD_MARK_DISABLED; - DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", pin); + drm_dbg_kms(&dev_priv->drm, + "HPD interrupt storm detected on PIN %d\n", pin); storm = true; } else { - DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", pin, + drm_dbg_kms(&dev_priv->drm, + "Received HPD interrupt on PIN %d - cnt: %d\n", + pin, hpd->stats[pin].count); } @@ -217,7 +205,8 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv) dev_priv->hotplug.stats[pin].state != HPD_MARK_DISABLED) continue; - DRM_INFO("HPD interrupt storm detected on connector %s: " + drm_info(&dev_priv->drm, + "HPD interrupt storm detected on connector %s: " "switching from hotplug detection to polling\n", connector->base.name); @@ -242,36 +231,38 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) container_of(work, typeof(*dev_priv), hotplug.reenable_work.work); struct drm_device *dev = &dev_priv->drm; + struct drm_connector_list_iter conn_iter; + struct intel_connector *connector; intel_wakeref_t wakeref; enum hpd_pin pin; wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); spin_lock_irq(&dev_priv->irq_lock); - for_each_hpd_pin(pin) { - struct drm_connector_list_iter conn_iter; - struct intel_connector *connector; - if (dev_priv->hotplug.stats[pin].state != HPD_DISABLED) + drm_connector_list_iter_begin(dev, &conn_iter); + for_each_intel_connector_iter(connector, &conn_iter) { + pin = intel_connector_hpd_pin(connector); + if (pin == HPD_NONE || + dev_priv->hotplug.stats[pin].state != HPD_DISABLED) continue; - dev_priv->hotplug.stats[pin].state = HPD_ENABLED; - - drm_connector_list_iter_begin(dev, &conn_iter); - for_each_intel_connector_iter(connector, &conn_iter) { - if (intel_connector_hpd_pin(connector) == pin) { - if (connector->base.polled != connector->polled) - DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n", - connector->base.name); - connector->base.polled = connector->polled; - if (!connector->base.polled) - connector->base.polled = DRM_CONNECTOR_POLL_HPD; - } - } - drm_connector_list_iter_end(&conn_iter); + if (connector->base.polled != connector->polled) + drm_dbg(&dev_priv->drm, + "Reenabling HPD on connector %s\n", + connector->base.name); + connector->base.polled = connector->polled; } + drm_connector_list_iter_end(&conn_iter); + + for_each_hpd_pin(pin) { + if (dev_priv->hotplug.stats[pin].state == HPD_DISABLED) + dev_priv->hotplug.stats[pin].state = HPD_ENABLED; + } + if (dev_priv->display_irqs_enabled && dev_priv->display.hpd_irq_setup) dev_priv->display.hpd_irq_setup(dev_priv); + spin_unlock_irq(&dev_priv->irq_lock); intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref); @@ -279,8 +270,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work) enum intel_hotplug_state intel_encoder_hotplug(struct intel_encoder *encoder, - struct intel_connector *connector, - bool irq_received) + struct intel_connector *connector) { struct drm_device *dev = connector->base.dev; enum drm_connector_status old_status; @@ -294,11 +284,12 @@ intel_encoder_hotplug(struct intel_encoder *encoder, if (old_status == connector->base.status) return INTEL_HOTPLUG_UNCHANGED; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n", - connector->base.base.id, - connector->base.name, - drm_get_connector_status_name(old_status), - drm_get_connector_status_name(connector->base.status)); + drm_dbg_kms(&to_i915(dev)->drm, + "[CONNECTOR:%d:%s] status updated from %s to %s\n", + connector->base.base.id, + connector->base.name, + drm_get_connector_status_name(old_status), + drm_get_connector_status_name(connector->base.status)); return INTEL_HOTPLUG_CHANGED; } @@ -372,7 +363,7 @@ static void i915_hotplug_work_func(struct work_struct *work) u32 hpd_retry_bits; mutex_lock(&dev->mode_config.mutex); - DRM_DEBUG_KMS("running encoder hotplug functions\n"); + drm_dbg_kms(&dev_priv->drm, "running encoder hotplug functions\n"); spin_lock_irq(&dev_priv->irq_lock); @@ -400,11 +391,17 @@ static void i915_hotplug_work_func(struct work_struct *work) struct intel_encoder *encoder = intel_attached_encoder(connector); - DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n", - connector->base.name, pin); + if (hpd_event_bits & hpd_bit) + connector->hotplug_retries = 0; + else + connector->hotplug_retries++; + + drm_dbg_kms(&dev_priv->drm, + "Connector %s (pin %i) received hotplug event. (retry %d)\n", + connector->base.name, pin, + connector->hotplug_retries); - switch (encoder->hotplug(encoder, connector, - hpd_event_bits & hpd_bit)) { + switch (encoder->hotplug(encoder, connector)) { case INTEL_HOTPLUG_UNCHANGED: break; case INTEL_HOTPLUG_CHANGED: @@ -486,9 +483,10 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, long_hpd = long_mask & BIT(pin); - DRM_DEBUG_DRIVER("digital hpd on [ENCODER:%d:%s] - %s\n", - encoder->base.base.id, encoder->base.name, - long_hpd ? "long" : "short"); + drm_dbg(&dev_priv->drm, + "digital hpd on [ENCODER:%d:%s] - %s\n", + encoder->base.base.id, encoder->base.name, + long_hpd ? "long" : "short"); queue_dig = true; if (long_hpd) { @@ -617,16 +615,17 @@ static void i915_hpd_poll_init_work(struct work_struct *work) drm_connector_list_iter_begin(dev, &conn_iter); for_each_intel_connector_iter(connector, &conn_iter) { - enum hpd_pin pin = intel_connector_hpd_pin(connector); + enum hpd_pin pin; + + pin = intel_connector_hpd_pin(connector); + if (pin == HPD_NONE) + continue; connector->base.polled = connector->polled; - if (pin != HPD_NONE && I915_HAS_HOTPLUG(dev_priv) && - !connector->base.polled) - connector->base.polled = enabled ? - DRM_CONNECTOR_POLL_CONNECT | - DRM_CONNECTOR_POLL_DISCONNECT : - DRM_CONNECTOR_POLL_HPD; + if (enabled && connector->base.polled == DRM_CONNECTOR_POLL_HPD) + connector->base.polled = DRM_CONNECTOR_POLL_CONNECT | + DRM_CONNECTOR_POLL_DISCONNECT; } drm_connector_list_iter_end(&conn_iter); diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.h b/drivers/gpu/drm/i915/display/intel_hotplug.h index 087b5f57b321..777b0743257e 100644 --- a/drivers/gpu/drm/i915/display/intel_hotplug.h +++ b/drivers/gpu/drm/i915/display/intel_hotplug.h @@ -8,8 +8,6 @@ #include <linux/types.h> -#include <drm/i915_drm.h> - struct drm_i915_private; struct intel_connector; struct intel_encoder; @@ -17,8 +15,7 @@ enum port; void intel_hpd_poll_init(struct drm_i915_private *dev_priv); enum intel_hotplug_state intel_encoder_hotplug(struct intel_encoder *encoder, - struct intel_connector *connector, - bool irq_received); + struct intel_connector *connector); void intel_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 pin_mask, u32 long_mask); void intel_hpd_init(struct drm_i915_private *dev_priv); diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c index 516e7179a5a4..ad5cc13037ae 100644 --- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c +++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c @@ -127,7 +127,8 @@ lpe_audio_platdev_create(struct drm_i915_private *dev_priv) kfree(pdata); if (IS_ERR(platdev)) { - DRM_ERROR("Failed to allocate LPE audio platform device\n"); + drm_err(&dev_priv->drm, + "Failed to allocate LPE audio platform device\n"); return platdev; } @@ -190,7 +191,8 @@ static bool lpe_audio_detect(struct drm_i915_private *dev_priv) }; if (!pci_dev_present(atom_hdaudio_ids)) { - DRM_INFO("HDaudio controller not detected, using LPE audio instead\n"); + drm_info(&dev_priv->drm, + "HDaudio controller not detected, using LPE audio instead\n"); lpe_present = true; } } @@ -203,18 +205,19 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv) dev_priv->lpe_audio.irq = irq_alloc_desc(0); if (dev_priv->lpe_audio.irq < 0) { - DRM_ERROR("Failed to allocate IRQ desc: %d\n", + drm_err(&dev_priv->drm, "Failed to allocate IRQ desc: %d\n", dev_priv->lpe_audio.irq); ret = dev_priv->lpe_audio.irq; goto err; } - DRM_DEBUG("irq = %d\n", dev_priv->lpe_audio.irq); + drm_dbg(&dev_priv->drm, "irq = %d\n", dev_priv->lpe_audio.irq); ret = lpe_audio_irq_init(dev_priv); if (ret) { - DRM_ERROR("Failed to initialize irqchip for lpe audio: %d\n", + drm_err(&dev_priv->drm, + "Failed to initialize irqchip for lpe audio: %d\n", ret); goto err_free_irq; } @@ -223,7 +226,8 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv) if (IS_ERR(dev_priv->lpe_audio.platdev)) { ret = PTR_ERR(dev_priv->lpe_audio.platdev); - DRM_ERROR("Failed to create lpe audio platform device: %d\n", + drm_err(&dev_priv->drm, + "Failed to create lpe audio platform device: %d\n", ret); goto err_free_irq; } @@ -259,8 +263,8 @@ void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv) ret = generic_handle_irq(dev_priv->lpe_audio.irq); if (ret) - DRM_ERROR_RATELIMITED("error handling LPE audio irq: %d\n", - ret); + drm_err_ratelimited(&dev_priv->drm, + "error handling LPE audio irq: %d\n", ret); } /** @@ -278,7 +282,8 @@ int intel_lpe_audio_init(struct drm_i915_private *dev_priv) if (lpe_audio_detect(dev_priv)) { ret = lpe_audio_setup(dev_priv); if (ret < 0) - DRM_ERROR("failed to setup LPE Audio bridge\n"); + drm_err(&dev_priv->drm, + "failed to setup LPE Audio bridge\n"); } return ret; } diff --git a/drivers/gpu/drm/i915/display/intel_lspcon.c b/drivers/gpu/drm/i915/display/intel_lspcon.c index d807c5648c87..6ff7b226f0a1 100644 --- a/drivers/gpu/drm/i915/display/intel_lspcon.c +++ b/drivers/gpu/drm/i915/display/intel_lspcon.c @@ -522,7 +522,7 @@ u32 lspcon_infoframes_enabled(struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config) { /* FIXME actually read this from the hw */ - return enc_to_intel_lspcon(encoder)->active; + return 0; } void lspcon_resume(struct intel_lspcon *lspcon) diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c index b7ad0b534790..872f2a489339 100644 --- a/drivers/gpu/drm/i915/display/intel_lvds.c +++ b/drivers/gpu/drm/i915/display/intel_lvds.c @@ -37,7 +37,6 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_atomic.h" @@ -182,8 +181,9 @@ static void intel_lvds_pps_get_hw_state(struct drm_i915_private *dev_priv, if (INTEL_GEN(dev_priv) <= 4 && pps->t1_t2 == 0 && pps->t5 == 0 && pps->t3 == 0 && pps->tx == 0) { - DRM_DEBUG_KMS("Panel power timings uninitialized, " - "setting defaults\n"); + drm_dbg_kms(&dev_priv->drm, + "Panel power timings uninitialized, " + "setting defaults\n"); /* Set T2 to 40ms and T5 to 200ms in 100 usec units */ pps->t1_t2 = 40 * 10; pps->t5 = 200 * 10; @@ -192,10 +192,10 @@ static void intel_lvds_pps_get_hw_state(struct drm_i915_private *dev_priv, pps->tx = 200 * 10; } - DRM_DEBUG_DRIVER("LVDS PPS:t1+t2 %d t3 %d t4 %d t5 %d tx %d " - "divider %d port %d powerdown_on_reset %d\n", - pps->t1_t2, pps->t3, pps->t4, pps->t5, pps->tx, - pps->divider, pps->port, pps->powerdown_on_reset); + drm_dbg(&dev_priv->drm, "LVDS PPS:t1+t2 %d t3 %d t4 %d t5 %d tx %d " + "divider %d port %d powerdown_on_reset %d\n", + pps->t1_t2, pps->t3, pps->t4, pps->t5, pps->tx, + pps->divider, pps->port, pps->powerdown_on_reset); } static void intel_lvds_pps_init_hw(struct drm_i915_private *dev_priv, @@ -220,7 +220,8 @@ static void intel_lvds_pps_init_hw(struct drm_i915_private *dev_priv, REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK, pps->divider) | REG_FIELD_PREP(PANEL_POWER_CYCLE_DELAY_MASK, DIV_ROUND_UP(pps->t4, 1000) + 1)); } -static void intel_pre_enable_lvds(struct intel_encoder *encoder, +static void intel_pre_enable_lvds(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -301,7 +302,8 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder, /* * Sets the power state for the panel. */ -static void intel_enable_lvds(struct intel_encoder *encoder, +static void intel_enable_lvds(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -317,12 +319,14 @@ static void intel_enable_lvds(struct intel_encoder *encoder, intel_de_posting_read(dev_priv, lvds_encoder->reg); if (intel_de_wait_for_set(dev_priv, PP_STATUS(0), PP_ON, 5000)) - DRM_ERROR("timed out waiting for panel to power on\n"); + drm_err(&dev_priv->drm, + "timed out waiting for panel to power on\n"); intel_panel_enable_backlight(pipe_config, conn_state); } -static void intel_disable_lvds(struct intel_encoder *encoder, +static void intel_disable_lvds(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -332,35 +336,39 @@ static void intel_disable_lvds(struct intel_encoder *encoder, intel_de_write(dev_priv, PP_CONTROL(0), intel_de_read(dev_priv, PP_CONTROL(0)) & ~PANEL_POWER_ON); if (intel_de_wait_for_clear(dev_priv, PP_STATUS(0), PP_ON, 1000)) - DRM_ERROR("timed out waiting for panel to power off\n"); + drm_err(&dev_priv->drm, + "timed out waiting for panel to power off\n"); intel_de_write(dev_priv, lvds_encoder->reg, intel_de_read(dev_priv, lvds_encoder->reg) & ~LVDS_PORT_EN); intel_de_posting_read(dev_priv, lvds_encoder->reg); } -static void gmch_disable_lvds(struct intel_encoder *encoder, +static void gmch_disable_lvds(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { intel_panel_disable_backlight(old_conn_state); - intel_disable_lvds(encoder, old_crtc_state, old_conn_state); + intel_disable_lvds(state, encoder, old_crtc_state, old_conn_state); } -static void pch_disable_lvds(struct intel_encoder *encoder, +static void pch_disable_lvds(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { intel_panel_disable_backlight(old_conn_state); } -static void pch_post_disable_lvds(struct intel_encoder *encoder, +static void pch_post_disable_lvds(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_disable_lvds(encoder, old_crtc_state, old_conn_state); + intel_disable_lvds(state, encoder, old_crtc_state, old_conn_state); } static enum drm_mode_status @@ -395,10 +403,11 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder, struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->uapi.crtc); unsigned int lvds_bpp; + int ret; /* Should never happen!! */ if (INTEL_GEN(dev_priv) < 4 && intel_crtc->pipe == 0) { - DRM_ERROR("Can't support LVDS on pipe A\n"); + drm_err(&dev_priv->drm, "Can't support LVDS on pipe A\n"); return -EINVAL; } @@ -408,8 +417,9 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder, lvds_bpp = 6*3; if (lvds_bpp != pipe_config->pipe_bpp && !pipe_config->bw_constrained) { - DRM_DEBUG_KMS("forcing display bpp (was %d) to LVDS (%d)\n", - pipe_config->pipe_bpp, lvds_bpp); + drm_dbg_kms(&dev_priv->drm, + "forcing display bpp (was %d) to LVDS (%d)\n", + pipe_config->pipe_bpp, lvds_bpp); pipe_config->pipe_bpp = lvds_bpp; } @@ -427,16 +437,15 @@ static int intel_lvds_compute_config(struct intel_encoder *intel_encoder, if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) return -EINVAL; - if (HAS_PCH_SPLIT(dev_priv)) { + if (HAS_PCH_SPLIT(dev_priv)) pipe_config->has_pch_encoder = true; - intel_pch_panel_fitting(intel_crtc, pipe_config, - conn_state->scaling_mode); - } else { - intel_gmch_panel_fitting(intel_crtc, pipe_config, - conn_state->scaling_mode); - - } + if (HAS_GMCH(dev_priv)) + ret = intel_gmch_panel_fitting(pipe_config, conn_state); + else + ret = intel_pch_panel_fitting(pipe_config, conn_state); + if (ret) + return ret; /* * XXX: It would be nice to support lower refresh rates on the @@ -833,7 +842,8 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) } if (!dev_priv->vbt.int_lvds_support) { - DRM_DEBUG_KMS("Internal LVDS support disabled by VBT\n"); + drm_dbg_kms(&dev_priv->drm, + "Internal LVDS support disabled by VBT\n"); return; } @@ -852,10 +862,12 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) pin = GMBUS_PIN_PANEL; if (!intel_bios_is_lvds_present(dev_priv, &pin)) { if ((lvds & LVDS_PORT_EN) == 0) { - DRM_DEBUG_KMS("LVDS is not present in VBT\n"); + drm_dbg_kms(&dev_priv->drm, + "LVDS is not present in VBT\n"); return; } - DRM_DEBUG_KMS("LVDS is not present in VBT, but enabled anyway\n"); + drm_dbg_kms(&dev_priv->drm, + "LVDS is not present in VBT, but enabled anyway\n"); } lvds_encoder = kzalloc(sizeof(*lvds_encoder), GFP_KERNEL); @@ -969,7 +981,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) */ fixed_mode = intel_encoder_current_mode(intel_encoder); if (fixed_mode) { - DRM_DEBUG_KMS("using current (BIOS) mode: "); + drm_dbg_kms(&dev_priv->drm, "using current (BIOS) mode: "); drm_mode_debug_printmodeline(fixed_mode); fixed_mode->type |= DRM_MODE_TYPE_PREFERRED; } @@ -985,8 +997,8 @@ out: intel_panel_setup_backlight(connector, INVALID_PIPE); lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder); - DRM_DEBUG_KMS("detected %s-link lvds configuration\n", - lvds_encoder->is_dual_link ? "dual" : "single"); + drm_dbg_kms(&dev_priv->drm, "detected %s-link lvds configuration\n", + lvds_encoder->is_dual_link ? "dual" : "single"); lvds_encoder->a3_power = lvds & LVDS_A3_POWER_MASK; @@ -995,7 +1007,7 @@ out: failed: mutex_unlock(&dev->mode_config.mutex); - DRM_DEBUG_KMS("No LVDS modes found, disabling.\n"); + drm_dbg_kms(&dev_priv->drm, "No LVDS modes found, disabling.\n"); drm_connector_cleanup(connector); drm_encoder_cleanup(encoder); kfree(lvds_encoder); diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c index dfd78fccd456..cc6b00959586 100644 --- a/drivers/gpu/drm/i915/display/intel_opregion.c +++ b/drivers/gpu/drm/i915/display/intel_opregion.c @@ -30,8 +30,6 @@ #include <linux/firmware.h> #include <acpi/video.h> -#include <drm/i915_drm.h> - #include "display/intel_panel.h" #include "i915_drv.h" diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c index 5f1207dec10e..66711e62fa71 100644 --- a/drivers/gpu/drm/i915/display/intel_overlay.c +++ b/drivers/gpu/drm/i915/display/intel_overlay.c @@ -27,7 +27,6 @@ */ #include <drm/drm_fourcc.h> -#include <drm/i915_drm.h> #include "gem/i915_gem_pm.h" #include "gt/intel_ring.h" @@ -282,7 +281,7 @@ static void intel_overlay_flip_prepare(struct intel_overlay *overlay, enum pipe pipe = overlay->crtc->pipe; struct intel_frontbuffer *from = NULL, *to = NULL; - WARN_ON(overlay->old_vma); + drm_WARN_ON(&overlay->i915->drm, overlay->old_vma); if (overlay->vma) from = intel_frontbuffer_get(overlay->vma->obj); @@ -324,7 +323,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay, /* check for underruns */ tmp = intel_de_read(dev_priv, DOVSTA); if (tmp & (1 << 17)) - DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); + drm_dbg(&dev_priv->drm, "overlay underrun, DOVSTA: %x\n", tmp); rq = alloc_request(overlay, NULL); if (IS_ERR(rq)) @@ -351,7 +350,7 @@ static void intel_overlay_release_old_vma(struct intel_overlay *overlay) struct i915_vma *vma; vma = fetch_and_zero(&overlay->old_vma); - if (WARN_ON(!vma)) + if (drm_WARN_ON(&overlay->i915->drm, !vma)) return; intel_frontbuffer_flip_complete(overlay->i915, @@ -397,7 +396,7 @@ static int intel_overlay_off(struct intel_overlay *overlay) struct i915_request *rq; u32 *cs, flip_addr = overlay->flip_addr; - WARN_ON(!overlay->active); + drm_WARN_ON(&overlay->i915->drm, !overlay->active); /* According to intel docs the overlay hw may hang (when switching * off) without loading the filter coeffs. It is however unclear whether @@ -1069,7 +1068,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, overlay = dev_priv->overlay; if (!overlay) { - DRM_DEBUG("userspace bug: no overlay\n"); + drm_dbg(&dev_priv->drm, "userspace bug: no overlay\n"); return -ENODEV; } @@ -1093,7 +1092,8 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data, drm_modeset_lock_all(dev); if (i915_gem_object_is_tiled(new_bo)) { - DRM_DEBUG_KMS("buffer used for overlay image can not be tiled\n"); + drm_dbg_kms(&dev_priv->drm, + "buffer used for overlay image can not be tiled\n"); ret = -EINVAL; goto out_unlock; } @@ -1228,7 +1228,7 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data, overlay = dev_priv->overlay; if (!overlay) { - DRM_DEBUG("userspace bug: no overlay\n"); + drm_dbg(&dev_priv->drm, "userspace bug: no overlay\n"); return -ENODEV; } @@ -1342,7 +1342,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv) if (!HAS_OVERLAY(dev_priv)) return; - engine = dev_priv->engine[RCS0]; + engine = dev_priv->gt.engine[RCS0]; if (!engine || !engine->kernel_context) return; @@ -1372,7 +1372,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv) update_reg_attrs(overlay, overlay->regs); dev_priv->overlay = overlay; - DRM_INFO("Initialized overlay support.\n"); + drm_info(&dev_priv->drm, "Initialized overlay support.\n"); return; out_free: diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c index 585688b6ebac..aa931f9f0d6a 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.c +++ b/drivers/gpu/drm/i915/display/intel_panel.c @@ -176,24 +176,23 @@ intel_panel_vbt_fixed_mode(struct intel_connector *connector) } /* adjusted_mode has been preset to be the panel's fixed mode */ -void -intel_pch_panel_fitting(struct intel_crtc *intel_crtc, - struct intel_crtc_state *pipe_config, - int fitting_mode) +int intel_pch_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { - const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; - int x = 0, y = 0, width = 0, height = 0; + const struct drm_display_mode *adjusted_mode = + &crtc_state->hw.adjusted_mode; + int x, y, width, height; /* Native modes don't need fitting */ - if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w && - adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h && - pipe_config->output_format != INTEL_OUTPUT_FORMAT_YCBCR420) - goto done; + if (adjusted_mode->crtc_hdisplay == crtc_state->pipe_src_w && + adjusted_mode->crtc_vdisplay == crtc_state->pipe_src_h && + crtc_state->output_format != INTEL_OUTPUT_FORMAT_YCBCR420) + return 0; - switch (fitting_mode) { + switch (conn_state->scaling_mode) { case DRM_MODE_SCALE_CENTER: - width = pipe_config->pipe_src_w; - height = pipe_config->pipe_src_h; + width = crtc_state->pipe_src_w; + height = crtc_state->pipe_src_h; x = (adjusted_mode->crtc_hdisplay - width + 1)/2; y = (adjusted_mode->crtc_vdisplay - height + 1)/2; break; @@ -202,18 +201,18 @@ intel_pch_panel_fitting(struct intel_crtc *intel_crtc, /* Scale but preserve the aspect ratio */ { u32 scaled_width = adjusted_mode->crtc_hdisplay - * pipe_config->pipe_src_h; - u32 scaled_height = pipe_config->pipe_src_w + * crtc_state->pipe_src_h; + u32 scaled_height = crtc_state->pipe_src_w * adjusted_mode->crtc_vdisplay; if (scaled_width > scaled_height) { /* pillar */ - width = scaled_height / pipe_config->pipe_src_h; + width = scaled_height / crtc_state->pipe_src_h; if (width & 1) width++; x = (adjusted_mode->crtc_hdisplay - width + 1) / 2; y = 0; height = adjusted_mode->crtc_vdisplay; } else if (scaled_width < scaled_height) { /* letter */ - height = scaled_width / pipe_config->pipe_src_w; + height = scaled_width / crtc_state->pipe_src_w; if (height & 1) height++; y = (adjusted_mode->crtc_vdisplay - height + 1) / 2; @@ -227,6 +226,10 @@ intel_pch_panel_fitting(struct intel_crtc *intel_crtc, } break; + case DRM_MODE_SCALE_NONE: + WARN_ON(adjusted_mode->crtc_hdisplay != crtc_state->pipe_src_w); + WARN_ON(adjusted_mode->crtc_vdisplay != crtc_state->pipe_src_h); + /* fall through */ case DRM_MODE_SCALE_FULLSCREEN: x = y = 0; width = adjusted_mode->crtc_hdisplay; @@ -234,14 +237,15 @@ intel_pch_panel_fitting(struct intel_crtc *intel_crtc, break; default: - WARN(1, "bad panel fit mode: %d\n", fitting_mode); - return; + MISSING_CASE(conn_state->scaling_mode); + return -EINVAL; } -done: - pipe_config->pch_pfit.pos = (x << 16) | y; - pipe_config->pch_pfit.size = (width << 16) | height; - pipe_config->pch_pfit.enabled = pipe_config->pch_pfit.size != 0; + drm_rect_init(&crtc_state->pch_pfit.dst, + x, y, width, height); + crtc_state->pch_pfit.enabled = true; + + return 0; } static void @@ -287,7 +291,7 @@ centre_vertically(struct drm_display_mode *adjusted_mode, adjusted_mode->crtc_vsync_end = adjusted_mode->crtc_vsync_start + sync_width; } -static inline u32 panel_fitter_scaling(u32 source, u32 target) +static u32 panel_fitter_scaling(u32 source, u32 target) { /* * Floating point operation is not supported. So the FACTOR @@ -300,13 +304,14 @@ static inline u32 panel_fitter_scaling(u32 source, u32 target) return (FACTOR * ratio + FACTOR/2) / FACTOR; } -static void i965_scale_aspect(struct intel_crtc_state *pipe_config, +static void i965_scale_aspect(struct intel_crtc_state *crtc_state, u32 *pfit_control) { - const struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; + const struct drm_display_mode *adjusted_mode = + &crtc_state->hw.adjusted_mode; u32 scaled_width = adjusted_mode->crtc_hdisplay * - pipe_config->pipe_src_h; - u32 scaled_height = pipe_config->pipe_src_w * + crtc_state->pipe_src_h; + u32 scaled_height = crtc_state->pipe_src_w * adjusted_mode->crtc_vdisplay; /* 965+ is easy, it does everything in hw */ @@ -316,18 +321,18 @@ static void i965_scale_aspect(struct intel_crtc_state *pipe_config, else if (scaled_width < scaled_height) *pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER; - else if (adjusted_mode->crtc_hdisplay != pipe_config->pipe_src_w) + else if (adjusted_mode->crtc_hdisplay != crtc_state->pipe_src_w) *pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO; } -static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, +static void i9xx_scale_aspect(struct intel_crtc_state *crtc_state, u32 *pfit_control, u32 *pfit_pgm_ratios, u32 *border) { - struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; + struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; u32 scaled_width = adjusted_mode->crtc_hdisplay * - pipe_config->pipe_src_h; - u32 scaled_height = pipe_config->pipe_src_w * + crtc_state->pipe_src_h; + u32 scaled_height = crtc_state->pipe_src_w * adjusted_mode->crtc_vdisplay; u32 bits; @@ -339,11 +344,11 @@ static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, if (scaled_width > scaled_height) { /* pillar */ centre_horizontally(adjusted_mode, scaled_height / - pipe_config->pipe_src_h); + crtc_state->pipe_src_h); *border = LVDS_BORDER_ENABLE; - if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay) { - bits = panel_fitter_scaling(pipe_config->pipe_src_h, + if (crtc_state->pipe_src_h != adjusted_mode->crtc_vdisplay) { + bits = panel_fitter_scaling(crtc_state->pipe_src_h, adjusted_mode->crtc_vdisplay); *pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT | @@ -355,11 +360,11 @@ static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, } else if (scaled_width < scaled_height) { /* letter */ centre_vertically(adjusted_mode, scaled_width / - pipe_config->pipe_src_w); + crtc_state->pipe_src_w); *border = LVDS_BORDER_ENABLE; - if (pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) { - bits = panel_fitter_scaling(pipe_config->pipe_src_w, + if (crtc_state->pipe_src_w != adjusted_mode->crtc_hdisplay) { + bits = panel_fitter_scaling(crtc_state->pipe_src_w, adjusted_mode->crtc_hdisplay); *pfit_pgm_ratios |= (bits << PFIT_HORIZ_SCALE_SHIFT | @@ -377,35 +382,35 @@ static void i9xx_scale_aspect(struct intel_crtc_state *pipe_config, } } -void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, - struct intel_crtc_state *pipe_config, - int fitting_mode) +int intel_gmch_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { - struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); u32 pfit_control = 0, pfit_pgm_ratios = 0, border = 0; - struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; + struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode; /* Native modes don't need fitting */ - if (adjusted_mode->crtc_hdisplay == pipe_config->pipe_src_w && - adjusted_mode->crtc_vdisplay == pipe_config->pipe_src_h) + if (adjusted_mode->crtc_hdisplay == crtc_state->pipe_src_w && + adjusted_mode->crtc_vdisplay == crtc_state->pipe_src_h) goto out; - switch (fitting_mode) { + switch (conn_state->scaling_mode) { case DRM_MODE_SCALE_CENTER: /* * For centered modes, we have to calculate border widths & * heights and modify the values programmed into the CRTC. */ - centre_horizontally(adjusted_mode, pipe_config->pipe_src_w); - centre_vertically(adjusted_mode, pipe_config->pipe_src_h); + centre_horizontally(adjusted_mode, crtc_state->pipe_src_w); + centre_vertically(adjusted_mode, crtc_state->pipe_src_h); border = LVDS_BORDER_ENABLE; break; case DRM_MODE_SCALE_ASPECT: /* Scale but preserve the aspect ratio */ if (INTEL_GEN(dev_priv) >= 4) - i965_scale_aspect(pipe_config, &pfit_control); + i965_scale_aspect(crtc_state, &pfit_control); else - i9xx_scale_aspect(pipe_config, &pfit_control, + i9xx_scale_aspect(crtc_state, &pfit_control, &pfit_pgm_ratios, &border); break; case DRM_MODE_SCALE_FULLSCREEN: @@ -413,8 +418,8 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, * Full scaling, even if it changes the aspect ratio. * Fortunately this is all done for us in hw. */ - if (pipe_config->pipe_src_h != adjusted_mode->crtc_vdisplay || - pipe_config->pipe_src_w != adjusted_mode->crtc_hdisplay) { + if (crtc_state->pipe_src_h != adjusted_mode->crtc_vdisplay || + crtc_state->pipe_src_w != adjusted_mode->crtc_hdisplay) { pfit_control |= PFIT_ENABLE; if (INTEL_GEN(dev_priv) >= 4) pfit_control |= PFIT_SCALING_AUTO; @@ -426,16 +431,14 @@ void intel_gmch_panel_fitting(struct intel_crtc *intel_crtc, } break; default: - drm_WARN(&dev_priv->drm, 1, "bad panel fit mode: %d\n", - fitting_mode); - return; + MISSING_CASE(conn_state->scaling_mode); + return -EINVAL; } /* 965+ wants fuzzy fitting */ /* FIXME: handle multiple panels by failing gracefully */ if (INTEL_GEN(dev_priv) >= 4) - pfit_control |= ((intel_crtc->pipe << PFIT_PIPE_SHIFT) | - PFIT_FILTER_FUZZY); + pfit_control |= PFIT_PIPE(crtc->pipe) | PFIT_FILTER_FUZZY; out: if ((pfit_control & PFIT_ENABLE) == 0) { @@ -444,12 +447,14 @@ out: } /* Make sure pre-965 set dither correctly for 18bpp panels. */ - if (INTEL_GEN(dev_priv) < 4 && pipe_config->pipe_bpp == 18) + if (INTEL_GEN(dev_priv) < 4 && crtc_state->pipe_bpp == 18) pfit_control |= PANEL_8TO6_DITHER_ENABLE; - pipe_config->gmch_pfit.control = pfit_control; - pipe_config->gmch_pfit.pgm_ratios = pfit_pgm_ratios; - pipe_config->gmch_pfit.lvds_border_bits = border; + crtc_state->gmch_pfit.control = pfit_control; + crtc_state->gmch_pfit.pgm_ratios = pfit_pgm_ratios; + crtc_state->gmch_pfit.lvds_border_bits = border; + + return 0; } /** @@ -485,8 +490,8 @@ static u32 scale(u32 source_val, } /* Scale user_level in range [0..user_max] to [hw_min..hw_max]. */ -static inline u32 scale_user_to_hw(struct intel_connector *connector, - u32 user_level, u32 user_max) +static u32 scale_user_to_hw(struct intel_connector *connector, + u32 user_level, u32 user_max) { struct intel_panel *panel = &connector->panel; @@ -496,8 +501,8 @@ static inline u32 scale_user_to_hw(struct intel_connector *connector, /* Scale user_level in range [0..user_max] to [0..hw_max], clamping the result * to [hw_min..hw_max]. */ -static inline u32 clamp_user_to_hw(struct intel_connector *connector, - u32 user_level, u32 user_max) +static u32 clamp_user_to_hw(struct intel_connector *connector, + u32 user_level, u32 user_max) { struct intel_panel *panel = &connector->panel; u32 hw_level; @@ -509,8 +514,8 @@ static inline u32 clamp_user_to_hw(struct intel_connector *connector, } /* Scale hw_level in range [hw_min..hw_max] to [0..user_max]. */ -static inline u32 scale_hw_to_user(struct intel_connector *connector, - u32 hw_level, u32 user_max) +static u32 scale_hw_to_user(struct intel_connector *connector, + u32 hw_level, u32 user_max) { struct intel_panel *panel = &connector->panel; @@ -685,9 +690,10 @@ static void intel_panel_actually_set_backlight(const struct drm_connector_state *conn_state, u32 level) { struct intel_connector *connector = to_intel_connector(conn_state->connector); + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_panel *panel = &connector->panel; - DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level); + drm_dbg_kms(&i915->drm, "set backlight PWM = %d\n", level); level = intel_panel_compute_brightness(connector, level); panel->backlight.set(conn_state, level); @@ -868,8 +874,8 @@ void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_st * another client is not activated. */ if (dev_priv->drm.switch_power_state == DRM_SWITCH_POWER_CHANGING) { - drm_dbg(&dev_priv->drm, - "Skipping backlight disable on vga switch\n"); + drm_dbg_kms(&dev_priv->drm, + "Skipping backlight disable on vga switch\n"); return; } @@ -1245,7 +1251,7 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector) mutex_unlock(&dev_priv->backlight_lock); - drm_dbg(&dev_priv->drm, "get backlight PWM = %d\n", val); + drm_dbg_kms(&dev_priv->drm, "get backlight PWM = %d\n", val); return val; } @@ -1336,6 +1342,7 @@ static const struct backlight_ops intel_backlight_device_ops = { int intel_backlight_device_register(struct intel_connector *connector) { + struct drm_i915_private *i915 = to_i915(connector->base.dev); struct intel_panel *panel = &connector->panel; struct backlight_properties props; @@ -1375,14 +1382,15 @@ int intel_backlight_device_register(struct intel_connector *connector) &intel_backlight_device_ops, &props); if (IS_ERR(panel->backlight.device)) { - DRM_ERROR("Failed to register backlight: %ld\n", - PTR_ERR(panel->backlight.device)); + drm_err(&i915->drm, "Failed to register backlight: %ld\n", + PTR_ERR(panel->backlight.device)); panel->backlight.device = NULL; return -ENODEV; } - DRM_DEBUG_KMS("Connector %s backlight sysfs interface registered\n", - connector->base.name); + drm_dbg_kms(&i915->drm, + "Connector %s backlight sysfs interface registered\n", + connector->base.name); return 0; } @@ -1883,6 +1891,7 @@ static int pwm_setup_backlight(struct intel_connector *connector, struct drm_i915_private *dev_priv = to_i915(dev); struct intel_panel *panel = &connector->panel; const char *desc; + u32 level, ns; int retval; /* Get the right PWM chip for DSI backlight according to VBT */ @@ -1907,8 +1916,12 @@ static int pwm_setup_backlight(struct intel_connector *connector, */ pwm_apply_args(panel->backlight.pwm); - retval = pwm_config(panel->backlight.pwm, CRC_PMIC_PWM_PERIOD_NS, - CRC_PMIC_PWM_PERIOD_NS); + panel->backlight.min = 0; /* 0% */ + panel->backlight.max = 100; /* 100% */ + level = intel_panel_compute_brightness(connector, 100); + ns = DIV_ROUND_UP(level * CRC_PMIC_PWM_PERIOD_NS, 100); + + retval = pwm_config(panel->backlight.pwm, ns, CRC_PMIC_PWM_PERIOD_NS); if (retval < 0) { drm_err(&dev_priv->drm, "Failed to configure the pwm chip\n"); pwm_put(panel->backlight.pwm); @@ -1916,11 +1929,10 @@ static int pwm_setup_backlight(struct intel_connector *connector, return retval; } - panel->backlight.min = 0; /* 0% */ - panel->backlight.max = 100; /* 100% */ - panel->backlight.level = DIV_ROUND_UP( - pwm_get_duty_cycle(panel->backlight.pwm) * 100, - CRC_PMIC_PWM_PERIOD_NS); + level = DIV_ROUND_UP(pwm_get_duty_cycle(panel->backlight.pwm) * 100, + CRC_PMIC_PWM_PERIOD_NS); + panel->backlight.level = + intel_panel_compute_brightness(connector, level); panel->backlight.enabled = panel->backlight.level != 0; drm_info(&dev_priv->drm, "Using %s PWM for LCD backlight control\n", @@ -1928,7 +1940,8 @@ static int pwm_setup_backlight(struct intel_connector *connector, return 0; } -void intel_panel_update_backlight(struct intel_encoder *encoder, +void intel_panel_update_backlight(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h index cedeea443336..968b95281cb4 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.h +++ b/drivers/gpu/drm/i915/display/intel_panel.h @@ -25,19 +25,18 @@ int intel_panel_init(struct intel_panel *panel, void intel_panel_fini(struct intel_panel *panel); void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode, struct drm_display_mode *adjusted_mode); -void intel_pch_panel_fitting(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config, - int fitting_mode); -void intel_gmch_panel_fitting(struct intel_crtc *crtc, - struct intel_crtc_state *pipe_config, - int fitting_mode); +int intel_pch_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); +int intel_gmch_panel_fitting(struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); void intel_panel_set_backlight_acpi(const struct drm_connector_state *conn_state, u32 level, u32 max); int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe); void intel_panel_enable_backlight(const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); -void intel_panel_update_backlight(struct intel_encoder *encoder, +void intel_panel_update_backlight(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state); void intel_panel_disable_backlight(const struct drm_connector_state *old_conn_state); diff --git a/drivers/gpu/drm/i915/display/intel_pipe_crc.c b/drivers/gpu/drm/i915/display/intel_pipe_crc.c index 59d7e3cb3445..a9a5df2fee4d 100644 --- a/drivers/gpu/drm/i915/display/intel_pipe_crc.c +++ b/drivers/gpu/drm/i915/display/intel_pipe_crc.c @@ -441,15 +441,11 @@ display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s) return 0; } -void intel_display_crc_init(struct drm_i915_private *dev_priv) +void intel_crtc_crc_init(struct intel_crtc *crtc) { - enum pipe pipe; + struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc; - for_each_pipe(dev_priv, pipe) { - struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; - - spin_lock_init(&pipe_crc->lock); - } + spin_lock_init(&pipe_crc->lock); } static int i8xx_crc_source_valid(struct drm_i915_private *dev_priv, @@ -587,7 +583,8 @@ int intel_crtc_verify_crc_source(struct drm_crtc *crtc, const char *source_name, int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name) { struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index]; + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_pipe_crc *pipe_crc = &intel_crtc->pipe_crc; enum intel_display_power_domain power_domain; enum intel_pipe_crc_source source; intel_wakeref_t wakeref; @@ -640,7 +637,7 @@ void intel_crtc_enable_pipe_crc(struct intel_crtc *intel_crtc) { struct drm_crtc *crtc = &intel_crtc->base; struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index]; + struct intel_pipe_crc *pipe_crc = &intel_crtc->pipe_crc; u32 val = 0; if (!crtc->crc.opened) @@ -660,7 +657,7 @@ void intel_crtc_disable_pipe_crc(struct intel_crtc *intel_crtc) { struct drm_crtc *crtc = &intel_crtc->base; struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[crtc->index]; + struct intel_pipe_crc *pipe_crc = &intel_crtc->pipe_crc; /* Swallow crc's until we stop generating them. */ spin_lock_irq(&pipe_crc->lock); diff --git a/drivers/gpu/drm/i915/display/intel_pipe_crc.h b/drivers/gpu/drm/i915/display/intel_pipe_crc.h index db258a756fc6..43012b189415 100644 --- a/drivers/gpu/drm/i915/display/intel_pipe_crc.h +++ b/drivers/gpu/drm/i915/display/intel_pipe_crc.h @@ -13,7 +13,7 @@ struct drm_i915_private; struct intel_crtc; #ifdef CONFIG_DEBUG_FS -void intel_display_crc_init(struct drm_i915_private *dev_priv); +void intel_crtc_crc_init(struct intel_crtc *crtc); int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name); int intel_crtc_verify_crc_source(struct drm_crtc *crtc, const char *source_name, size_t *values_cnt); @@ -22,7 +22,7 @@ const char *const *intel_crtc_get_crc_sources(struct drm_crtc *crtc, void intel_crtc_disable_pipe_crc(struct intel_crtc *crtc); void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc); #else -static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {} +static inline void intel_crtc_crc_init(struct intel_crtc *crtc) {} #define intel_crtc_set_crc_source NULL #define intel_crtc_verify_crc_source NULL #define intel_crtc_get_crc_sources NULL diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index b4942b6445ae..b7a2c102648a 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -30,6 +30,7 @@ #include "intel_display_types.h" #include "intel_psr.h" #include "intel_sprite.h" +#include "intel_hdmi.h" /** * DOC: Panel Self Refresh (PSR/SRD) @@ -137,41 +138,42 @@ static void psr_irq_control(struct drm_i915_private *dev_priv) intel_de_write(dev_priv, imr_reg, val); } -static void psr_event_print(u32 val, bool psr2_enabled) +static void psr_event_print(struct drm_i915_private *i915, + u32 val, bool psr2_enabled) { - DRM_DEBUG_KMS("PSR exit events: 0x%x\n", val); + drm_dbg_kms(&i915->drm, "PSR exit events: 0x%x\n", val); if (val & PSR_EVENT_PSR2_WD_TIMER_EXPIRE) - DRM_DEBUG_KMS("\tPSR2 watchdog timer expired\n"); + drm_dbg_kms(&i915->drm, "\tPSR2 watchdog timer expired\n"); if ((val & PSR_EVENT_PSR2_DISABLED) && psr2_enabled) - DRM_DEBUG_KMS("\tPSR2 disabled\n"); + drm_dbg_kms(&i915->drm, "\tPSR2 disabled\n"); if (val & PSR_EVENT_SU_DIRTY_FIFO_UNDERRUN) - DRM_DEBUG_KMS("\tSU dirty FIFO underrun\n"); + drm_dbg_kms(&i915->drm, "\tSU dirty FIFO underrun\n"); if (val & PSR_EVENT_SU_CRC_FIFO_UNDERRUN) - DRM_DEBUG_KMS("\tSU CRC FIFO underrun\n"); + drm_dbg_kms(&i915->drm, "\tSU CRC FIFO underrun\n"); if (val & PSR_EVENT_GRAPHICS_RESET) - DRM_DEBUG_KMS("\tGraphics reset\n"); + drm_dbg_kms(&i915->drm, "\tGraphics reset\n"); if (val & PSR_EVENT_PCH_INTERRUPT) - DRM_DEBUG_KMS("\tPCH interrupt\n"); + drm_dbg_kms(&i915->drm, "\tPCH interrupt\n"); if (val & PSR_EVENT_MEMORY_UP) - DRM_DEBUG_KMS("\tMemory up\n"); + drm_dbg_kms(&i915->drm, "\tMemory up\n"); if (val & PSR_EVENT_FRONT_BUFFER_MODIFY) - DRM_DEBUG_KMS("\tFront buffer modification\n"); + drm_dbg_kms(&i915->drm, "\tFront buffer modification\n"); if (val & PSR_EVENT_WD_TIMER_EXPIRE) - DRM_DEBUG_KMS("\tPSR watchdog timer expired\n"); + drm_dbg_kms(&i915->drm, "\tPSR watchdog timer expired\n"); if (val & PSR_EVENT_PIPE_REGISTERS_UPDATE) - DRM_DEBUG_KMS("\tPIPE registers updated\n"); + drm_dbg_kms(&i915->drm, "\tPIPE registers updated\n"); if (val & PSR_EVENT_REGISTER_UPDATE) - DRM_DEBUG_KMS("\tRegister updated\n"); + drm_dbg_kms(&i915->drm, "\tRegister updated\n"); if (val & PSR_EVENT_HDCP_ENABLE) - DRM_DEBUG_KMS("\tHDCP enabled\n"); + drm_dbg_kms(&i915->drm, "\tHDCP enabled\n"); if (val & PSR_EVENT_KVMR_SESSION_ENABLE) - DRM_DEBUG_KMS("\tKVMR session enabled\n"); + drm_dbg_kms(&i915->drm, "\tKVMR session enabled\n"); if (val & PSR_EVENT_VBI_ENABLE) - DRM_DEBUG_KMS("\tVBI enabled\n"); + drm_dbg_kms(&i915->drm, "\tVBI enabled\n"); if (val & PSR_EVENT_LPSP_MODE_EXIT) - DRM_DEBUG_KMS("\tLPSP mode exited\n"); + drm_dbg_kms(&i915->drm, "\tLPSP mode exited\n"); if ((val & PSR_EVENT_PSR_DISABLE) && !psr2_enabled) - DRM_DEBUG_KMS("\tPSR disabled\n"); + drm_dbg_kms(&i915->drm, "\tPSR disabled\n"); } void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir) @@ -209,7 +211,7 @@ void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir) intel_de_write(dev_priv, PSR_EVENT(cpu_transcoder), val); - psr_event_print(val, psr2_enabled); + psr_event_print(dev_priv, val, psr2_enabled); } } @@ -249,18 +251,21 @@ static bool intel_dp_get_alpm_status(struct intel_dp *intel_dp) static u8 intel_dp_get_sink_sync_latency(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u8 val = 8; /* assume the worst if we can't read the value */ if (drm_dp_dpcd_readb(&intel_dp->aux, DP_SYNCHRONIZATION_LATENCY_IN_SINK, &val) == 1) val &= DP_MAX_RESYNC_FRAME_COUNT_MASK; else - DRM_DEBUG_KMS("Unable to get sink synchronization latency, assuming 8 frames\n"); + drm_dbg_kms(&i915->drm, + "Unable to get sink synchronization latency, assuming 8 frames\n"); return val; } static u16 intel_dp_get_su_x_granulartiy(struct intel_dp *intel_dp) { + struct drm_i915_private *i915 = dp_to_i915(intel_dp); u16 val; ssize_t r; @@ -273,7 +278,8 @@ static u16 intel_dp_get_su_x_granulartiy(struct intel_dp *intel_dp) r = drm_dp_dpcd_read(&intel_dp->aux, DP_PSR2_SU_X_GRANULARITY, &val, 2); if (r != 2) - DRM_DEBUG_KMS("Unable to read DP_PSR2_SU_X_GRANULARITY\n"); + drm_dbg_kms(&i915->drm, + "Unable to read DP_PSR2_SU_X_GRANULARITY\n"); /* * Spec says that if the value read is 0 the default granularity should @@ -304,7 +310,7 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp) drm_dbg_kms(&dev_priv->drm, "eDP panel supports PSR version %x\n", intel_dp->psr_dpcd[0]); - if (drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_NO_PSR)) { + if (drm_dp_has_quirk(&intel_dp->desc, 0, DP_DPCD_QUIRK_NO_PSR)) { drm_dbg_kms(&dev_priv->drm, "PSR support not currently available for this panel\n"); return; @@ -352,39 +358,6 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp) } } -static void intel_psr_setup_vsc(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) -{ - struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); - struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - struct dp_sdp psr_vsc; - - if (dev_priv->psr.psr2_enabled) { - /* Prepare VSC Header for SU as per EDP 1.4 spec, Table 6.11 */ - memset(&psr_vsc, 0, sizeof(psr_vsc)); - psr_vsc.sdp_header.HB0 = 0; - psr_vsc.sdp_header.HB1 = 0x7; - if (dev_priv->psr.colorimetry_support) { - psr_vsc.sdp_header.HB2 = 0x5; - psr_vsc.sdp_header.HB3 = 0x13; - } else { - psr_vsc.sdp_header.HB2 = 0x4; - psr_vsc.sdp_header.HB3 = 0xe; - } - } else { - /* Prepare VSC packet as per EDP 1.3 spec, Table 3.10 */ - memset(&psr_vsc, 0, sizeof(psr_vsc)); - psr_vsc.sdp_header.HB0 = 0; - psr_vsc.sdp_header.HB1 = 0x7; - psr_vsc.sdp_header.HB2 = 0x2; - psr_vsc.sdp_header.HB3 = 0x8; - } - - intel_dig_port->write_infoframe(&intel_dig_port->base, - crtc_state, - DP_SDP_VSC, &psr_vsc, sizeof(psr_vsc)); -} - static void hsw_psr_setup_aux(struct intel_dp *intel_dp) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); @@ -751,6 +724,8 @@ void intel_psr_compute_config(struct intel_dp *intel_dp, if (intel_dp != dev_priv->psr.dp) return; + if (!psr_global_enabled(dev_priv)) + return; /* * HSW spec explicitly says PSR is tied to port A. * BDW+ platforms have a instance of PSR registers per transcoder but @@ -793,6 +768,7 @@ void intel_psr_compute_config(struct intel_dp *intel_dp, crtc_state->has_psr = true; crtc_state->has_psr2 = intel_psr2_config_valid(intel_dp, crtc_state); + crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_VSC); } static void intel_psr_activate(struct intel_dp *intel_dp) @@ -875,9 +851,12 @@ static void intel_psr_enable_source(struct intel_dp *intel_dp, } static void intel_psr_enable_locked(struct drm_i915_private *dev_priv, - const struct intel_crtc_state *crtc_state) + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { struct intel_dp *intel_dp = dev_priv->psr.dp; + struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp); + struct intel_encoder *encoder = &intel_dig_port->base; u32 val; drm_WARN_ON(&dev_priv->drm, dev_priv->psr.enabled); @@ -916,7 +895,9 @@ static void intel_psr_enable_locked(struct drm_i915_private *dev_priv, drm_dbg_kms(&dev_priv->drm, "Enabling PSR%s\n", dev_priv->psr.psr2_enabled ? "2" : "1"); - intel_psr_setup_vsc(intel_dp, crtc_state); + intel_dp_compute_psr_vsc_sdp(intel_dp, crtc_state, conn_state, + &dev_priv->psr.vsc); + intel_write_dp_vsc_sdp(encoder, crtc_state, &dev_priv->psr.vsc); intel_psr_enable_sink(intel_dp); intel_psr_enable_source(intel_dp, crtc_state); dev_priv->psr.enabled = true; @@ -928,18 +909,22 @@ static void intel_psr_enable_locked(struct drm_i915_private *dev_priv, * intel_psr_enable - Enable PSR * @intel_dp: Intel DP * @crtc_state: new CRTC state + * @conn_state: new CONNECTOR state * * This function can only be called after the pipe is fully trained and enabled. */ void intel_psr_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); - if (!crtc_state->has_psr) + if (!CAN_PSR(dev_priv) || dev_priv->psr.dp != intel_dp) return; - if (drm_WARN_ON(&dev_priv->drm, !CAN_PSR(dev_priv))) + dev_priv->psr.force_mode_changed = false; + + if (!crtc_state->has_psr) return; drm_WARN_ON(&dev_priv->drm, dev_priv->drrs.dp); @@ -951,7 +936,7 @@ void intel_psr_enable(struct intel_dp *intel_dp, goto unlock; } - intel_psr_enable_locked(dev_priv, crtc_state); + intel_psr_enable_locked(dev_priv, crtc_state, conn_state); unlock: mutex_unlock(&dev_priv->psr.lock); @@ -1084,13 +1069,15 @@ static void psr_force_hw_tracking_exit(struct drm_i915_private *dev_priv) * intel_psr_update - Update PSR state * @intel_dp: Intel DP * @crtc_state: new CRTC state + * @conn_state: new CONNECTOR state * * This functions will update PSR states, disabling, enabling or switching PSR * version when executing fastsets. For full modeset, intel_psr_disable() and * intel_psr_enable() should be called instead. */ void intel_psr_update(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state) + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); struct i915_psr *psr = &dev_priv->psr; @@ -1099,6 +1086,8 @@ void intel_psr_update(struct intel_dp *intel_dp, if (!CAN_PSR(dev_priv) || READ_ONCE(psr->dp) != intel_dp) return; + dev_priv->psr.force_mode_changed = false; + mutex_lock(&dev_priv->psr.lock); enable = crtc_state->has_psr && psr_global_enabled(dev_priv); @@ -1125,7 +1114,7 @@ void intel_psr_update(struct intel_dp *intel_dp, intel_psr_disable_locked(intel_dp); if (enable) - intel_psr_enable_locked(dev_priv, crtc_state); + intel_psr_enable_locked(dev_priv, crtc_state, conn_state); unlock: mutex_unlock(&dev_priv->psr.lock); @@ -1629,7 +1618,7 @@ void intel_psr_atomic_check(struct drm_connector *connector, struct drm_crtc_state *crtc_state; if (!CAN_PSR(dev_priv) || !new_state->crtc || - dev_priv->psr.initially_probed) + !dev_priv->psr.force_mode_changed) return; intel_connector = to_intel_connector(connector); @@ -1640,5 +1629,18 @@ void intel_psr_atomic_check(struct drm_connector *connector, crtc_state = drm_atomic_get_new_crtc_state(new_state->state, new_state->crtc); crtc_state->mode_changed = true; - dev_priv->psr.initially_probed = true; +} + +void intel_psr_set_force_mode_changed(struct intel_dp *intel_dp) +{ + struct drm_i915_private *dev_priv; + + if (!intel_dp) + return; + + dev_priv = dp_to_i915(intel_dp); + if (!CAN_PSR(dev_priv) || intel_dp != dev_priv->psr.dp) + return; + + dev_priv->psr.force_mode_changed = true; } diff --git a/drivers/gpu/drm/i915/display/intel_psr.h b/drivers/gpu/drm/i915/display/intel_psr.h index c58a1d438808..b4515186d5f4 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.h +++ b/drivers/gpu/drm/i915/display/intel_psr.h @@ -17,11 +17,13 @@ struct intel_dp; #define CAN_PSR(dev_priv) (HAS_PSR(dev_priv) && dev_priv->psr.sink_support) void intel_psr_init_dpcd(struct intel_dp *intel_dp); void intel_psr_enable(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); void intel_psr_disable(struct intel_dp *intel_dp, const struct intel_crtc_state *old_crtc_state); void intel_psr_update(struct intel_dp *intel_dp, - const struct intel_crtc_state *crtc_state); + const struct intel_crtc_state *crtc_state, + const struct drm_connector_state *conn_state); int intel_psr_debug_set(struct drm_i915_private *dev_priv, u64 value); void intel_psr_invalidate(struct drm_i915_private *dev_priv, unsigned frontbuffer_bits, @@ -40,5 +42,6 @@ bool intel_psr_enabled(struct intel_dp *intel_dp); void intel_psr_atomic_check(struct drm_connector *connector, struct drm_connector_state *old_state, struct drm_connector_state *new_state); +void intel_psr_set_force_mode_changed(struct intel_dp *intel_dp); #endif /* __INTEL_PSR_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c index 9d235d270dac..46beb155d835 100644 --- a/drivers/gpu/drm/i915/display/intel_quirks.c +++ b/drivers/gpu/drm/i915/display/intel_quirks.c @@ -82,6 +82,16 @@ static const struct intel_dmi_quirk intel_dmi_quirks[] = { DMI_MATCH(DMI_PRODUCT_NAME, ""), }, }, + { + .callback = intel_dmi_reverse_brightness, + .ident = "Thundersoft TST178 tablet", + /* DMI strings are too generic, also match on BIOS date */ + .matches = {DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AMI Corporation"), + DMI_EXACT_MATCH(DMI_BOARD_NAME, "Aptio CRB"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "To be filled by O.E.M."), + DMI_EXACT_MATCH(DMI_BIOS_DATE, "04/15/2014"), + }, + }, { } /* terminating entry */ }, .hook = quirk_invert_brightness, diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index a4921b549f8b..bc6c26818e15 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -34,7 +34,6 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_atomic.h" @@ -1431,7 +1430,8 @@ static void intel_sdvo_update_props(struct intel_sdvo *intel_sdvo, #undef UPDATE_PROPERTY } -static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder, +static void intel_sdvo_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *intel_encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { @@ -1728,7 +1728,8 @@ static void intel_sdvo_enable_audio(struct intel_sdvo *intel_sdvo, SDVO_AUDIO_PRESENCE_DETECT); } -static void intel_disable_sdvo(struct intel_encoder *encoder, +static void intel_disable_sdvo(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *conn_state) { @@ -1776,20 +1777,23 @@ static void intel_disable_sdvo(struct intel_encoder *encoder, } } -static void pch_disable_sdvo(struct intel_encoder *encoder, +static void pch_disable_sdvo(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { } -static void pch_post_disable_sdvo(struct intel_encoder *encoder, +static void pch_post_disable_sdvo(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { - intel_disable_sdvo(encoder, old_crtc_state, old_conn_state); + intel_disable_sdvo(state, encoder, old_crtc_state, old_conn_state); } -static void intel_enable_sdvo(struct intel_encoder *encoder, +static void intel_enable_sdvo(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -1935,12 +1939,11 @@ static void intel_sdvo_enable_hotplug(struct intel_encoder *encoder) static enum intel_hotplug_state intel_sdvo_hotplug(struct intel_encoder *encoder, - struct intel_connector *connector, - bool irq_received) + struct intel_connector *connector) { intel_sdvo_enable_hotplug(encoder); - return intel_encoder_hotplug(encoder, connector, irq_received); + return intel_encoder_hotplug(encoder, connector); } static bool @@ -2721,6 +2724,7 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device) * Some SDVO devices have one-shot hotplug interrupts. * Ensure that they get re-enabled when an interrupt happens. */ + intel_connector->polled = DRM_CONNECTOR_POLL_HPD; intel_encoder->hotplug = intel_sdvo_hotplug; intel_sdvo_enable_hotplug(intel_encoder); } else { diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.h b/drivers/gpu/drm/i915/display/intel_sdvo.h index a66f224aa17d..72065e4360d5 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.h +++ b/drivers/gpu/drm/i915/display/intel_sdvo.h @@ -8,8 +8,6 @@ #include <linux/types.h> -#include <drm/i915_drm.h> - #include "i915_reg.h" struct drm_i915_private; diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c index 7abeefe8dce5..0000ec7055f7 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.c +++ b/drivers/gpu/drm/i915/display/intel_sprite.c @@ -37,10 +37,10 @@ #include <drm/drm_fourcc.h> #include <drm/drm_plane_helper.h> #include <drm/drm_rect.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_trace.h" +#include "i915_vgpu.h" #include "intel_atomic_plane.h" #include "intel_display_types.h" #include "intel_frontbuffer.h" @@ -284,6 +284,16 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) bool rotated = drm_rotation_90_or_270(plane_state->hw.rotation); /* + * FIXME hsub/vsub vs. block size is a mess. Pre-tgl CCS + * abuses hsub/vsub so we can't use them here. But as they + * are limited to 32bpp RGB formats we don't actually need + * to check anything. + */ + if (fb->modifier == I915_FORMAT_MOD_Y_TILED_CCS || + fb->modifier == I915_FORMAT_MOD_Yf_TILED_CCS) + return 0; + + /* * Hardware doesn't handle subpixel coordinates. * Adjust to (macro)pixel boundary, but be careful not to * increase the source viewport size, because that could @@ -297,26 +307,26 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state) drm_rect_init(src, src_x << 16, src_y << 16, src_w << 16, src_h << 16); - if (!fb->format->is_yuv) - return 0; - - /* YUV specific checks */ - if (!rotated) { + if (fb->format->format == DRM_FORMAT_RGB565 && rotated) { + hsub = 2; + vsub = 2; + } else { hsub = fb->format->hsub; vsub = fb->format->vsub; - } else { - hsub = vsub = max(fb->format->hsub, fb->format->vsub); } + if (rotated) + hsub = vsub = max(hsub, vsub); + if (src_x % hsub || src_w % hsub) { - DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of %u for %sYUV planes\n", - src_x, src_w, hsub, rotated ? "rotated " : ""); + DRM_DEBUG_KMS("src x/w (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_x, src_w, hsub, yesno(rotated)); return -EINVAL; } if (src_y % vsub || src_h % vsub) { - DRM_DEBUG_KMS("src y/h (%u, %u) must be a multiple of %u for %sYUV planes\n", - src_y, src_h, vsub, rotated ? "rotated " : ""); + DRM_DEBUG_KMS("src y/h (%u, %u) must be a multiple of %u (rotated: %s)\n", + src_y, src_h, vsub, yesno(rotated)); return -EINVAL; } @@ -355,9 +365,8 @@ static int skl_plane_min_cdclk(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { struct drm_i915_private *dev_priv = to_i915(plane_state->uapi.plane->dev); - unsigned int pixel_rate = crtc_state->pixel_rate; - unsigned int src_w, src_h, dst_w, dst_h; unsigned int num, den; + unsigned int pixel_rate = intel_plane_pixel_rate(crtc_state, plane_state); skl_plane_ratio(crtc_state, plane_state, &num, &den); @@ -365,17 +374,7 @@ static int skl_plane_min_cdclk(const struct intel_crtc_state *crtc_state, if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) den *= 2; - src_w = drm_rect_width(&plane_state->uapi.src) >> 16; - src_h = drm_rect_height(&plane_state->uapi.src) >> 16; - dst_w = drm_rect_width(&plane_state->uapi.dst); - dst_h = drm_rect_height(&plane_state->uapi.dst); - - /* Downscaling limits the maximum pixel rate */ - dst_w = min(src_w, dst_w); - dst_h = min(src_h, dst_h); - - return DIV64_U64_ROUND_UP(mul_u32_u32(pixel_rate * num, src_w * src_h), - mul_u32_u32(den, dst_w * dst_h)); + return DIV_ROUND_UP(pixel_rate * num, den); } static unsigned int @@ -2077,6 +2076,18 @@ vlv_sprite_check(struct intel_crtc_state *crtc_state, return 0; } +static bool intel_format_is_p01x(u32 format) +{ + switch (format) { + case DRM_FORMAT_P010: + case DRM_FORMAT_P012: + case DRM_FORMAT_P016: + return true; + default: + return false; + } +} + static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state) { @@ -2155,6 +2166,15 @@ static int skl_plane_check_fb(const struct intel_crtc_state *crtc_state, return -EINVAL; } + /* Wa_1606054188:tgl */ + if (IS_TIGERLAKE(dev_priv) && + plane_state->ckey.flags & I915_SET_COLORKEY_SOURCE && + intel_format_is_p01x(fb->format->format)) { + drm_dbg_kms(&dev_priv->drm, + "Source color keying not supported with P01x formats\n"); + return -EINVAL; + } + return 0; } @@ -2483,6 +2503,7 @@ static const u32 skl_plane_formats[] = { DRM_FORMAT_YVYU, DRM_FORMAT_UYVY, DRM_FORMAT_VYUY, + DRM_FORMAT_XYUV8888, }; static const u32 skl_planar_formats[] = { @@ -2501,6 +2522,7 @@ static const u32 skl_planar_formats[] = { DRM_FORMAT_UYVY, DRM_FORMAT_VYUY, DRM_FORMAT_NV12, + DRM_FORMAT_XYUV8888, }; static const u32 glk_planar_formats[] = { @@ -2519,6 +2541,7 @@ static const u32 glk_planar_formats[] = { DRM_FORMAT_UYVY, DRM_FORMAT_VYUY, DRM_FORMAT_NV12, + DRM_FORMAT_XYUV8888, DRM_FORMAT_P010, DRM_FORMAT_P012, DRM_FORMAT_P016, @@ -2542,6 +2565,7 @@ static const u32 icl_sdr_y_plane_formats[] = { DRM_FORMAT_Y210, DRM_FORMAT_Y212, DRM_FORMAT_Y216, + DRM_FORMAT_XYUV8888, DRM_FORMAT_XVYU2101010, DRM_FORMAT_XVYU12_16161616, DRM_FORMAT_XVYU16161616, @@ -2569,6 +2593,7 @@ static const u32 icl_sdr_uv_plane_formats[] = { DRM_FORMAT_Y210, DRM_FORMAT_Y212, DRM_FORMAT_Y216, + DRM_FORMAT_XYUV8888, DRM_FORMAT_XVYU2101010, DRM_FORMAT_XVYU12_16161616, DRM_FORMAT_XVYU16161616, @@ -2600,6 +2625,7 @@ static const u32 icl_hdr_plane_formats[] = { DRM_FORMAT_Y210, DRM_FORMAT_Y212, DRM_FORMAT_Y216, + DRM_FORMAT_XYUV8888, DRM_FORMAT_XVYU2101010, DRM_FORMAT_XVYU12_16161616, DRM_FORMAT_XVYU16161616, @@ -2770,6 +2796,7 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane, case DRM_FORMAT_UYVY: case DRM_FORMAT_VYUY: case DRM_FORMAT_NV12: + case DRM_FORMAT_XYUV8888: case DRM_FORMAT_P010: case DRM_FORMAT_P012: case DRM_FORMAT_P016: @@ -2797,19 +2824,25 @@ static bool skl_plane_format_mod_supported(struct drm_plane *_plane, } } -static bool gen12_plane_supports_mc_ccs(enum plane_id plane_id) +static bool gen12_plane_supports_mc_ccs(struct drm_i915_private *dev_priv, + enum plane_id plane_id) { + /* Wa_14010477008:tgl[a0..c0] */ + if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_C0)) + return false; + return plane_id < PLANE_SPRITE4; } static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, u32 format, u64 modifier) { + struct drm_i915_private *dev_priv = to_i915(_plane->dev); struct intel_plane *plane = to_intel_plane(_plane); switch (modifier) { case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: - if (!gen12_plane_supports_mc_ccs(plane->id)) + if (!gen12_plane_supports_mc_ccs(dev_priv, plane->id)) return false; /* fall through */ case DRM_FORMAT_MOD_LINEAR: @@ -2834,6 +2867,7 @@ static bool gen12_plane_format_mod_supported(struct drm_plane *_plane, case DRM_FORMAT_UYVY: case DRM_FORMAT_VYUY: case DRM_FORMAT_NV12: + case DRM_FORMAT_XYUV8888: case DRM_FORMAT_P010: case DRM_FORMAT_P012: case DRM_FORMAT_P016: @@ -2978,9 +3012,10 @@ static const u32 *icl_get_plane_formats(struct drm_i915_private *dev_priv, } } -static const u64 *gen12_get_plane_modifiers(enum plane_id plane_id) +static const u64 *gen12_get_plane_modifiers(struct drm_i915_private *dev_priv, + enum plane_id plane_id) { - if (gen12_plane_supports_mc_ccs(plane_id)) + if (gen12_plane_supports_mc_ccs(dev_priv, plane_id)) return gen12_plane_format_modifiers_mc_ccs; else return gen12_plane_format_modifiers_rc_ccs; @@ -3011,7 +3046,6 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv, struct intel_plane *plane; enum drm_plane_type plane_type; unsigned int supported_rotations; - unsigned int possible_crtcs; const u64 *modifiers; const u32 *formats; int num_formats; @@ -3051,7 +3085,7 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv, plane->has_ccs = skl_plane_has_ccs(dev_priv, pipe, plane_id); if (INTEL_GEN(dev_priv) >= 12) { - modifiers = gen12_get_plane_modifiers(plane_id); + modifiers = gen12_get_plane_modifiers(dev_priv, plane_id); plane_funcs = &gen12_plane_funcs; } else { if (plane->has_ccs) @@ -3066,10 +3100,8 @@ skl_universal_plane_create(struct drm_i915_private *dev_priv, else plane_type = DRM_PLANE_TYPE_OVERLAY; - possible_crtcs = BIT(pipe); - ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, - possible_crtcs, plane_funcs, + 0, plane_funcs, formats, num_formats, modifiers, plane_type, "plane %d%c", plane_id + 1, @@ -3120,7 +3152,6 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv, { struct intel_plane *plane; const struct drm_plane_funcs *plane_funcs; - unsigned long possible_crtcs; unsigned int supported_rotations; const u64 *modifiers; const u32 *formats; @@ -3205,10 +3236,8 @@ intel_sprite_plane_create(struct drm_i915_private *dev_priv, plane->id = PLANE_SPRITE0 + sprite; plane->frontbuffer_bit = INTEL_FRONTBUFFER(pipe, plane->id); - possible_crtcs = BIT(pipe); - ret = drm_universal_plane_init(&dev_priv->drm, &plane->base, - possible_crtcs, plane_funcs, + 0, plane_funcs, formats, num_formats, modifiers, DRM_PLANE_TYPE_OVERLAY, "sprite %c", sprite_name(pipe, sprite)); diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c index 9b850c11aa78..b161c15baf86 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.c +++ b/drivers/gpu/drm/i915/display/intel_tc.c @@ -34,6 +34,7 @@ tc_port_load_fia_params(struct drm_i915_private *i915, if (INTEL_INFO(i915)->display.has_modular_fia) { modular_fia = intel_uncore_read(&i915->uncore, PORT_TX_DFLEXDPSP(FIA1)); + drm_WARN_ON(&i915->drm, modular_fia == 0xffffffff); modular_fia &= MODULAR_FIA_MASK; } else { modular_fia = 0; @@ -52,6 +53,62 @@ tc_port_load_fia_params(struct drm_i915_private *i915, } } +static enum intel_display_power_domain +tc_cold_get_power_domain(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + + if (INTEL_GEN(i915) == 11) + return intel_legacy_aux_to_power_domain(dig_port->aux_ch); + else + return POWER_DOMAIN_TC_COLD_OFF; +} + +static intel_wakeref_t +tc_cold_block(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + enum intel_display_power_domain domain; + + if (INTEL_GEN(i915) == 11 && !dig_port->tc_legacy_port) + return 0; + + domain = tc_cold_get_power_domain(dig_port); + return intel_display_power_get(i915, domain); +} + +static void +tc_cold_unblock(struct intel_digital_port *dig_port, intel_wakeref_t wakeref) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + enum intel_display_power_domain domain; + + /* + * wakeref == -1, means some error happened saving save_depot_stack but + * power should still be put down and 0 is a invalid save_depot_stack + * id so can be used to skip it for non TC legacy ports. + */ + if (wakeref == 0) + return; + + domain = tc_cold_get_power_domain(dig_port); + intel_display_power_put_async(i915, domain, wakeref); +} + +static void +assert_tc_cold_blocked(struct intel_digital_port *dig_port) +{ + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + bool enabled; + + if (INTEL_GEN(i915) == 11 && !dig_port->tc_legacy_port) + return; + + enabled = intel_display_power_is_enabled(i915, + tc_cold_get_power_domain(dig_port)); + drm_WARN_ON(&i915->drm, !enabled); +} + u32 intel_tc_port_get_lane_mask(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); @@ -62,6 +119,7 @@ u32 intel_tc_port_get_lane_mask(struct intel_digital_port *dig_port) PORT_TX_DFLEXDPSP(dig_port->tc_phy_fia)); drm_WARN_ON(&i915->drm, lane_mask == 0xffffffff); + assert_tc_cold_blocked(dig_port); lane_mask &= DP_LANE_ASSIGNMENT_MASK(dig_port->tc_phy_fia_idx); return lane_mask >> DP_LANE_ASSIGNMENT_SHIFT(dig_port->tc_phy_fia_idx); @@ -77,6 +135,7 @@ u32 intel_tc_port_get_pin_assignment_mask(struct intel_digital_port *dig_port) PORT_TX_DFLEXPA1(dig_port->tc_phy_fia)); drm_WARN_ON(&i915->drm, pin_mask == 0xffffffff); + assert_tc_cold_blocked(dig_port); return (pin_mask & DP_PIN_ASSIGNMENT_MASK(dig_port->tc_phy_fia_idx)) >> DP_PIN_ASSIGNMENT_SHIFT(dig_port->tc_phy_fia_idx); @@ -91,6 +150,8 @@ int intel_tc_port_fia_max_lane_count(struct intel_digital_port *dig_port) if (dig_port->tc_mode != TC_PORT_DP_ALT) return 4; + assert_tc_cold_blocked(dig_port); + lane_mask = 0; with_intel_display_power(i915, POWER_DOMAIN_DISPLAY_CORE, wakeref) lane_mask = intel_tc_port_get_lane_mask(dig_port); @@ -123,6 +184,8 @@ void intel_tc_port_set_fia_lane_count(struct intel_digital_port *dig_port, drm_WARN_ON(&i915->drm, lane_reversal && dig_port->tc_mode != TC_PORT_LEGACY); + assert_tc_cold_blocked(dig_port); + val = intel_uncore_read(uncore, PORT_TX_DFLEXDPMLE1(dig_port->tc_phy_fia)); val &= ~DFLEXDPMLE1_DPMLETC_MASK(dig_port->tc_phy_fia_idx); @@ -152,6 +215,7 @@ void intel_tc_port_set_fia_lane_count(struct intel_digital_port *dig_port, static void tc_port_fixup_legacy_flag(struct intel_digital_port *dig_port, u32 live_status_mask) { + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); u32 valid_hpd_mask; if (dig_port->tc_legacy_port) @@ -164,8 +228,9 @@ static void tc_port_fixup_legacy_flag(struct intel_digital_port *dig_port, return; /* If live status mismatches the VBT flag, trust the live status. */ - DRM_ERROR("Port %s: live status %08x mismatches the legacy port flag, fix flag\n", - dig_port->tc_port_name, live_status_mask); + drm_err(&i915->drm, + "Port %s: live status %08x mismatches the legacy port flag, fix flag\n", + dig_port->tc_port_name, live_status_mask); dig_port->tc_legacy_port = !dig_port->tc_legacy_port; } @@ -173,8 +238,8 @@ static void tc_port_fixup_legacy_flag(struct intel_digital_port *dig_port, static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port) { struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); - enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port); struct intel_uncore *uncore = &i915->uncore; + u32 isr_bit = i915->hotplug.pch_hpd[dig_port->base.hpd_pin]; u32 mask = 0; u32 val; @@ -193,7 +258,7 @@ static u32 tc_port_live_status_mask(struct intel_digital_port *dig_port) if (val & TC_LIVE_STATE_TC(dig_port->tc_phy_fia_idx)) mask |= BIT(TC_PORT_DP_ALT); - if (intel_uncore_read(uncore, SDEISR) & SDE_TC_HOTPLUG_ICP(tc_port)) + if (intel_uncore_read(uncore, SDEISR) & isr_bit) mask |= BIT(TC_PORT_LEGACY); /* The sink can be connected only in a single mode. */ @@ -233,8 +298,7 @@ static bool icl_tc_phy_set_safe_mode(struct intel_digital_port *dig_port, if (val == 0xffffffff) { drm_dbg_kms(&i915->drm, "Port %s: PHY in TCCOLD, can't set safe-mode to %s\n", - dig_port->tc_port_name, - enableddisabled(enable)); + dig_port->tc_port_name, enableddisabled(enable)); return false; } @@ -286,11 +350,12 @@ static bool icl_tc_phy_is_in_safe_mode(struct intel_digital_port *dig_port) static void icl_tc_phy_connect(struct intel_digital_port *dig_port, int required_lanes) { + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); int max_lanes; if (!icl_tc_phy_status_complete(dig_port)) { - DRM_DEBUG_KMS("Port %s: PHY not ready\n", - dig_port->tc_port_name); + drm_dbg_kms(&i915->drm, "Port %s: PHY not ready\n", + dig_port->tc_port_name); goto out_set_tbt_alt_mode; } @@ -311,15 +376,16 @@ static void icl_tc_phy_connect(struct intel_digital_port *dig_port, * became disconnected. Not necessary for legacy mode. */ if (!(tc_port_live_status_mask(dig_port) & BIT(TC_PORT_DP_ALT))) { - DRM_DEBUG_KMS("Port %s: PHY sudden disconnect\n", - dig_port->tc_port_name); + drm_dbg_kms(&i915->drm, "Port %s: PHY sudden disconnect\n", + dig_port->tc_port_name); goto out_set_safe_mode; } if (max_lanes < required_lanes) { - DRM_DEBUG_KMS("Port %s: PHY max lanes %d < required lanes %d\n", - dig_port->tc_port_name, - max_lanes, required_lanes); + drm_dbg_kms(&i915->drm, + "Port %s: PHY max lanes %d < required lanes %d\n", + dig_port->tc_port_name, + max_lanes, required_lanes); goto out_set_safe_mode; } @@ -357,15 +423,17 @@ static void icl_tc_phy_disconnect(struct intel_digital_port *dig_port) static bool icl_tc_phy_is_connected(struct intel_digital_port *dig_port) { + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); + if (!icl_tc_phy_status_complete(dig_port)) { - DRM_DEBUG_KMS("Port %s: PHY status not complete\n", - dig_port->tc_port_name); + drm_dbg_kms(&i915->drm, "Port %s: PHY status not complete\n", + dig_port->tc_port_name); return dig_port->tc_mode == TC_PORT_TBT_ALT; } if (icl_tc_phy_is_in_safe_mode(dig_port)) { - DRM_DEBUG_KMS("Port %s: PHY still in safe mode\n", - dig_port->tc_port_name); + drm_dbg_kms(&i915->drm, "Port %s: PHY still in safe mode\n", + dig_port->tc_port_name); return false; } @@ -415,9 +483,14 @@ static void intel_tc_port_reset_mode(struct intel_digital_port *dig_port, enum tc_port_mode old_tc_mode = dig_port->tc_mode; intel_display_power_flush_work(i915); - drm_WARN_ON(&i915->drm, - intel_display_power_is_enabled(i915, - intel_aux_power_domain(dig_port))); + if (INTEL_GEN(i915) != 11 || !dig_port->tc_legacy_port) { + enum intel_display_power_domain aux_domain; + bool aux_powered; + + aux_domain = intel_aux_power_domain(dig_port); + aux_powered = intel_display_power_is_enabled(i915, aux_domain); + drm_WARN_ON(&i915->drm, aux_powered); + } icl_tc_phy_disconnect(dig_port); icl_tc_phy_connect(dig_port, required_lanes); @@ -438,10 +511,13 @@ intel_tc_port_link_init_refcount(struct intel_digital_port *dig_port, void intel_tc_port_sanitize(struct intel_digital_port *dig_port) { + struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev); struct intel_encoder *encoder = &dig_port->base; + intel_wakeref_t tc_cold_wref; int active_links = 0; mutex_lock(&dig_port->tc_lock); + tc_cold_wref = tc_cold_block(dig_port); dig_port->tc_mode = intel_tc_port_get_current_mode(dig_port); if (dig_port->dp.is_mst) @@ -451,8 +527,9 @@ void intel_tc_port_sanitize(struct intel_digital_port *dig_port) if (active_links) { if (!icl_tc_phy_is_connected(dig_port)) - DRM_DEBUG_KMS("Port %s: PHY disconnected with %d active link(s)\n", - dig_port->tc_port_name, active_links); + drm_dbg_kms(&i915->drm, + "Port %s: PHY disconnected with %d active link(s)\n", + dig_port->tc_port_name, active_links); intel_tc_port_link_init_refcount(dig_port, active_links); goto out; @@ -462,10 +539,11 @@ void intel_tc_port_sanitize(struct intel_digital_port *dig_port) icl_tc_phy_connect(dig_port, 1); out: - DRM_DEBUG_KMS("Port %s: sanitize mode (%s)\n", - dig_port->tc_port_name, - tc_port_mode_name(dig_port->tc_mode)); + drm_dbg_kms(&i915->drm, "Port %s: sanitize mode (%s)\n", + dig_port->tc_port_name, + tc_port_mode_name(dig_port->tc_mode)); + tc_cold_unblock(dig_port, tc_cold_wref); mutex_unlock(&dig_port->tc_lock); } @@ -484,13 +562,19 @@ static bool intel_tc_port_needs_reset(struct intel_digital_port *dig_port) * connected ports are usable, and avoids exposing to the users objects they * can't really use. */ -bool intel_tc_port_connected(struct intel_digital_port *dig_port) +bool intel_tc_port_connected(struct intel_encoder *encoder) { + struct intel_digital_port *dig_port = enc_to_dig_port(encoder); bool is_connected; + intel_wakeref_t tc_cold_wref; intel_tc_port_lock(dig_port); + tc_cold_wref = tc_cold_block(dig_port); + is_connected = tc_port_live_status_mask(dig_port) & BIT(dig_port->tc_mode); + + tc_cold_unblock(dig_port, tc_cold_wref); intel_tc_port_unlock(dig_port); return is_connected; @@ -506,9 +590,16 @@ static void __intel_tc_port_lock(struct intel_digital_port *dig_port, mutex_lock(&dig_port->tc_lock); - if (!dig_port->tc_link_refcount && - intel_tc_port_needs_reset(dig_port)) - intel_tc_port_reset_mode(dig_port, required_lanes); + if (!dig_port->tc_link_refcount) { + intel_wakeref_t tc_cold_wref; + + tc_cold_wref = tc_cold_block(dig_port); + + if (intel_tc_port_needs_reset(dig_port)) + intel_tc_port_reset_mode(dig_port, required_lanes); + + tc_cold_unblock(dig_port, tc_cold_wref); + } drm_WARN_ON(&i915->drm, dig_port->tc_lock_wakeref); dig_port->tc_lock_wakeref = wakeref; diff --git a/drivers/gpu/drm/i915/display/intel_tc.h b/drivers/gpu/drm/i915/display/intel_tc.h index 463f1b3c836f..b619e4736f85 100644 --- a/drivers/gpu/drm/i915/display/intel_tc.h +++ b/drivers/gpu/drm/i915/display/intel_tc.h @@ -10,8 +10,9 @@ #include <linux/types.h> struct intel_digital_port; +struct intel_encoder; -bool intel_tc_port_connected(struct intel_digital_port *dig_port); +bool intel_tc_port_connected(struct intel_encoder *encoder); u32 intel_tc_port_get_lane_mask(struct intel_digital_port *dig_port); u32 intel_tc_port_get_pin_assignment_mask(struct intel_digital_port *dig_port); int intel_tc_port_fia_max_lane_count(struct intel_digital_port *dig_port); diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c index 4f81ee26b7ab..fbe12aad7d58 100644 --- a/drivers/gpu/drm/i915/display/intel_tv.c +++ b/drivers/gpu/drm/i915/display/intel_tv.c @@ -33,7 +33,6 @@ #include <drm/drm_atomic_helper.h> #include <drm/drm_crtc.h> #include <drm/drm_edid.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "intel_connector.h" @@ -915,7 +914,8 @@ intel_tv_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe) } static void -intel_enable_tv(struct intel_encoder *encoder, +intel_enable_tv(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -931,7 +931,8 @@ intel_enable_tv(struct intel_encoder *encoder, } static void -intel_disable_tv(struct intel_encoder *encoder, +intel_disable_tv(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -1415,7 +1416,8 @@ static void set_color_conversion(struct drm_i915_private *dev_priv, (color_conversion->bv << 16) | color_conversion->av); } -static void intel_tv_pre_enable(struct intel_encoder *encoder, +static void intel_tv_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -1699,13 +1701,13 @@ intel_tv_detect(struct drm_connector *connector, struct drm_modeset_acquire_ctx *ctx, bool force) { + struct drm_i915_private *i915 = to_i915(connector->dev); struct intel_tv *intel_tv = intel_attached_tv(to_intel_connector(connector)); enum drm_connector_status status; int type; - DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force=%d\n", - connector->base.id, connector->name, - force); + drm_dbg_kms(&i915->drm, "[CONNECTOR:%d:%s] force=%d\n", + connector->base.id, connector->name, force); if (force) { struct intel_load_detect_pipe tmp; diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h index 05c7cbe32eb4..aef7fe932d1a 100644 --- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h +++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h @@ -462,7 +462,7 @@ struct bdb_general_definitions { * number = (block_size - sizeof(bdb_general_definitions))/ * defs->child_dev_size; */ - u8 devices[0]; + u8 devices[]; } __packed; /* @@ -839,7 +839,7 @@ struct bdb_mipi_config { struct bdb_mipi_sequence { u8 version; - u8 data[0]; /* up to 6 variable length blocks */ + u8 data[]; /* up to 6 variable length blocks */ } __packed; /* diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 7cba57ae72fe..95ad87d4ccb3 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -6,8 +6,6 @@ * Manasi Navare <manasi.d.navare@intel.com> */ -#include <drm/i915_drm.h> - #include "i915_drv.h" #include "intel_display_types.h" #include "intel_dsi.h" diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c index b9f2e3ce5185..f582ab52f0b0 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi.c +++ b/drivers/gpu/drm/i915/display/vlv_dsi.c @@ -267,7 +267,6 @@ static int intel_dsi_compute_config(struct intel_encoder *encoder, struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi, base); struct intel_connector *intel_connector = intel_dsi->attached_connector; - struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc); const struct drm_display_mode *fixed_mode = intel_connector->panel.fixed_mode; struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode; int ret; @@ -279,11 +278,11 @@ static int intel_dsi_compute_config(struct intel_encoder *encoder, intel_fixed_panel_mode(fixed_mode, adjusted_mode); if (HAS_GMCH(dev_priv)) - intel_gmch_panel_fitting(crtc, pipe_config, - conn_state->scaling_mode); + ret = intel_gmch_panel_fitting(pipe_config, conn_state); else - intel_pch_panel_fitting(crtc, pipe_config, - conn_state->scaling_mode); + ret = intel_pch_panel_fitting(pipe_config, conn_state); + if (ret) + return ret; } if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN) @@ -759,7 +758,8 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder); * DSI port enable has to be done before pipe and plane enable, so we do it in * the pre_enable hook instead of the enable hook. */ -static void intel_dsi_pre_enable(struct intel_encoder *encoder, +static void intel_dsi_pre_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *pipe_config, const struct drm_connector_state *conn_state) { @@ -858,11 +858,12 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder, intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_ON); } -static void bxt_dsi_enable(struct intel_encoder *encoder, +static void bxt_dsi_enable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, const struct drm_connector_state *conn_state) { - WARN_ON(crtc_state->has_pch_encoder); + drm_WARN_ON(state->base.dev, crtc_state->has_pch_encoder); intel_crtc_vblank_on(crtc_state); } @@ -871,14 +872,16 @@ static void bxt_dsi_enable(struct intel_encoder *encoder, * DSI port disable has to be done after pipe and plane disable, so we do it in * the post_disable hook. */ -static void intel_dsi_disable(struct intel_encoder *encoder, +static void intel_dsi_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { + struct drm_i915_private *i915 = to_i915(encoder->base.dev); struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder); enum port port; - DRM_DEBUG_KMS("\n"); + drm_dbg_kms(&i915->drm, "\n"); intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_BACKLIGHT_OFF); intel_panel_disable_backlight(old_conn_state); @@ -906,7 +909,8 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder) vlv_dsi_clear_device_ready(encoder); } -static void intel_dsi_post_disable(struct intel_encoder *encoder, +static void intel_dsi_post_disable(struct intel_atomic_state *state, + struct intel_encoder *encoder, const struct intel_crtc_state *old_crtc_state, const struct drm_connector_state *old_conn_state) { @@ -1591,59 +1595,6 @@ static const struct drm_connector_funcs intel_dsi_connector_funcs = { .atomic_duplicate_state = intel_digital_connector_duplicate_state, }; -static enum drm_panel_orientation -vlv_dsi_get_hw_panel_orientation(struct intel_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - struct intel_encoder *encoder = intel_attached_encoder(connector); - enum intel_display_power_domain power_domain; - enum drm_panel_orientation orientation; - struct intel_plane *plane; - struct intel_crtc *crtc; - intel_wakeref_t wakeref; - enum pipe pipe; - u32 val; - - if (!encoder->get_hw_state(encoder, &pipe)) - return DRM_MODE_PANEL_ORIENTATION_UNKNOWN; - - crtc = intel_get_crtc_for_pipe(dev_priv, pipe); - plane = to_intel_plane(crtc->base.primary); - - power_domain = POWER_DOMAIN_PIPE(pipe); - wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain); - if (!wakeref) - return DRM_MODE_PANEL_ORIENTATION_UNKNOWN; - - val = intel_de_read(dev_priv, DSPCNTR(plane->i9xx_plane)); - - if (!(val & DISPLAY_PLANE_ENABLE)) - orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN; - else if (val & DISPPLANE_ROTATE_180) - orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP; - else - orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL; - - intel_display_power_put(dev_priv, power_domain, wakeref); - - return orientation; -} - -static enum drm_panel_orientation -vlv_dsi_get_panel_orientation(struct intel_connector *connector) -{ - struct drm_i915_private *dev_priv = to_i915(connector->base.dev); - enum drm_panel_orientation orientation; - - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) { - orientation = vlv_dsi_get_hw_panel_orientation(connector); - if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN) - return orientation; - } - - return intel_dsi_get_panel_orientation(connector); -} - static void vlv_dsi_add_properties(struct intel_connector *connector) { struct drm_i915_private *dev_priv = to_i915(connector->base.dev); @@ -1660,10 +1611,9 @@ static void vlv_dsi_add_properties(struct intel_connector *connector) connector->base.state->scaling_mode = DRM_MODE_SCALE_ASPECT; - connector->base.display_info.panel_orientation = - vlv_dsi_get_panel_orientation(connector); - drm_connector_init_panel_orientation_property( + drm_connector_set_panel_orientation_with_quirk( &connector->base, + intel_dsi_get_panel_orientation(connector), connector->panel.fixed_mode->hdisplay, connector->panel.fixed_mode->vdisplay); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c index 34be4c0ee7c5..bc0223716906 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_clflush.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_clflush.c @@ -108,7 +108,7 @@ bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, if (clflush) { i915_sw_fence_await_reservation(&clflush->base.chain, obj->base.resv, NULL, true, - I915_FENCE_TIMEOUT, + i915_fence_timeout(to_i915(obj->base.dev)), I915_FENCE_GFP); dma_resv_add_excl_fence(obj->base.resv, &clflush->base.dma); dma_fence_work_commit(&clflush->base); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c index 81366aa4812b..d3a86a4d5c04 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_client_blt.c @@ -6,7 +6,6 @@ #include "i915_drv.h" #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" -#include "gt/intel_engine_pool.h" #include "i915_gem_client_blt.h" #include "i915_gem_object_blt.h" @@ -217,7 +216,7 @@ static void clear_pages_worker(struct work_struct *work) 0); out_request: if (unlikely(err)) { - i915_request_skip(rq, err); + i915_request_set_error_once(rq, err); err = 0; } @@ -289,8 +288,7 @@ int i915_gem_schedule_fill_pages_blt(struct drm_i915_gem_object *obj, i915_gem_object_lock(obj); err = i915_sw_fence_await_reservation(&work->wait, - obj->base.resv, NULL, - true, I915_FENCE_TIMEOUT, + obj->base.resv, NULL, true, 0, I915_FENCE_GFP); if (err < 0) { dma_fence_set_error(&work->dma, err); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 3e82739bdbc0..900ea8b7fc8f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -67,10 +67,9 @@ #include <linux/log2.h> #include <linux/nospec.h> -#include <drm/i915_drm.h> - #include "gt/gen6_ppgtt.h" #include "gt/intel_context.h" +#include "gt/intel_context_param.h" #include "gt/intel_engine_heartbeat.h" #include "gt/intel_engine_user.h" #include "gt/intel_ring.h" @@ -131,9 +130,7 @@ static void lut_close(struct i915_gem_context *ctx) if (&lut->obj_link != &obj->lut_list) { i915_lut_handle_free(lut); radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); - if (atomic_dec_and_test(&vma->open_count) && - !i915_vma_is_ggtt(vma)) - i915_vma_close(vma); + i915_vma_close(vma); i915_gem_object_put(obj); } @@ -243,7 +240,6 @@ static void __free_engines(struct i915_gem_engines *e, unsigned int count) if (!e->engines[count]) continue; - RCU_INIT_POINTER(e->engines[count]->gem_context, NULL); intel_context_put(e->engines[count]); } kfree(e); @@ -256,7 +252,51 @@ static void free_engines(struct i915_gem_engines *e) static void free_engines_rcu(struct rcu_head *rcu) { - free_engines(container_of(rcu, struct i915_gem_engines, rcu)); + struct i915_gem_engines *engines = + container_of(rcu, struct i915_gem_engines, rcu); + + i915_sw_fence_fini(&engines->fence); + free_engines(engines); +} + +static int __i915_sw_fence_call +engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) +{ + struct i915_gem_engines *engines = + container_of(fence, typeof(*engines), fence); + + switch (state) { + case FENCE_COMPLETE: + if (!list_empty(&engines->link)) { + struct i915_gem_context *ctx = engines->ctx; + unsigned long flags; + + spin_lock_irqsave(&ctx->stale.lock, flags); + list_del(&engines->link); + spin_unlock_irqrestore(&ctx->stale.lock, flags); + } + i915_gem_context_put(engines->ctx); + break; + + case FENCE_FREE: + init_rcu_head(&engines->rcu); + call_rcu(&engines->rcu, free_engines_rcu); + break; + } + + return NOTIFY_DONE; +} + +static struct i915_gem_engines *alloc_engines(unsigned int count) +{ + struct i915_gem_engines *e; + + e = kzalloc(struct_size(e, engines, count), GFP_KERNEL); + if (!e) + return NULL; + + i915_sw_fence_init(&e->fence, engines_notify); + return e; } static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) @@ -266,12 +306,10 @@ static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) struct i915_gem_engines *e; enum intel_engine_id id; - e = kzalloc(struct_size(e, engines, I915_NUM_ENGINES), GFP_KERNEL); + e = alloc_engines(I915_NUM_ENGINES); if (!e) return ERR_PTR(-ENOMEM); - e->ctx = ctx; - for_each_engine(engine, gt, id) { struct intel_context *ce; @@ -305,7 +343,6 @@ static void i915_gem_context_free(struct i915_gem_context *ctx) list_del(&ctx->link); spin_unlock(&ctx->i915->gem.contexts.lock); - free_engines(rcu_access_pointer(ctx->engines)); mutex_destroy(&ctx->engines_mutex); if (ctx->timeline) @@ -492,30 +529,71 @@ static void kill_engines(struct i915_gem_engines *engines) static void kill_stale_engines(struct i915_gem_context *ctx) { struct i915_gem_engines *pos, *next; - unsigned long flags; - spin_lock_irqsave(&ctx->stale.lock, flags); + spin_lock_irq(&ctx->stale.lock); + GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) { - if (!i915_sw_fence_await(&pos->fence)) + if (!i915_sw_fence_await(&pos->fence)) { + list_del_init(&pos->link); continue; + } - spin_unlock_irqrestore(&ctx->stale.lock, flags); + spin_unlock_irq(&ctx->stale.lock); kill_engines(pos); - spin_lock_irqsave(&ctx->stale.lock, flags); + spin_lock_irq(&ctx->stale.lock); + GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence)); list_safe_reset_next(pos, next, link); list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */ i915_sw_fence_complete(&pos->fence); } - spin_unlock_irqrestore(&ctx->stale.lock, flags); + spin_unlock_irq(&ctx->stale.lock); } static void kill_context(struct i915_gem_context *ctx) { kill_stale_engines(ctx); - kill_engines(__context_engines_static(ctx)); +} + +static void engines_idle_release(struct i915_gem_context *ctx, + struct i915_gem_engines *engines) +{ + struct i915_gem_engines_iter it; + struct intel_context *ce; + + INIT_LIST_HEAD(&engines->link); + + engines->ctx = i915_gem_context_get(ctx); + + for_each_gem_engine(ce, engines, it) { + int err; + + /* serialises with execbuf */ + set_bit(CONTEXT_CLOSED_BIT, &ce->flags); + if (!intel_context_pin_if_active(ce)) + continue; + + /* Wait until context is finally scheduled out and retired */ + err = i915_sw_fence_await_active(&engines->fence, + &ce->active, + I915_ACTIVE_AWAIT_BARRIER); + intel_context_unpin(ce); + if (err) + goto kill; + } + + spin_lock_irq(&ctx->stale.lock); + if (!i915_gem_context_is_closed(ctx)) + list_add_tail(&engines->link, &ctx->stale.engines); + spin_unlock_irq(&ctx->stale.lock); + +kill: + if (list_empty(&engines->link)) /* raced, already closed */ + kill_engines(engines); + + i915_sw_fence_commit(&engines->fence); } static void set_closed_name(struct i915_gem_context *ctx) @@ -539,11 +617,16 @@ static void context_close(struct i915_gem_context *ctx) { struct i915_address_space *vm; + /* Flush any concurrent set_engines() */ + mutex_lock(&ctx->engines_mutex); + engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1)); i915_gem_context_set_closed(ctx); - set_closed_name(ctx); + mutex_unlock(&ctx->engines_mutex); mutex_lock(&ctx->mutex); + set_closed_name(ctx); + vm = i915_gem_context_vm(ctx); if (vm) i915_vm_close(vm); @@ -668,33 +751,67 @@ err_free: return ERR_PTR(err); } -static void +static inline struct i915_gem_engines * +__context_engines_await(const struct i915_gem_context *ctx) +{ + struct i915_gem_engines *engines; + + rcu_read_lock(); + do { + engines = rcu_dereference(ctx->engines); + GEM_BUG_ON(!engines); + + if (unlikely(!i915_sw_fence_await(&engines->fence))) + continue; + + if (likely(engines == rcu_access_pointer(ctx->engines))) + break; + + i915_sw_fence_complete(&engines->fence); + } while (1); + rcu_read_unlock(); + + return engines; +} + +static int context_apply_all(struct i915_gem_context *ctx, - void (*fn)(struct intel_context *ce, void *data), + int (*fn)(struct intel_context *ce, void *data), void *data) { struct i915_gem_engines_iter it; + struct i915_gem_engines *e; struct intel_context *ce; + int err = 0; + + e = __context_engines_await(ctx); + for_each_gem_engine(ce, e, it) { + err = fn(ce, data); + if (err) + break; + } + i915_sw_fence_complete(&e->fence); - for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) - fn(ce, data); - i915_gem_context_unlock_engines(ctx); + return err; } -static void __apply_ppgtt(struct intel_context *ce, void *vm) +static int __apply_ppgtt(struct intel_context *ce, void *vm) { i915_vm_put(ce->vm); ce->vm = i915_vm_get(vm); + return 0; } static struct i915_address_space * __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) { - struct i915_address_space *old = i915_gem_context_vm(ctx); + struct i915_address_space *old; + old = rcu_replace_pointer(ctx->vm, + i915_vm_open(vm), + lockdep_is_held(&ctx->mutex)); GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); - rcu_assign_pointer(ctx->vm, i915_vm_open(vm)); context_apply_all(ctx, __apply_ppgtt, vm); return old; @@ -722,9 +839,10 @@ static void __set_timeline(struct intel_timeline **dst, intel_timeline_put(old); } -static void __apply_timeline(struct intel_context *ce, void *timeline) +static int __apply_timeline(struct intel_context *ce, void *timeline) { __set_timeline(&ce->timeline, timeline); + return 0; } static void __assign_timeline(struct i915_gem_context *ctx, @@ -806,6 +924,7 @@ void i915_gem_init__contexts(struct drm_i915_private *i915) void i915_gem_driver_release__contexts(struct drm_i915_private *i915) { flush_work(&i915->gem.contexts.free_work); + rcu_barrier(); /* and flush the left over RCU frees */ } static int gem_context_register(struct i915_gem_context *ctx, @@ -981,6 +1100,7 @@ static int context_barrier_task(struct i915_gem_context *ctx, { struct context_barrier_task *cb; struct i915_gem_engines_iter it; + struct i915_gem_engines *e; struct intel_context *ce; int err = 0; @@ -997,7 +1117,13 @@ static int context_barrier_task(struct i915_gem_context *ctx, return err; } - for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { + e = __context_engines_await(ctx); + if (!e) { + i915_active_release(&cb->base); + return -ENOENT; + } + + for_each_gem_engine(ce, e, it) { struct i915_request *rq; if (I915_SELFTEST_ONLY(context_barrier_inject_fault & @@ -1028,7 +1154,7 @@ static int context_barrier_task(struct i915_gem_context *ctx, if (err) break; } - i915_gem_context_unlock_engines(ctx); + i915_sw_fence_complete(&e->fence); cb->task = err ? NULL : task; /* caller needs to unwind instead */ cb->data = data; @@ -1215,10 +1341,67 @@ out: return err; } -static int -user_to_context_sseu(struct drm_i915_private *i915, - const struct drm_i915_gem_context_param_sseu *user, - struct intel_sseu *context) +static int __apply_ringsize(struct intel_context *ce, void *sz) +{ + return intel_context_set_ring_size(ce, (unsigned long)sz); +} + +static int set_ringsize(struct i915_gem_context *ctx, + struct drm_i915_gem_context_param *args) +{ + if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915)) + return -ENODEV; + + if (args->size) + return -EINVAL; + + if (!IS_ALIGNED(args->value, I915_GTT_PAGE_SIZE)) + return -EINVAL; + + if (args->value < I915_GTT_PAGE_SIZE) + return -EINVAL; + + if (args->value > 128 * I915_GTT_PAGE_SIZE) + return -EINVAL; + + return context_apply_all(ctx, + __apply_ringsize, + __intel_context_ring_size(args->value)); +} + +static int __get_ringsize(struct intel_context *ce, void *arg) +{ + long sz; + + sz = intel_context_get_ring_size(ce); + GEM_BUG_ON(sz > INT_MAX); + + return sz; /* stop on first engine */ +} + +static int get_ringsize(struct i915_gem_context *ctx, + struct drm_i915_gem_context_param *args) +{ + int sz; + + if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915)) + return -ENODEV; + + if (args->size) + return -EINVAL; + + sz = context_apply_all(ctx, __get_ringsize, NULL); + if (sz < 0) + return sz; + + args->value = sz; + return 0; +} + +int +i915_gem_user_to_context_sseu(struct drm_i915_private *i915, + const struct drm_i915_gem_context_param_sseu *user, + struct intel_sseu *context) { const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; @@ -1353,7 +1536,7 @@ static int set_sseu(struct i915_gem_context *ctx, goto out_ce; } - ret = user_to_context_sseu(i915, &user_sseu, &sseu); + ret = i915_gem_user_to_context_sseu(i915, &user_sseu, &sseu); if (ret) goto out_ce; @@ -1390,7 +1573,7 @@ set_engines__load_balance(struct i915_user_extension __user *base, void *data) if (!HAS_EXECLISTS(i915)) return -ENODEV; - if (USES_GUC_SUBMISSION(i915)) + if (intel_uc_uses_guc_submission(&i915->gt.uc)) return -ENODEV; /* not implement yet */ if (get_user(idx, &ext->engine_index)) @@ -1562,77 +1745,6 @@ static const i915_user_extension_fn set_engines__extensions[] = { [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, }; -static int engines_notify(struct i915_sw_fence *fence, - enum i915_sw_fence_notify state) -{ - struct i915_gem_engines *engines = - container_of(fence, typeof(*engines), fence); - - switch (state) { - case FENCE_COMPLETE: - if (!list_empty(&engines->link)) { - struct i915_gem_context *ctx = engines->ctx; - unsigned long flags; - - spin_lock_irqsave(&ctx->stale.lock, flags); - list_del(&engines->link); - spin_unlock_irqrestore(&ctx->stale.lock, flags); - } - break; - - case FENCE_FREE: - init_rcu_head(&engines->rcu); - call_rcu(&engines->rcu, free_engines_rcu); - break; - } - - return NOTIFY_DONE; -} - -static void engines_idle_release(struct i915_gem_engines *engines) -{ - struct i915_gem_engines_iter it; - struct intel_context *ce; - unsigned long flags; - - GEM_BUG_ON(!engines); - i915_sw_fence_init(&engines->fence, engines_notify); - - INIT_LIST_HEAD(&engines->link); - spin_lock_irqsave(&engines->ctx->stale.lock, flags); - if (!i915_gem_context_is_closed(engines->ctx)) - list_add(&engines->link, &engines->ctx->stale.engines); - spin_unlock_irqrestore(&engines->ctx->stale.lock, flags); - if (list_empty(&engines->link)) /* raced, already closed */ - goto kill; - - for_each_gem_engine(ce, engines, it) { - struct dma_fence *fence; - int err; - - if (!ce->timeline) - continue; - - fence = i915_active_fence_get(&ce->timeline->last_request); - if (!fence) - continue; - - err = i915_sw_fence_await_dma_fence(&engines->fence, - fence, 0, - GFP_KERNEL); - - dma_fence_put(fence); - if (err < 0) - goto kill; - } - goto out; - -kill: - kill_engines(engines); -out: - i915_sw_fence_commit(&engines->fence); -} - static int set_engines(struct i915_gem_context *ctx, const struct drm_i915_gem_context_param *args) @@ -1669,14 +1781,10 @@ set_engines(struct i915_gem_context *ctx, * first 64 engines defined here. */ num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); - - set.engines = kmalloc(struct_size(set.engines, engines, num_engines), - GFP_KERNEL); + set.engines = alloc_engines(num_engines); if (!set.engines) return -ENOMEM; - set.engines->ctx = ctx; - for (n = 0; n < num_engines; n++) { struct i915_engine_class_instance ci; struct intel_engine_cs *engine; @@ -1729,6 +1837,11 @@ set_engines(struct i915_gem_context *ctx, replace: mutex_lock(&ctx->engines_mutex); + if (i915_gem_context_is_closed(ctx)) { + mutex_unlock(&ctx->engines_mutex); + free_engines(set.engines); + return -ENOENT; + } if (args->size) i915_gem_context_set_user_engines(ctx); else @@ -1737,7 +1850,7 @@ replace: mutex_unlock(&ctx->engines_mutex); /* Keep track of old engine sets for kill_context() */ - engines_idle_release(set.engines); + engines_idle_release(ctx, set.engines); return 0; } @@ -1748,7 +1861,7 @@ __copy_engines(struct i915_gem_engines *e) struct i915_gem_engines *copy; unsigned int n; - copy = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); + copy = alloc_engines(e->num_engines); if (!copy) return ERR_PTR(-ENOMEM); @@ -1852,17 +1965,19 @@ set_persistence(struct i915_gem_context *ctx, return __context_set_persistence(ctx, args->value); } -static void __apply_priority(struct intel_context *ce, void *arg) +static int __apply_priority(struct intel_context *ce, void *arg) { struct i915_gem_context *ctx = arg; if (!intel_engine_has_semaphores(ce->engine)) - return; + return 0; if (ctx->sched.priority >= I915_PRIORITY_NORMAL) intel_context_set_use_semaphores(ce); else intel_context_clear_use_semaphores(ce); + + return 0; } static int set_priority(struct i915_gem_context *ctx, @@ -1955,6 +2070,10 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, ret = set_persistence(ctx, args); break; + case I915_CONTEXT_PARAM_RINGSIZE: + ret = set_ringsize(ctx, args); + break; + case I915_CONTEXT_PARAM_BAN_PERIOD: default: ret = -EINVAL; @@ -1983,6 +2102,18 @@ static int create_setparam(struct i915_user_extension __user *ext, void *data) return ctx_setparam(arg->fpriv, arg->ctx, &local.param); } +static int copy_ring_size(struct intel_context *dst, + struct intel_context *src) +{ + long sz; + + sz = intel_context_get_ring_size(src); + if (sz < 0) + return sz; + + return intel_context_set_ring_size(dst, sz); +} + static int clone_engines(struct i915_gem_context *dst, struct i915_gem_context *src) { @@ -1991,12 +2122,10 @@ static int clone_engines(struct i915_gem_context *dst, bool user_engines; unsigned long n; - clone = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); + clone = alloc_engines(e->num_engines); if (!clone) goto err_unlock; - clone->ctx = dst; - for (n = 0; n < e->num_engines; n++) { struct intel_engine_cs *engine; @@ -2026,6 +2155,12 @@ static int clone_engines(struct i915_gem_context *dst, } intel_context_set_gem(clone->engines[n], dst); + + /* Copy across the preferred ringsize */ + if (copy_ring_size(clone->engines[n], e->engines[n])) { + __free_engines(clone, n + 1); + goto err_unlock; + } } clone->num_engines = n; @@ -2033,8 +2168,7 @@ static int clone_engines(struct i915_gem_context *dst, i915_gem_context_unlock_engines(src); /* Serialised by constructor */ - free_engines(__context_engines_static(dst)); - RCU_INIT_POINTER(dst->engines, clone); + engines_idle_release(dst, rcu_replace_pointer(dst->engines, clone, 1)); if (user_engines) i915_gem_context_set_user_engines(dst); else @@ -2388,6 +2522,10 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, args->value = i915_gem_context_is_persistent(ctx); break; + case I915_CONTEXT_PARAM_RINGSIZE: + ret = get_ringsize(ctx, args); + break; + case I915_CONTEXT_PARAM_BAN_PERIOD: default: ret = -EINVAL; @@ -2461,6 +2599,9 @@ i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) const struct i915_gem_engines *e = it->engines; struct intel_context *ctx; + if (unlikely(!e)) + return NULL; + do { if (it->idx >= e->num_engines) return NULL; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.h b/drivers/gpu/drm/i915/gem/i915_gem_context.h index 3ae61a355d87..3702b2fb27ab 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.h @@ -192,12 +192,16 @@ i915_gem_context_unlock_engines(struct i915_gem_context *ctx) static inline struct intel_context * i915_gem_context_get_engine(struct i915_gem_context *ctx, unsigned int idx) { - struct intel_context *ce = ERR_PTR(-EINVAL); + struct intel_context *ce; rcu_read_lock(); { struct i915_gem_engines *e = rcu_dereference(ctx->engines); - if (likely(idx < e->num_engines && e->engines[idx])) + if (unlikely(!e)) /* context was closed! */ + ce = ERR_PTR(-ENOENT); + else if (likely(idx < e->num_engines && e->engines[idx])) ce = intel_context_get(e->engines[idx]); + else + ce = ERR_PTR(-EINVAL); } rcu_read_unlock(); return ce; @@ -207,7 +211,6 @@ static inline void i915_gem_engines_iter_init(struct i915_gem_engines_iter *it, struct i915_gem_engines *engines) { - GEM_BUG_ON(!engines); it->engines = engines; it->idx = 0; } @@ -222,4 +225,8 @@ i915_gem_engines_iter_next(struct i915_gem_engines_iter *it); struct i915_lut_handle *i915_lut_handle_alloc(void); void i915_lut_handle_free(struct i915_lut_handle *lut); +int i915_gem_user_to_context_sseu(struct drm_i915_private *i915, + const struct drm_i915_gem_context_param_sseu *user, + struct intel_sseu *context); + #endif /* !__I915_GEM_CONTEXT_H__ */ diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c index 0cc40e77bbd2..7f76fc68f498 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c @@ -368,8 +368,7 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_vma *vma; - GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); - if (!atomic_read(&obj->bind_count)) + if (list_empty(&obj->vma.list)) return; mutex_lock(&i915->ggtt.vm.mutex); @@ -400,12 +399,8 @@ static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj) void i915_gem_object_unpin_from_display_plane(struct i915_vma *vma) { - struct drm_i915_gem_object *obj = vma->obj; - - assert_object_held(obj); - /* Bump the LRU to try and avoid premature eviction whilst flipping */ - i915_gem_object_bump_inactive_ggtt(obj); + i915_gem_object_bump_inactive_ggtt(vma->obj); i915_vma_unpin(vma); } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 87fa5f42c39a..c0d59d48e198 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -10,14 +10,13 @@ #include <linux/uaccess.h> #include <drm/drm_syncobj.h> -#include <drm/i915_drm.h> #include "display/intel_frontbuffer.h" #include "gem/i915_gem_ioctls.h" #include "gt/intel_context.h" -#include "gt/intel_engine_pool.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_buffer_pool.h" #include "gt/intel_gt_pm.h" #include "gt/intel_ring.h" @@ -28,6 +27,24 @@ #include "i915_sw_fence_work.h" #include "i915_trace.h" +struct eb_vma { + struct i915_vma *vma; + unsigned int flags; + + /** This vma's place in the execbuf reservation list */ + struct drm_i915_gem_exec_object2 *exec; + struct list_head bind_link; + struct list_head reloc_link; + + struct hlist_node node; + u32 handle; +}; + +struct eb_vma_array { + struct kref kref; + struct eb_vma vma[]; +}; + enum { FORCE_CPU_RELOC = 1, FORCE_GTT_RELOC, @@ -35,17 +52,14 @@ enum { #define DBG_FORCE_RELOC 0 /* choose one of the above! */ }; -#define __EXEC_OBJECT_HAS_REF BIT(31) -#define __EXEC_OBJECT_HAS_PIN BIT(30) -#define __EXEC_OBJECT_HAS_FENCE BIT(29) -#define __EXEC_OBJECT_NEEDS_MAP BIT(28) -#define __EXEC_OBJECT_NEEDS_BIAS BIT(27) -#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 27) /* all of the above */ -#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE) +#define __EXEC_OBJECT_HAS_PIN BIT(31) +#define __EXEC_OBJECT_HAS_FENCE BIT(30) +#define __EXEC_OBJECT_NEEDS_MAP BIT(29) +#define __EXEC_OBJECT_NEEDS_BIAS BIT(28) +#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28) /* all of the above */ #define __EXEC_HAS_RELOC BIT(31) -#define __EXEC_VALIDATED BIT(30) -#define __EXEC_INTERNAL_FLAGS (~0u << 30) +#define __EXEC_INTERNAL_FLAGS (~0u << 31) #define UPDATE PIN_OFFSET_FIXED #define BATCH_OFFSET_BIAS (256*1024) @@ -220,15 +234,14 @@ struct i915_execbuffer { struct drm_file *file; /** per-file lookup tables and limits */ struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */ struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */ - struct i915_vma **vma; - unsigned int *flags; + struct eb_vma *vma; struct intel_engine_cs *engine; /** engine to queue the request to */ struct intel_context *context; /* logical state for the request */ struct i915_gem_context *gem_context; /** caller's context */ struct i915_request *request; /** our request to build */ - struct i915_vma *batch; /** identity of the batch obj/vma */ + struct eb_vma *batch; /** identity of the batch obj/vma */ struct i915_vma *trampoline; /** trampoline used for chaining */ /** actual size of execobj[] as we may extend it for the cmdparser */ @@ -255,7 +268,9 @@ struct i915_execbuffer { bool has_fence : 1; bool needs_unfenced : 1; + struct i915_vma *target; struct i915_request *rq; + struct i915_vma *rq_vma; u32 *rq_cmd; unsigned int rq_size; } reloc_cache; @@ -274,10 +289,9 @@ struct i915_execbuffer { */ int lut_size; struct hlist_head *buckets; /** ht for relocation handles */ + struct eb_vma_array *array; }; -#define exec_entry(EB, VMA) (&(EB)->exec[(VMA)->exec_flags - (EB)->flags]) - static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) { return intel_engine_requires_cmd_parser(eb->engine) || @@ -285,8 +299,62 @@ static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) eb->args->batch_len); } +static struct eb_vma_array *eb_vma_array_create(unsigned int count) +{ + struct eb_vma_array *arr; + + arr = kvmalloc(struct_size(arr, vma, count), GFP_KERNEL | __GFP_NOWARN); + if (!arr) + return NULL; + + kref_init(&arr->kref); + arr->vma[0].vma = NULL; + + return arr; +} + +static inline void eb_unreserve_vma(struct eb_vma *ev) +{ + struct i915_vma *vma = ev->vma; + + if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) + __i915_vma_unpin_fence(vma); + + if (ev->flags & __EXEC_OBJECT_HAS_PIN) + __i915_vma_unpin(vma); + + ev->flags &= ~(__EXEC_OBJECT_HAS_PIN | + __EXEC_OBJECT_HAS_FENCE); +} + +static void eb_vma_array_destroy(struct kref *kref) +{ + struct eb_vma_array *arr = container_of(kref, typeof(*arr), kref); + struct eb_vma *ev = arr->vma; + + while (ev->vma) { + eb_unreserve_vma(ev); + i915_vma_put(ev->vma); + ev++; + } + + kvfree(arr); +} + +static void eb_vma_array_put(struct eb_vma_array *arr) +{ + kref_put(&arr->kref, eb_vma_array_destroy); +} + static int eb_create(struct i915_execbuffer *eb) { + /* Allocate an extra slot for use by the command parser + sentinel */ + eb->array = eb_vma_array_create(eb->buffer_count + 2); + if (!eb->array) + return -ENOMEM; + + eb->vma = eb->array->vma; + if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { unsigned int size = 1 + ilog2(eb->buffer_count); @@ -320,8 +388,10 @@ static int eb_create(struct i915_execbuffer *eb) break; } while (--size); - if (unlikely(!size)) + if (unlikely(!size)) { + eb_vma_array_put(eb->array); return -ENOMEM; + } eb->lut_size = size; } else { @@ -361,12 +431,38 @@ eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry, return false; } +static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry, + unsigned int exec_flags) +{ + u64 pin_flags = 0; + + if (exec_flags & EXEC_OBJECT_NEEDS_GTT) + pin_flags |= PIN_GLOBAL; + + /* + * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset, + * limit address to the first 4GBs for unflagged objects. + */ + if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) + pin_flags |= PIN_ZONE_4G; + + if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) + pin_flags |= PIN_MAPPABLE; + + if (exec_flags & EXEC_OBJECT_PINNED) + pin_flags |= entry->offset | PIN_OFFSET_FIXED; + else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) + pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; + + return pin_flags; +} + static inline bool eb_pin_vma(struct i915_execbuffer *eb, const struct drm_i915_gem_exec_object2 *entry, - struct i915_vma *vma) + struct eb_vma *ev) { - unsigned int exec_flags = *vma->exec_flags; + struct i915_vma *vma = ev->vma; u64 pin_flags; if (vma->node.size) @@ -375,44 +471,35 @@ eb_pin_vma(struct i915_execbuffer *eb, pin_flags = entry->offset & PIN_OFFSET_MASK; pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED; - if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT)) + if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT)) pin_flags |= PIN_GLOBAL; - if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags))) - return false; + /* Attempt to reuse the current location if available */ + if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags))) { + if (entry->flags & EXEC_OBJECT_PINNED) + return false; + + /* Failing that pick any _free_ space if suitable */ + if (unlikely(i915_vma_pin(vma, + entry->pad_to_size, + entry->alignment, + eb_pin_flags(entry, ev->flags) | + PIN_USER | PIN_NOEVICT))) + return false; + } - if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) { + if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { if (unlikely(i915_vma_pin_fence(vma))) { i915_vma_unpin(vma); return false; } if (vma->fence) - exec_flags |= __EXEC_OBJECT_HAS_FENCE; + ev->flags |= __EXEC_OBJECT_HAS_FENCE; } - *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN; - return !eb_vma_misplaced(entry, vma, exec_flags); -} - -static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags) -{ - GEM_BUG_ON(!(flags & __EXEC_OBJECT_HAS_PIN)); - - if (unlikely(flags & __EXEC_OBJECT_HAS_FENCE)) - __i915_vma_unpin_fence(vma); - - __i915_vma_unpin(vma); -} - -static inline void -eb_unreserve_vma(struct i915_vma *vma, unsigned int *flags) -{ - if (!(*flags & __EXEC_OBJECT_HAS_PIN)) - return; - - __eb_unreserve_vma(vma, *flags); - *flags &= ~__EXEC_OBJECT_RESERVED; + ev->flags |= __EXEC_OBJECT_HAS_PIN; + return !eb_vma_misplaced(entry, vma, ev->flags); } static int @@ -420,11 +507,11 @@ eb_validate_vma(struct i915_execbuffer *eb, struct drm_i915_gem_exec_object2 *entry, struct i915_vma *vma) { - struct drm_i915_private *i915 = eb->i915; if (unlikely(entry->flags & eb->invalid_flags)) return -EINVAL; - if (unlikely(entry->alignment && !is_power_of_2(entry->alignment))) + if (unlikely(entry->alignment && + !is_power_of_2_u64(entry->alignment))) return -EINVAL; /* @@ -442,14 +529,6 @@ eb_validate_vma(struct i915_execbuffer *eb, } else { entry->pad_to_size = 0; } - - if (unlikely(vma->exec_flags)) { - drm_dbg(&i915->drm, - "Object [handle %d, index %d] appears more than once in object list\n", - entry->handle, (int)(entry - eb->exec)); - return -EINVAL; - } - /* * From drm_mm perspective address space is continuous, * so from this point we're always using non-canonical @@ -472,41 +551,29 @@ eb_validate_vma(struct i915_execbuffer *eb, return 0; } -static int +static void eb_add_vma(struct i915_execbuffer *eb, unsigned int i, unsigned batch_idx, struct i915_vma *vma) { struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; - int err; + struct eb_vma *ev = &eb->vma[i]; GEM_BUG_ON(i915_vma_is_closed(vma)); - if (!(eb->args->flags & __EXEC_VALIDATED)) { - err = eb_validate_vma(eb, entry, vma); - if (unlikely(err)) - return err; - } + ev->vma = vma; + ev->exec = entry; + ev->flags = entry->flags; if (eb->lut_size > 0) { - vma->exec_handle = entry->handle; - hlist_add_head(&vma->exec_node, + ev->handle = entry->handle; + hlist_add_head(&ev->node, &eb->buckets[hash_32(entry->handle, eb->lut_size)]); } if (entry->relocation_count) - list_add_tail(&vma->reloc_link, &eb->relocs); - - /* - * Stash a pointer from the vma to execobj, so we can query its flags, - * size, alignment etc as provided by the user. Also we stash a pointer - * to the vma inside the execobj so that we can use a direct lookup - * to find the right target VMA when doing relocations. - */ - eb->vma[i] = vma; - eb->flags[i] = entry->flags; - vma->exec_flags = &eb->flags[i]; + list_add_tail(&ev->reloc_link, &eb->relocs); /* * SNA is doing fancy tricks with compressing batch buffers, which leads @@ -519,30 +586,23 @@ eb_add_vma(struct i915_execbuffer *eb, */ if (i == batch_idx) { if (entry->relocation_count && - !(eb->flags[i] & EXEC_OBJECT_PINNED)) - eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS; + !(ev->flags & EXEC_OBJECT_PINNED)) + ev->flags |= __EXEC_OBJECT_NEEDS_BIAS; if (eb->reloc_cache.has_fence) - eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE; + ev->flags |= EXEC_OBJECT_NEEDS_FENCE; - eb->batch = vma; + eb->batch = ev; } - err = 0; - if (eb_pin_vma(eb, entry, vma)) { + if (eb_pin_vma(eb, entry, ev)) { if (entry->offset != vma->node.start) { entry->offset = vma->node.start | UPDATE; eb->args->flags |= __EXEC_HAS_RELOC; } } else { - eb_unreserve_vma(vma, vma->exec_flags); - - list_add_tail(&vma->exec_link, &eb->unbound); - if (drm_mm_node_allocated(&vma->node)) - err = i915_vma_unbind(vma); - if (unlikely(err)) - vma->exec_flags = NULL; + eb_unreserve_vma(ev); + list_add_tail(&ev->bind_link, &eb->unbound); } - return err; } static inline int use_cpu_reloc(const struct reloc_cache *cache, @@ -563,37 +623,23 @@ static inline int use_cpu_reloc(const struct reloc_cache *cache, } static int eb_reserve_vma(const struct i915_execbuffer *eb, - struct i915_vma *vma) + struct eb_vma *ev, + u64 pin_flags) { - struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma); - unsigned int exec_flags = *vma->exec_flags; - u64 pin_flags; + struct drm_i915_gem_exec_object2 *entry = ev->exec; + struct i915_vma *vma = ev->vma; int err; - pin_flags = PIN_USER | PIN_NONBLOCK; - if (exec_flags & EXEC_OBJECT_NEEDS_GTT) - pin_flags |= PIN_GLOBAL; - - /* - * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset, - * limit address to the first 4GBs for unflagged objects. - */ - if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) - pin_flags |= PIN_ZONE_4G; - - if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) - pin_flags |= PIN_MAPPABLE; - - if (exec_flags & EXEC_OBJECT_PINNED) { - pin_flags |= entry->offset | PIN_OFFSET_FIXED; - pin_flags &= ~PIN_NONBLOCK; /* force overlapping checks */ - } else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) { - pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; + if (drm_mm_node_allocated(&vma->node) && + eb_vma_misplaced(entry, vma, ev->flags)) { + err = i915_vma_unbind(vma); + if (err) + return err; } err = i915_vma_pin(vma, entry->pad_to_size, entry->alignment, - pin_flags); + eb_pin_flags(entry, ev->flags) | pin_flags); if (err) return err; @@ -602,7 +648,7 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb, eb->args->flags |= __EXEC_HAS_RELOC; } - if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) { + if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { err = i915_vma_pin_fence(vma); if (unlikely(err)) { i915_vma_unpin(vma); @@ -610,11 +656,11 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb, } if (vma->fence) - exec_flags |= __EXEC_OBJECT_HAS_FENCE; + ev->flags |= __EXEC_OBJECT_HAS_FENCE; } - *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN; - GEM_BUG_ON(eb_vma_misplaced(entry, vma, exec_flags)); + ev->flags |= __EXEC_OBJECT_HAS_PIN; + GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags)); return 0; } @@ -622,10 +668,11 @@ static int eb_reserve_vma(const struct i915_execbuffer *eb, static int eb_reserve(struct i915_execbuffer *eb) { const unsigned int count = eb->buffer_count; + unsigned int pin_flags = PIN_USER | PIN_NONBLOCK; struct list_head last; - struct i915_vma *vma; + struct eb_vma *ev; unsigned int i, pass; - int err; + int err = 0; /* * Attempt to pin all of the buffers into the GTT. @@ -641,44 +688,54 @@ static int eb_reserve(struct i915_execbuffer *eb) * room for the earlier objects *unless* we need to defragment. */ + if (mutex_lock_interruptible(&eb->i915->drm.struct_mutex)) + return -EINTR; + pass = 0; - err = 0; do { - list_for_each_entry(vma, &eb->unbound, exec_link) { - err = eb_reserve_vma(eb, vma); + list_for_each_entry(ev, &eb->unbound, bind_link) { + err = eb_reserve_vma(eb, ev, pin_flags); if (err) break; } - if (err != -ENOSPC) - return err; + if (!(err == -ENOSPC || err == -EAGAIN)) + break; /* Resort *all* the objects into priority order */ INIT_LIST_HEAD(&eb->unbound); INIT_LIST_HEAD(&last); for (i = 0; i < count; i++) { - unsigned int flags = eb->flags[i]; - struct i915_vma *vma = eb->vma[i]; + unsigned int flags; + ev = &eb->vma[i]; + flags = ev->flags; if (flags & EXEC_OBJECT_PINNED && flags & __EXEC_OBJECT_HAS_PIN) continue; - eb_unreserve_vma(vma, &eb->flags[i]); + eb_unreserve_vma(ev); if (flags & EXEC_OBJECT_PINNED) /* Pinned must have their slot */ - list_add(&vma->exec_link, &eb->unbound); + list_add(&ev->bind_link, &eb->unbound); else if (flags & __EXEC_OBJECT_NEEDS_MAP) /* Map require the lowest 256MiB (aperture) */ - list_add_tail(&vma->exec_link, &eb->unbound); + list_add_tail(&ev->bind_link, &eb->unbound); else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) /* Prioritise 4GiB region for restricted bo */ - list_add(&vma->exec_link, &last); + list_add(&ev->bind_link, &last); else - list_add_tail(&vma->exec_link, &last); + list_add_tail(&ev->bind_link, &last); } list_splice_tail(&last, &eb->unbound); + if (err == -EAGAIN) { + mutex_unlock(&eb->i915->drm.struct_mutex); + flush_workqueue(eb->i915->mm.userptr_wq); + mutex_lock(&eb->i915->drm.struct_mutex); + continue; + } + switch (pass++) { case 0: break; @@ -689,13 +746,20 @@ static int eb_reserve(struct i915_execbuffer *eb) err = i915_gem_evict_vm(eb->context->vm); mutex_unlock(&eb->context->vm->mutex); if (err) - return err; + goto unlock; break; default: - return -ENOSPC; + err = -ENOSPC; + goto unlock; } + + pin_flags = PIN_USER; } while (1); + +unlock: + mutex_unlock(&eb->i915->drm.struct_mutex); + return err; } static unsigned int eb_batch_index(const struct i915_execbuffer *eb) @@ -725,148 +789,148 @@ static int eb_select_context(struct i915_execbuffer *eb) return 0; } -static int eb_lookup_vmas(struct i915_execbuffer *eb) +static int __eb_add_lut(struct i915_execbuffer *eb, + u32 handle, struct i915_vma *vma) { - struct radix_tree_root *handles_vma = &eb->gem_context->handles_vma; - struct drm_i915_gem_object *obj; - unsigned int i, batch; + struct i915_gem_context *ctx = eb->gem_context; + struct i915_lut_handle *lut; int err; - INIT_LIST_HEAD(&eb->relocs); - INIT_LIST_HEAD(&eb->unbound); + lut = i915_lut_handle_alloc(); + if (unlikely(!lut)) + return -ENOMEM; - batch = eb_batch_index(eb); + i915_vma_get(vma); + if (!atomic_fetch_inc(&vma->open_count)) + i915_vma_reopen(vma); + lut->handle = handle; + lut->ctx = ctx; - mutex_lock(&eb->gem_context->mutex); - if (unlikely(i915_gem_context_is_closed(eb->gem_context))) { + /* Check that the context hasn't been closed in the meantime */ + err = -EINTR; + if (!mutex_lock_interruptible(&ctx->mutex)) { err = -ENOENT; - goto err_ctx; + if (likely(!i915_gem_context_is_closed(ctx))) + err = radix_tree_insert(&ctx->handles_vma, handle, vma); + if (err == 0) { /* And nor has this handle */ + struct drm_i915_gem_object *obj = vma->obj; + + i915_gem_object_lock(obj); + if (idr_find(&eb->file->object_idr, handle) == obj) { + list_add(&lut->obj_link, &obj->lut_list); + } else { + radix_tree_delete(&ctx->handles_vma, handle); + err = -ENOENT; + } + i915_gem_object_unlock(obj); + } + mutex_unlock(&ctx->mutex); } + if (unlikely(err)) + goto err; - for (i = 0; i < eb->buffer_count; i++) { - u32 handle = eb->exec[i].handle; - struct i915_lut_handle *lut; + return 0; + +err: + i915_vma_close(vma); + i915_vma_put(vma); + i915_lut_handle_free(lut); + return err; +} + +static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) +{ + do { + struct drm_i915_gem_object *obj; struct i915_vma *vma; + int err; - vma = radix_tree_lookup(handles_vma, handle); + rcu_read_lock(); + vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); + if (likely(vma)) + vma = i915_vma_tryget(vma); + rcu_read_unlock(); if (likely(vma)) - goto add_vma; + return vma; obj = i915_gem_object_lookup(eb->file, handle); - if (unlikely(!obj)) { - err = -ENOENT; - goto err_vma; - } + if (unlikely(!obj)) + return ERR_PTR(-ENOENT); vma = i915_vma_instance(obj, eb->context->vm, NULL); if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto err_obj; + i915_gem_object_put(obj); + return vma; } - lut = i915_lut_handle_alloc(); - if (unlikely(!lut)) { - err = -ENOMEM; - goto err_obj; - } + err = __eb_add_lut(eb, handle, vma); + if (likely(!err)) + return vma; - err = radix_tree_insert(handles_vma, handle, vma); - if (unlikely(err)) { - i915_lut_handle_free(lut); - goto err_obj; - } + i915_gem_object_put(obj); + if (err != -EEXIST) + return ERR_PTR(err); + } while (1); +} - /* transfer ref to lut */ - if (!atomic_fetch_inc(&vma->open_count)) - i915_vma_reopen(vma); - lut->handle = handle; - lut->ctx = eb->gem_context; +static int eb_lookup_vmas(struct i915_execbuffer *eb) +{ + unsigned int batch = eb_batch_index(eb); + unsigned int i; + int err = 0; - i915_gem_object_lock(obj); - list_add(&lut->obj_link, &obj->lut_list); - i915_gem_object_unlock(obj); + INIT_LIST_HEAD(&eb->relocs); + INIT_LIST_HEAD(&eb->unbound); -add_vma: - err = eb_add_vma(eb, i, batch, vma); - if (unlikely(err)) - goto err_vma; + for (i = 0; i < eb->buffer_count; i++) { + struct i915_vma *vma; - GEM_BUG_ON(vma != eb->vma[i]); - GEM_BUG_ON(vma->exec_flags != &eb->flags[i]); - GEM_BUG_ON(drm_mm_node_allocated(&vma->node) && - eb_vma_misplaced(&eb->exec[i], vma, eb->flags[i])); - } + vma = eb_lookup_vma(eb, eb->exec[i].handle); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + break; + } - mutex_unlock(&eb->gem_context->mutex); + err = eb_validate_vma(eb, &eb->exec[i], vma); + if (unlikely(err)) { + i915_vma_put(vma); + break; + } - eb->args->flags |= __EXEC_VALIDATED; - return eb_reserve(eb); + eb_add_vma(eb, i, batch, vma); + } -err_obj: - i915_gem_object_put(obj); -err_vma: - eb->vma[i] = NULL; -err_ctx: - mutex_unlock(&eb->gem_context->mutex); + eb->vma[i].vma = NULL; return err; } -static struct i915_vma * +static struct eb_vma * eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) { if (eb->lut_size < 0) { if (handle >= -eb->lut_size) return NULL; - return eb->vma[handle]; + return &eb->vma[handle]; } else { struct hlist_head *head; - struct i915_vma *vma; + struct eb_vma *ev; head = &eb->buckets[hash_32(handle, eb->lut_size)]; - hlist_for_each_entry(vma, head, exec_node) { - if (vma->exec_handle == handle) - return vma; + hlist_for_each_entry(ev, head, node) { + if (ev->handle == handle) + return ev; } return NULL; } } -static void eb_release_vmas(const struct i915_execbuffer *eb) -{ - const unsigned int count = eb->buffer_count; - unsigned int i; - - for (i = 0; i < count; i++) { - struct i915_vma *vma = eb->vma[i]; - unsigned int flags = eb->flags[i]; - - if (!vma) - break; - - GEM_BUG_ON(vma->exec_flags != &eb->flags[i]); - vma->exec_flags = NULL; - eb->vma[i] = NULL; - - if (flags & __EXEC_OBJECT_HAS_PIN) - __eb_unreserve_vma(vma, flags); - - if (flags & __EXEC_OBJECT_HAS_REF) - i915_vma_put(vma); - } -} - -static void eb_reset_vmas(const struct i915_execbuffer *eb) -{ - eb_release_vmas(eb); - if (eb->lut_size > 0) - memset(eb->buckets, 0, - sizeof(struct hlist_head) << eb->lut_size); -} - static void eb_destroy(const struct i915_execbuffer *eb) { GEM_BUG_ON(eb->reloc_cache.rq); + if (eb->array) + eb_vma_array_put(eb->array); + if (eb->lut_size > 0) kfree(eb->buckets); } @@ -891,7 +955,7 @@ static void reloc_cache_init(struct reloc_cache *cache, cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; cache->node.flags = 0; cache->rq = NULL; - cache->rq_size = 0; + cache->target = NULL; } static inline void *unmask_page(unsigned long p) @@ -913,27 +977,122 @@ static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) return &i915->ggtt; } -static void reloc_gpu_flush(struct reloc_cache *cache) +#define RELOC_TAIL 4 + +static int reloc_gpu_chain(struct reloc_cache *cache) { - GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32)); - cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END; + struct intel_gt_buffer_pool_node *pool; + struct i915_request *rq = cache->rq; + struct i915_vma *batch; + u32 *cmd; + int err; - __i915_gem_object_flush_map(cache->rq->batch->obj, 0, cache->rq_size); - i915_gem_object_unpin_map(cache->rq->batch->obj); + pool = intel_gt_get_buffer_pool(rq->engine->gt, PAGE_SIZE); + if (IS_ERR(pool)) + return PTR_ERR(pool); - intel_gt_chipset_flush(cache->rq->engine->gt); + batch = i915_vma_instance(pool->obj, rq->context->vm, NULL); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + goto out_pool; + } - i915_request_add(cache->rq); - cache->rq = NULL; + err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK); + if (err) + goto out_pool; + + GEM_BUG_ON(cache->rq_size + RELOC_TAIL > PAGE_SIZE / sizeof(u32)); + cmd = cache->rq_cmd + cache->rq_size; + *cmd++ = MI_ARB_CHECK; + if (cache->gen >= 8) + *cmd++ = MI_BATCH_BUFFER_START_GEN8; + else if (cache->gen >= 6) + *cmd++ = MI_BATCH_BUFFER_START; + else + *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; + *cmd++ = lower_32_bits(batch->node.start); + *cmd++ = upper_32_bits(batch->node.start); /* Always 0 for gen<8 */ + i915_gem_object_flush_map(cache->rq_vma->obj); + i915_gem_object_unpin_map(cache->rq_vma->obj); + cache->rq_vma = NULL; + + err = intel_gt_buffer_pool_mark_active(pool, rq); + if (err == 0) { + i915_vma_lock(batch); + err = i915_request_await_object(rq, batch->obj, false); + if (err == 0) + err = i915_vma_move_to_active(batch, rq, 0); + i915_vma_unlock(batch); + } + i915_vma_unpin(batch); + if (err) + goto out_pool; + + cmd = i915_gem_object_pin_map(batch->obj, + cache->has_llc ? + I915_MAP_FORCE_WB : + I915_MAP_FORCE_WC); + if (IS_ERR(cmd)) { + err = PTR_ERR(cmd); + goto out_pool; + } + + /* Return with batch mapping (cmd) still pinned */ + cache->rq_cmd = cmd; + cache->rq_size = 0; + cache->rq_vma = batch; + +out_pool: + intel_gt_buffer_pool_put(pool); + return err; +} + +static unsigned int reloc_bb_flags(const struct reloc_cache *cache) +{ + return cache->gen > 5 ? 0 : I915_DISPATCH_SECURE; +} + +static int reloc_gpu_flush(struct reloc_cache *cache) +{ + struct i915_request *rq; + int err; + + rq = fetch_and_zero(&cache->rq); + if (!rq) + return 0; + + if (cache->rq_vma) { + struct drm_i915_gem_object *obj = cache->rq_vma->obj; + + GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32)); + cache->rq_cmd[cache->rq_size++] = MI_BATCH_BUFFER_END; + + __i915_gem_object_flush_map(obj, + 0, sizeof(u32) * cache->rq_size); + i915_gem_object_unpin_map(obj); + } + + err = 0; + if (rq->engine->emit_init_breadcrumb) + err = rq->engine->emit_init_breadcrumb(rq); + if (!err) + err = rq->engine->emit_bb_start(rq, + rq->batch->node.start, + PAGE_SIZE, + reloc_bb_flags(cache)); + if (err) + i915_request_set_error_once(rq, err); + + intel_gt_chipset_flush(rq->engine->gt); + i915_request_add(rq); + + return err; } static void reloc_cache_reset(struct reloc_cache *cache) { void *vaddr; - if (cache->rq) - reloc_gpu_flush(cache); - if (!cache->vaddr) return; @@ -1126,17 +1285,17 @@ static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) } static int __reloc_gpu_alloc(struct i915_execbuffer *eb, - struct i915_vma *vma, + struct intel_engine_cs *engine, unsigned int len) { struct reloc_cache *cache = &eb->reloc_cache; - struct intel_engine_pool_node *pool; + struct intel_gt_buffer_pool_node *pool; struct i915_request *rq; struct i915_vma *batch; u32 *cmd; int err; - pool = intel_engine_get_pool(eb->engine, PAGE_SIZE); + pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE); if (IS_ERR(pool)) return PTR_ERR(pool); @@ -1149,7 +1308,7 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, goto out_pool; } - batch = i915_vma_instance(pool->obj, vma->vm, NULL); + batch = i915_vma_instance(pool->obj, eb->context->vm, NULL); if (IS_ERR(batch)) { err = PTR_ERR(batch); goto err_unmap; @@ -1159,26 +1318,32 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, if (err) goto err_unmap; - rq = i915_request_create(eb->context); + if (engine == eb->context->engine) { + rq = i915_request_create(eb->context); + } else { + struct intel_context *ce; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto err_unpin; + } + + i915_vm_put(ce->vm); + ce->vm = i915_vm_get(eb->context->vm); + + rq = intel_context_create_request(ce); + intel_context_put(ce); + } if (IS_ERR(rq)) { err = PTR_ERR(rq); goto err_unpin; } - err = intel_engine_pool_mark_active(pool, rq); - if (err) - goto err_request; - - err = reloc_move_to_gpu(rq, vma); + err = intel_gt_buffer_pool_mark_active(pool, rq); if (err) goto err_request; - err = eb->engine->emit_bb_start(rq, - batch->node.start, PAGE_SIZE, - cache->gen > 5 ? 0 : I915_DISPATCH_SECURE); - if (err) - goto skip_request; - i915_vma_lock(batch); err = i915_request_await_object(rq, batch->obj, false); if (err == 0) @@ -1193,12 +1358,13 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, cache->rq = rq; cache->rq_cmd = cmd; cache->rq_size = 0; + cache->rq_vma = batch; /* Return with batch mapping (cmd) still pinned */ goto out_pool; skip_request: - i915_request_skip(rq, err); + i915_request_set_error_once(rq, err); err_request: i915_request_add(rq); err_unpin: @@ -1206,134 +1372,216 @@ err_unpin: err_unmap: i915_gem_object_unpin_map(pool->obj); out_pool: - intel_engine_pool_put(pool); + intel_gt_buffer_pool_put(pool); return err; } +static bool reloc_can_use_engine(const struct intel_engine_cs *engine) +{ + return engine->class != VIDEO_DECODE_CLASS || !IS_GEN(engine->i915, 6); +} + static u32 *reloc_gpu(struct i915_execbuffer *eb, struct i915_vma *vma, unsigned int len) { struct reloc_cache *cache = &eb->reloc_cache; u32 *cmd; - - if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1)) - reloc_gpu_flush(cache); + int err; if (unlikely(!cache->rq)) { - int err; + struct intel_engine_cs *engine = eb->engine; - if (!intel_engine_can_store_dword(eb->engine)) - return ERR_PTR(-ENODEV); + if (!reloc_can_use_engine(engine)) { + engine = engine->gt->engine_class[COPY_ENGINE_CLASS][0]; + if (!engine) + return ERR_PTR(-ENODEV); + } - err = __reloc_gpu_alloc(eb, vma, len); + err = __reloc_gpu_alloc(eb, engine, len); if (unlikely(err)) return ERR_PTR(err); } + if (vma != cache->target) { + err = reloc_move_to_gpu(cache->rq, vma); + if (unlikely(err)) { + i915_request_set_error_once(cache->rq, err); + return ERR_PTR(err); + } + + cache->target = vma; + } + + if (unlikely(cache->rq_size + len > + PAGE_SIZE / sizeof(u32) - RELOC_TAIL)) { + err = reloc_gpu_chain(cache); + if (unlikely(err)) { + i915_request_set_error_once(cache->rq, err); + return ERR_PTR(err); + } + } + + GEM_BUG_ON(cache->rq_size + len >= PAGE_SIZE / sizeof(u32)); cmd = cache->rq_cmd + cache->rq_size; cache->rq_size += len; return cmd; } -static u64 -relocate_entry(struct i915_vma *vma, - const struct drm_i915_gem_relocation_entry *reloc, - struct i915_execbuffer *eb, - const struct i915_vma *target) +static inline bool use_reloc_gpu(struct i915_vma *vma) { - u64 offset = reloc->offset; - u64 target_offset = relocation_target(reloc, target); - bool wide = eb->reloc_cache.use_64bit_reloc; - void *vaddr; + if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) + return true; - if (!eb->reloc_cache.vaddr && - (DBG_FORCE_RELOC == FORCE_GPU_RELOC || - !dma_resv_test_signaled_rcu(vma->resv, true))) { - const unsigned int gen = eb->reloc_cache.gen; - unsigned int len; - u32 *batch; - u64 addr; - - if (wide) - len = offset & 7 ? 8 : 5; - else if (gen >= 4) - len = 4; - else - len = 3; + if (DBG_FORCE_RELOC) + return false; - batch = reloc_gpu(eb, vma, len); - if (IS_ERR(batch)) - goto repeat; + return !dma_resv_test_signaled_rcu(vma->resv, true); +} - addr = gen8_canonical_addr(vma->node.start + offset); - if (wide) { - if (offset & 7) { - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = lower_32_bits(addr); - *batch++ = upper_32_bits(addr); - *batch++ = lower_32_bits(target_offset); - - addr = gen8_canonical_addr(addr + 4); - - *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = lower_32_bits(addr); - *batch++ = upper_32_bits(addr); - *batch++ = upper_32_bits(target_offset); - } else { - *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1; - *batch++ = lower_32_bits(addr); - *batch++ = upper_32_bits(addr); - *batch++ = lower_32_bits(target_offset); - *batch++ = upper_32_bits(target_offset); - } - } else if (gen >= 6) { +static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) +{ + struct page *page; + unsigned long addr; + + GEM_BUG_ON(vma->pages != vma->obj->mm.pages); + + page = i915_gem_object_get_page(vma->obj, offset >> PAGE_SHIFT); + addr = PFN_PHYS(page_to_pfn(page)); + GEM_BUG_ON(overflows_type(addr, u32)); /* expected dma32 */ + + return addr + offset_in_page(offset); +} + +static bool __reloc_entry_gpu(struct i915_execbuffer *eb, + struct i915_vma *vma, + u64 offset, + u64 target_addr) +{ + const unsigned int gen = eb->reloc_cache.gen; + unsigned int len; + u32 *batch; + u64 addr; + + if (gen >= 8) + len = offset & 7 ? 8 : 5; + else if (gen >= 4) + len = 4; + else + len = 3; + + batch = reloc_gpu(eb, vma, len); + if (IS_ERR(batch)) + return false; + + addr = gen8_canonical_addr(vma->node.start + offset); + if (gen >= 8) { + if (offset & 7) { *batch++ = MI_STORE_DWORD_IMM_GEN4; - *batch++ = 0; - *batch++ = addr; - *batch++ = target_offset; - } else if (gen >= 4) { - *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; - *batch++ = 0; - *batch++ = addr; - *batch++ = target_offset; + *batch++ = lower_32_bits(addr); + *batch++ = upper_32_bits(addr); + *batch++ = lower_32_bits(target_addr); + + addr = gen8_canonical_addr(addr + 4); + + *batch++ = MI_STORE_DWORD_IMM_GEN4; + *batch++ = lower_32_bits(addr); + *batch++ = upper_32_bits(addr); + *batch++ = upper_32_bits(target_addr); } else { - *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; - *batch++ = addr; - *batch++ = target_offset; + *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1; + *batch++ = lower_32_bits(addr); + *batch++ = upper_32_bits(addr); + *batch++ = lower_32_bits(target_addr); + *batch++ = upper_32_bits(target_addr); } - - goto out; + } else if (gen >= 6) { + *batch++ = MI_STORE_DWORD_IMM_GEN4; + *batch++ = 0; + *batch++ = addr; + *batch++ = target_addr; + } else if (IS_I965G(eb->i915)) { + *batch++ = MI_STORE_DWORD_IMM_GEN4; + *batch++ = 0; + *batch++ = vma_phys_addr(vma, offset); + *batch++ = target_addr; + } else if (gen >= 4) { + *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *batch++ = 0; + *batch++ = addr; + *batch++ = target_addr; + } else if (gen >= 3 && + !(IS_I915G(eb->i915) || IS_I915GM(eb->i915))) { + *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; + *batch++ = addr; + *batch++ = target_addr; + } else { + *batch++ = MI_STORE_DWORD_IMM; + *batch++ = vma_phys_addr(vma, offset); + *batch++ = target_addr; } + return true; +} + +static bool reloc_entry_gpu(struct i915_execbuffer *eb, + struct i915_vma *vma, + u64 offset, + u64 target_addr) +{ + if (eb->reloc_cache.vaddr) + return false; + + if (!use_reloc_gpu(vma)) + return false; + + return __reloc_entry_gpu(eb, vma, offset, target_addr); +} + +static u64 +relocate_entry(struct i915_vma *vma, + const struct drm_i915_gem_relocation_entry *reloc, + struct i915_execbuffer *eb, + const struct i915_vma *target) +{ + u64 target_addr = relocation_target(reloc, target); + u64 offset = reloc->offset; + + if (!reloc_entry_gpu(eb, vma, offset, target_addr)) { + bool wide = eb->reloc_cache.use_64bit_reloc; + void *vaddr; + repeat: - vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT); - if (IS_ERR(vaddr)) - return PTR_ERR(vaddr); + vaddr = reloc_vaddr(vma->obj, + &eb->reloc_cache, + offset >> PAGE_SHIFT); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); - clflush_write32(vaddr + offset_in_page(offset), - lower_32_bits(target_offset), - eb->reloc_cache.vaddr); + GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); + clflush_write32(vaddr + offset_in_page(offset), + lower_32_bits(target_addr), + eb->reloc_cache.vaddr); - if (wide) { - offset += sizeof(u32); - target_offset >>= 32; - wide = false; - goto repeat; + if (wide) { + offset += sizeof(u32); + target_addr >>= 32; + wide = false; + goto repeat; + } } -out: return target->node.start | UPDATE; } static u64 eb_relocate_entry(struct i915_execbuffer *eb, - struct i915_vma *vma, + struct eb_vma *ev, const struct drm_i915_gem_relocation_entry *reloc) { struct drm_i915_private *i915 = eb->i915; - struct i915_vma *target; + struct eb_vma *target; int err; /* we've already hold a reference to all valid objects */ @@ -1365,7 +1613,7 @@ eb_relocate_entry(struct i915_execbuffer *eb, } if (reloc->write_domain) { - *target->exec_flags |= EXEC_OBJECT_WRITE; + target->flags |= EXEC_OBJECT_WRITE; /* * Sandybridge PPGTT errata: We need a global gtt mapping @@ -1375,7 +1623,8 @@ eb_relocate_entry(struct i915_execbuffer *eb, */ if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && IS_GEN(eb->i915, 6)) { - err = i915_vma_bind(target, target->obj->cache_level, + err = i915_vma_bind(target->vma, + target->vma->obj->cache_level, PIN_GLOBAL, NULL); if (WARN_ONCE(err, "Unexpected failure to bind target VMA!")) @@ -1388,17 +1637,17 @@ eb_relocate_entry(struct i915_execbuffer *eb, * more work needs to be done. */ if (!DBG_FORCE_RELOC && - gen8_canonical_addr(target->node.start) == reloc->presumed_offset) + gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) return 0; /* Check that the relocation address is valid... */ if (unlikely(reloc->offset > - vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { + ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { drm_dbg(&i915->drm, "Relocation beyond object bounds: " "target %d offset %d size %d.\n", reloc->target_handle, (int)reloc->offset, - (int)vma->size); + (int)ev->vma->size); return -EINVAL; } if (unlikely(reloc->offset & 3)) { @@ -1417,22 +1666,21 @@ eb_relocate_entry(struct i915_execbuffer *eb, * do relocations we are already stalling, disable the user's opt * out of our synchronisation. */ - *vma->exec_flags &= ~EXEC_OBJECT_ASYNC; + ev->flags &= ~EXEC_OBJECT_ASYNC; /* and update the user's relocation entry */ - return relocate_entry(vma, reloc, eb, target); + return relocate_entry(ev->vma, reloc, eb, target->vma); } -static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) +static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) { #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) struct drm_i915_gem_relocation_entry stack[N_RELOC(512)]; - struct drm_i915_gem_relocation_entry __user *urelocs; - const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma); - unsigned int remain; + const struct drm_i915_gem_exec_object2 *entry = ev->exec; + struct drm_i915_gem_relocation_entry __user *urelocs = + u64_to_user_ptr(entry->relocs_ptr); + unsigned long remain = entry->relocation_count; - urelocs = u64_to_user_ptr(entry->relocs_ptr); - remain = entry->relocation_count; if (unlikely(remain > N_RELOC(ULONG_MAX))) return -EINVAL; @@ -1441,13 +1689,13 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) * to read. However, if the array is not writable the user loses * the updated relocation values. */ - if (unlikely(!access_ok(urelocs, remain*sizeof(*urelocs)))) + if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs)))) return -EFAULT; do { struct drm_i915_gem_relocation_entry *r = stack; unsigned int count = - min_t(unsigned int, remain, ARRAY_SIZE(stack)); + min_t(unsigned long, remain, ARRAY_SIZE(stack)); unsigned int copied; /* @@ -1458,9 +1706,7 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) * we would try to acquire the struct mutex again. Obviously * this is bad and so lockdep complains vehemently. */ - pagefault_disable(); - copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0])); - pagefault_enable(); + copied = __copy_from_user(r, urelocs, count * sizeof(r[0])); if (unlikely(copied)) { remain = -EFAULT; goto out; @@ -1468,7 +1714,7 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) remain -= count; do { - u64 offset = eb_relocate_entry(eb, vma, r); + u64 offset = eb_relocate_entry(eb, ev, r); if (likely(offset == 0)) { } else if ((s64)offset < 0) { @@ -1497,10 +1743,8 @@ static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma) * can read from this userspace address. */ offset = gen8_canonical_addr(offset & ~UPDATE); - if (unlikely(__put_user(offset, &urelocs[r-stack].presumed_offset))) { - remain = -EFAULT; - goto out; - } + __put_user(offset, + &urelocs[r - stack].presumed_offset); } } while (r++, --count); urelocs += ARRAY_SIZE(stack); @@ -1510,280 +1754,39 @@ out: return remain; } -static int -eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma) -{ - const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma); - struct drm_i915_gem_relocation_entry *relocs = - u64_to_ptr(typeof(*relocs), entry->relocs_ptr); - unsigned int i; - int err; - - for (i = 0; i < entry->relocation_count; i++) { - u64 offset = eb_relocate_entry(eb, vma, &relocs[i]); - - if ((s64)offset < 0) { - err = (int)offset; - goto err; - } - } - err = 0; -err: - reloc_cache_reset(&eb->reloc_cache); - return err; -} - -static int check_relocations(const struct drm_i915_gem_exec_object2 *entry) -{ - const char __user *addr, *end; - unsigned long size; - char __maybe_unused c; - - size = entry->relocation_count; - if (size == 0) - return 0; - - if (size > N_RELOC(ULONG_MAX)) - return -EINVAL; - - addr = u64_to_user_ptr(entry->relocs_ptr); - size *= sizeof(struct drm_i915_gem_relocation_entry); - if (!access_ok(addr, size)) - return -EFAULT; - - end = addr + size; - for (; addr < end; addr += PAGE_SIZE) { - int err = __get_user(c, addr); - if (err) - return err; - } - return __get_user(c, end - 1); -} - -static int eb_copy_relocations(const struct i915_execbuffer *eb) +static int eb_relocate(struct i915_execbuffer *eb) { - struct drm_i915_gem_relocation_entry *relocs; - const unsigned int count = eb->buffer_count; - unsigned int i; int err; - for (i = 0; i < count; i++) { - const unsigned int nreloc = eb->exec[i].relocation_count; - struct drm_i915_gem_relocation_entry __user *urelocs; - unsigned long size; - unsigned long copied; - - if (nreloc == 0) - continue; - - err = check_relocations(&eb->exec[i]); - if (err) - goto err; - - urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); - size = nreloc * sizeof(*relocs); - - relocs = kvmalloc_array(size, 1, GFP_KERNEL); - if (!relocs) { - err = -ENOMEM; - goto err; - } - - /* copy_from_user is limited to < 4GiB */ - copied = 0; - do { - unsigned int len = - min_t(u64, BIT_ULL(31), size - copied); - - if (__copy_from_user((char *)relocs + copied, - (char __user *)urelocs + copied, - len)) - goto end; - - copied += len; - } while (copied < size); - - /* - * As we do not update the known relocation offsets after - * relocating (due to the complexities in lock handling), - * we need to mark them as invalid now so that we force the - * relocation processing next time. Just in case the target - * object is evicted and then rebound into its old - * presumed_offset before the next execbuffer - if that - * happened we would make the mistake of assuming that the - * relocations were valid. - */ - if (!user_access_begin(urelocs, size)) - goto end; - - for (copied = 0; copied < nreloc; copied++) - unsafe_put_user(-1, - &urelocs[copied].presumed_offset, - end_user); - user_access_end(); - - eb->exec[i].relocs_ptr = (uintptr_t)relocs; - } - - return 0; - -end_user: - user_access_end(); -end: - kvfree(relocs); - err = -EFAULT; -err: - while (i--) { - relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); - if (eb->exec[i].relocation_count) - kvfree(relocs); - } - return err; -} - -static int eb_prefault_relocations(const struct i915_execbuffer *eb) -{ - const unsigned int count = eb->buffer_count; - unsigned int i; - - for (i = 0; i < count; i++) { - int err; + err = eb_lookup_vmas(eb); + if (err) + return err; - err = check_relocations(&eb->exec[i]); + if (!list_empty(&eb->unbound)) { + err = eb_reserve(eb); if (err) return err; } - return 0; -} - -static noinline int eb_relocate_slow(struct i915_execbuffer *eb) -{ - struct drm_device *dev = &eb->i915->drm; - bool have_copy = false; - struct i915_vma *vma; - int err = 0; - -repeat: - if (signal_pending(current)) { - err = -ERESTARTSYS; - goto out; - } - - /* We may process another execbuffer during the unlock... */ - eb_reset_vmas(eb); - mutex_unlock(&dev->struct_mutex); - - /* - * We take 3 passes through the slowpatch. - * - * 1 - we try to just prefault all the user relocation entries and - * then attempt to reuse the atomic pagefault disabled fast path again. - * - * 2 - we copy the user entries to a local buffer here outside of the - * local and allow ourselves to wait upon any rendering before - * relocations - * - * 3 - we already have a local copy of the relocation entries, but - * were interrupted (EAGAIN) whilst waiting for the objects, try again. - */ - if (!err) { - err = eb_prefault_relocations(eb); - } else if (!have_copy) { - err = eb_copy_relocations(eb); - have_copy = err == 0; - } else { - cond_resched(); - err = 0; - } - if (err) { - mutex_lock(&dev->struct_mutex); - goto out; - } - - /* A frequent cause for EAGAIN are currently unavailable client pages */ - flush_workqueue(eb->i915->mm.userptr_wq); - - err = i915_mutex_lock_interruptible(dev); - if (err) { - mutex_lock(&dev->struct_mutex); - goto out; - } - - /* reacquire the objects */ - err = eb_lookup_vmas(eb); - if (err) - goto err; - - GEM_BUG_ON(!eb->batch); + /* The objects are in their final locations, apply the relocations. */ + if (eb->args->flags & __EXEC_HAS_RELOC) { + struct eb_vma *ev; + int flush; - list_for_each_entry(vma, &eb->relocs, reloc_link) { - if (!have_copy) { - pagefault_disable(); - err = eb_relocate_vma(eb, vma); - pagefault_enable(); - if (err) - goto repeat; - } else { - err = eb_relocate_vma_slow(eb, vma); + list_for_each_entry(ev, &eb->relocs, reloc_link) { + err = eb_relocate_vma(eb, ev); if (err) - goto err; + break; } - } - /* - * Leave the user relocations as are, this is the painfully slow path, - * and we want to avoid the complication of dropping the lock whilst - * having buffers reserved in the aperture and so causing spurious - * ENOSPC for random operations. - */ - -err: - if (err == -EAGAIN) - goto repeat; - -out: - if (have_copy) { - const unsigned int count = eb->buffer_count; - unsigned int i; - - for (i = 0; i < count; i++) { - const struct drm_i915_gem_exec_object2 *entry = - &eb->exec[i]; - struct drm_i915_gem_relocation_entry *relocs; - - if (!entry->relocation_count) - continue; - - relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr); - kvfree(relocs); - } + flush = reloc_gpu_flush(&eb->reloc_cache); + if (!err) + err = flush; } return err; } -static int eb_relocate(struct i915_execbuffer *eb) -{ - if (eb_lookup_vmas(eb)) - goto slow; - - /* The objects are in their final locations, apply the relocations. */ - if (eb->args->flags & __EXEC_HAS_RELOC) { - struct i915_vma *vma; - - list_for_each_entry(vma, &eb->relocs, reloc_link) { - if (eb_relocate_vma(eb, vma)) - goto slow; - } - } - - return 0; - -slow: - return eb_relocate_slow(eb); -} - static int eb_move_to_gpu(struct i915_execbuffer *eb) { const unsigned int count = eb->buffer_count; @@ -1794,27 +1797,19 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) ww_acquire_init(&acquire, &reservation_ww_class); for (i = 0; i < count; i++) { - struct i915_vma *vma = eb->vma[i]; + struct eb_vma *ev = &eb->vma[i]; + struct i915_vma *vma = ev->vma; err = ww_mutex_lock_interruptible(&vma->resv->lock, &acquire); - if (!err) - continue; - - GEM_BUG_ON(err == -EALREADY); /* No duplicate vma */ - if (err == -EDEADLK) { GEM_BUG_ON(i == 0); do { int j = i - 1; - ww_mutex_unlock(&eb->vma[j]->resv->lock); + ww_mutex_unlock(&eb->vma[j].vma->resv->lock); - swap(eb->flags[i], eb->flags[j]); swap(eb->vma[i], eb->vma[j]); - eb->vma[i]->exec_flags = &eb->flags[i]; } while (--i); - GEM_BUG_ON(vma != eb->vma[0]); - vma->exec_flags = &eb->flags[0]; err = ww_mutex_lock_slow_interruptible(&vma->resv->lock, &acquire); @@ -1825,8 +1820,9 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) ww_acquire_done(&acquire); while (i--) { - unsigned int flags = eb->flags[i]; - struct i915_vma *vma = eb->vma[i]; + struct eb_vma *ev = &eb->vma[i]; + struct i915_vma *vma = ev->vma; + unsigned int flags = ev->flags; struct drm_i915_gem_object *obj = vma->obj; assert_vma_held(vma); @@ -1868,26 +1864,21 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) err = i915_vma_move_to_active(vma, eb->request, flags); i915_vma_unlock(vma); - - __eb_unreserve_vma(vma, flags); - vma->exec_flags = NULL; - - if (unlikely(flags & __EXEC_OBJECT_HAS_REF)) - i915_vma_put(vma); + eb_unreserve_vma(ev); } ww_acquire_fini(&acquire); + eb_vma_array_put(fetch_and_zero(&eb->array)); + if (unlikely(err)) goto err_skip; - eb->exec = NULL; - /* Unconditionally flush any chipset caches (for streaming writes). */ intel_gt_chipset_flush(eb->engine->gt); return 0; err_skip: - i915_request_skip(eb->request, err); + i915_request_set_error_once(eb->request, err); return err; } @@ -2008,7 +1999,7 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, if (!pw) return -ENOMEM; - err = i915_active_acquire(&eb->batch->active); + err = i915_active_acquire(&eb->batch->vma->active); if (err) goto err_free; @@ -2025,7 +2016,7 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, dma_fence_work_init(&pw->base, &eb_parse_ops); pw->engine = eb->engine; - pw->batch = eb->batch; + pw->batch = eb->batch->vma; pw->batch_offset = eb->batch_start_offset; pw->batch_length = eb->batch_len; pw->shadow = shadow; @@ -2056,7 +2047,7 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, dma_resv_add_excl_fence(shadow->resv, &pw->base.dma); dma_resv_unlock(shadow->resv); - dma_fence_work_commit(&pw->base); + dma_fence_work_commit_imm(&pw->base); return 0; err_batch_unlock: @@ -2067,7 +2058,7 @@ err_trampoline: err_shadow: i915_active_release(&shadow->active); err_batch: - i915_active_release(&eb->batch->active); + i915_active_release(&eb->batch->vma->active); err_free: kfree(pw); return err; @@ -2076,7 +2067,7 @@ err_free: static int eb_parse(struct i915_execbuffer *eb) { struct drm_i915_private *i915 = eb->i915; - struct intel_engine_pool_node *pool; + struct intel_gt_buffer_pool_node *pool; struct i915_vma *shadow, *trampoline; unsigned int len; int err; @@ -2099,7 +2090,7 @@ static int eb_parse(struct i915_execbuffer *eb) len += I915_CMD_PARSER_TRAMPOLINE_SIZE; } - pool = intel_engine_get_pool(eb->engine, len); + pool = intel_gt_get_buffer_pool(eb->engine->gt, len); if (IS_ERR(pool)) return PTR_ERR(pool); @@ -2130,15 +2121,13 @@ static int eb_parse(struct i915_execbuffer *eb) if (err) goto err_trampoline; - eb->vma[eb->buffer_count] = i915_vma_get(shadow); - eb->flags[eb->buffer_count] = - __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF; - shadow->exec_flags = &eb->flags[eb->buffer_count]; - eb->buffer_count++; + eb->vma[eb->buffer_count].vma = i915_vma_get(shadow); + eb->vma[eb->buffer_count].flags = __EXEC_OBJECT_HAS_PIN; + eb->batch = &eb->vma[eb->buffer_count++]; + eb->vma[eb->buffer_count].vma = NULL; eb->trampoline = trampoline; eb->batch_start_offset = 0; - eb->batch = shadow; shadow->private = pool; return 0; @@ -2149,7 +2138,7 @@ err_trampoline: err_shadow: i915_vma_unpin(shadow); err: - intel_engine_pool_put(pool); + intel_gt_buffer_pool_put(pool); return err; } @@ -2165,7 +2154,7 @@ add_to_client(struct i915_request *rq, struct drm_file *file) spin_unlock(&file_priv->mm.lock); } -static int eb_submit(struct i915_execbuffer *eb) +static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) { int err; @@ -2192,7 +2181,7 @@ static int eb_submit(struct i915_execbuffer *eb) } err = eb->engine->emit_bb_start(eb->request, - eb->batch->node.start + + batch->node.start + eb->batch_start_offset, eb->batch_len, eb->batch_flags); @@ -2327,15 +2316,22 @@ static int __eb_pin_engine(struct i915_execbuffer *eb, struct intel_context *ce) intel_context_timeline_unlock(tl); if (rq) { - if (i915_request_wait(rq, - I915_WAIT_INTERRUPTIBLE, - MAX_SCHEDULE_TIMEOUT) < 0) { - i915_request_put(rq); - err = -EINTR; - goto err_exit; - } + bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; + long timeout; + timeout = MAX_SCHEDULE_TIMEOUT; + if (nonblock) + timeout = 0; + + timeout = i915_request_wait(rq, + I915_WAIT_INTERRUPTIBLE, + timeout); i915_request_put(rq); + + if (timeout < 0) { + err = nonblock ? -EWOULDBLOCK : timeout; + goto err_exit; + } } eb->engine = ce->engine; @@ -2560,6 +2556,47 @@ signal_fence_array(struct i915_execbuffer *eb, } } +static void retire_requests(struct intel_timeline *tl, struct i915_request *end) +{ + struct i915_request *rq, *rn; + + list_for_each_entry_safe(rq, rn, &tl->requests, link) + if (rq == end || !i915_request_retire(rq)) + break; +} + +static void eb_request_add(struct i915_execbuffer *eb) +{ + struct i915_request *rq = eb->request; + struct intel_timeline * const tl = i915_request_timeline(rq); + struct i915_sched_attr attr = {}; + struct i915_request *prev; + + lockdep_assert_held(&tl->mutex); + lockdep_unpin_lock(&tl->mutex, rq->cookie); + + trace_i915_request_add(rq); + + prev = __i915_request_commit(rq); + + /* Check that the context wasn't destroyed before submission */ + if (likely(!intel_context_is_closed(eb->context))) { + attr = eb->gem_context->sched; + } else { + /* Serialise with context_close via the add_to_timeline */ + i915_request_set_error_once(rq, -ENOENT); + __i915_request_skip(rq); + } + + __i915_request_queue(rq, &attr); + + /* Try to clean up the client's timeline after submitting the request */ + if (prev) + retire_requests(tl, prev); + + mutex_unlock(&tl->mutex); +} + static int i915_gem_do_execbuffer(struct drm_device *dev, struct drm_file *file, @@ -2570,8 +2607,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, struct drm_i915_private *i915 = to_i915(dev); struct i915_execbuffer eb; struct dma_fence *in_fence = NULL; - struct dma_fence *exec_fence = NULL; struct sync_file *out_fence = NULL; + struct i915_vma *batch; int out_fence_fd = -1; int err; @@ -2586,9 +2623,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, args->flags |= __EXEC_HAS_RELOC; eb.exec = exec; - eb.vma = (struct i915_vma **)(exec + args->buffer_count + 1); - eb.vma[0] = NULL; - eb.flags = (unsigned int *)(eb.vma + args->buffer_count + 1); eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; reloc_cache_init(&eb.reloc_cache, eb.i915); @@ -2615,30 +2649,22 @@ i915_gem_do_execbuffer(struct drm_device *dev, if (args->flags & I915_EXEC_IS_PINNED) eb.batch_flags |= I915_DISPATCH_PINNED; - if (args->flags & I915_EXEC_FENCE_IN) { +#define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT) + if (args->flags & IN_FENCES) { + if ((args->flags & IN_FENCES) == IN_FENCES) + return -EINVAL; + in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); if (!in_fence) return -EINVAL; } - - if (args->flags & I915_EXEC_FENCE_SUBMIT) { - if (in_fence) { - err = -EINVAL; - goto err_in_fence; - } - - exec_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); - if (!exec_fence) { - err = -EINVAL; - goto err_in_fence; - } - } +#undef IN_FENCES if (args->flags & I915_EXEC_FENCE_OUT) { out_fence_fd = get_unused_fd_flags(O_CLOEXEC); if (out_fence_fd < 0) { err = out_fence_fd; - goto err_exec_fence; + goto err_in_fence; } } @@ -2656,10 +2682,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, if (unlikely(err)) goto err_context; - err = i915_mutex_lock_interruptible(dev); - if (err) - goto err_engine; - err = eb_relocate(&eb); if (err) { /* @@ -2673,21 +2695,23 @@ i915_gem_do_execbuffer(struct drm_device *dev, goto err_vma; } - if (unlikely(*eb.batch->exec_flags & EXEC_OBJECT_WRITE)) { + if (unlikely(eb.batch->flags & EXEC_OBJECT_WRITE)) { drm_dbg(&i915->drm, "Attempting to use self-modifying batch buffer\n"); err = -EINVAL; goto err_vma; } - if (eb.batch_start_offset > eb.batch->size || - eb.batch_len > eb.batch->size - eb.batch_start_offset) { + + if (range_overflows_t(u64, + eb.batch_start_offset, eb.batch_len, + eb.batch->vma->size)) { drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); err = -EINVAL; goto err_vma; } if (eb.batch_len == 0) - eb.batch_len = eb.batch->size - eb.batch_start_offset; + eb.batch_len = eb.batch->vma->size - eb.batch_start_offset; err = eb_parse(&eb); if (err) @@ -2697,6 +2721,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure * batch" bit. Hence we need to pin secure batches into the global gtt. * hsw should have this fixed, but bdw mucks it up again. */ + batch = eb.batch->vma; if (eb.batch_flags & I915_DISPATCH_SECURE) { struct i915_vma *vma; @@ -2710,13 +2735,13 @@ i915_gem_do_execbuffer(struct drm_device *dev, * fitting due to fragmentation. * So this is actually safe. */ - vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0); + vma = i915_gem_object_ggtt_pin(batch->obj, NULL, 0, 0, 0); if (IS_ERR(vma)) { err = PTR_ERR(vma); - goto err_vma; + goto err_parse; } - eb.batch = vma; + batch = vma; } /* All GPU relocation batches must be submitted prior to the user rq */ @@ -2730,14 +2755,13 @@ i915_gem_do_execbuffer(struct drm_device *dev, } if (in_fence) { - err = i915_request_await_dma_fence(eb.request, in_fence); - if (err < 0) - goto err_request; - } - - if (exec_fence) { - err = i915_request_await_execution(eb.request, exec_fence, - eb.engine->bond_execute); + if (args->flags & I915_EXEC_FENCE_SUBMIT) + err = i915_request_await_execution(eb.request, + in_fence, + eb.engine->bond_execute); + else + err = i915_request_await_dma_fence(eb.request, + in_fence); if (err < 0) goto err_request; } @@ -2763,16 +2787,16 @@ i915_gem_do_execbuffer(struct drm_device *dev, * inactive_list and lose its active reference. Hence we do not need * to explicitly hold another reference here. */ - eb.request->batch = eb.batch; - if (eb.batch->private) - intel_engine_pool_mark_active(eb.batch->private, eb.request); + eb.request->batch = batch; + if (batch->private) + intel_gt_buffer_pool_mark_active(batch->private, eb.request); trace_i915_request_queue(eb.request, eb.batch_flags); - err = eb_submit(&eb); + err = eb_submit(&eb, batch); err_request: add_to_client(eb.request, file); i915_request_get(eb.request); - i915_request_add(eb.request); + eb_request_add(&eb); if (fences) signal_fence_array(&eb, fences); @@ -2791,16 +2815,13 @@ err_request: err_batch_unpin: if (eb.batch_flags & I915_DISPATCH_SECURE) - i915_vma_unpin(eb.batch); - if (eb.batch->private) - intel_engine_pool_put(eb.batch->private); + i915_vma_unpin(batch); +err_parse: + if (batch->private) + intel_gt_buffer_pool_put(batch->private); err_vma: - if (eb.exec) - eb_release_vmas(&eb); if (eb.trampoline) i915_vma_unpin(eb.trampoline); - mutex_unlock(&dev->struct_mutex); -err_engine: eb_unpin_engine(&eb); err_context: i915_gem_context_put(eb.gem_context); @@ -2809,8 +2830,6 @@ err_destroy: err_out_fence: if (out_fence_fd != -1) put_unused_fd(out_fence_fd); -err_exec_fence: - dma_fence_put(exec_fence); err_in_fence: dma_fence_put(in_fence); return err; @@ -2818,9 +2837,7 @@ err_in_fence: static size_t eb_element_size(void) { - return (sizeof(struct drm_i915_gem_exec_object2) + - sizeof(struct i915_vma *) + - sizeof(unsigned int)); + return sizeof(struct drm_i915_gem_exec_object2); } static bool check_buffer_count(size_t count) @@ -2876,7 +2893,7 @@ i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data, /* Copy in the exec list from userland */ exec_list = kvmalloc_array(count, sizeof(*exec_list), __GFP_NOWARN | GFP_KERNEL); - exec2_list = kvmalloc_array(count + 1, eb_element_size(), + exec2_list = kvmalloc_array(count, eb_element_size(), __GFP_NOWARN | GFP_KERNEL); if (exec_list == NULL || exec2_list == NULL) { drm_dbg(&i915->drm, @@ -2954,8 +2971,7 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, if (err) return err; - /* Allocate an extra slot for use by the command parser */ - exec2_list = kvmalloc_array(count + 1, eb_element_size(), + exec2_list = kvmalloc_array(count, eb_element_size(), __GFP_NOWARN | GFP_KERNEL); if (exec2_list == NULL) { drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n", @@ -3022,3 +3038,7 @@ end:; kvfree(exec2_list); return err; } + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftests/i915_gem_execbuffer.c" +#endif diff --git a/drivers/gpu/drm/i915/gem/i915_gem_fence.c b/drivers/gpu/drm/i915/gem/i915_gem_fence.c index 2f6100ec2608..8ab842c80f99 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_fence.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_fence.c @@ -72,8 +72,8 @@ i915_gem_object_lock_fence(struct drm_i915_gem_object *obj) 0, 0); if (i915_sw_fence_await_reservation(&stub->chain, - obj->base.resv, NULL, - true, I915_FENCE_TIMEOUT, + obj->base.resv, NULL, true, + i915_fence_timeout(to_i915(obj->base.dev)), I915_FENCE_GFP) < 0) goto err; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index 9cfb0e41ff06..cbbff81aa0af 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -8,8 +8,6 @@ #include <linux/slab.h> #include <linux/swiotlb.h> -#include <drm/i915_drm.h> - #include "i915_drv.h" #include "i915_gem.h" #include "i915_gem_object.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index e8cccc131c40..70f5f82da288 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -396,6 +396,38 @@ err: return i915_error_to_vmf_fault(ret); } +static int +vm_access(struct vm_area_struct *area, unsigned long addr, + void *buf, int len, int write) +{ + struct i915_mmap_offset *mmo = area->vm_private_data; + struct drm_i915_gem_object *obj = mmo->obj; + void *vaddr; + + if (i915_gem_object_is_readonly(obj) && write) + return -EACCES; + + addr -= area->vm_start; + if (addr >= obj->base.size) + return -EINVAL; + + /* As this is primarily for debugging, let's focus on simplicity */ + vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); + + if (write) { + memcpy(vaddr + addr, buf, len); + __i915_gem_object_flush_map(obj, addr, len); + } else { + memcpy(buf, vaddr + addr, len); + } + + i915_gem_object_unpin_map(obj); + + return len; +} + void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) { struct i915_vma *vma; @@ -745,12 +777,14 @@ static void vm_close(struct vm_area_struct *vma) static const struct vm_operations_struct vm_ops_gtt = { .fault = vm_fault_gtt, + .access = vm_access, .open = vm_open, .close = vm_close, }; static const struct vm_operations_struct vm_ops_cpu = { .fault = vm_fault_cpu, + .access = vm_access, .open = vm_open, .close = vm_close, }; @@ -775,7 +809,7 @@ static struct file *mmap_singleton(struct drm_i915_private *i915) struct file *file; rcu_read_lock(); - file = i915->gem.mmap_singleton; + file = READ_ONCE(i915->gem.mmap_singleton); if (file && !get_file_rcu(file)) file = NULL; rcu_read_unlock(); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c index 35985218bd85..99356c00c19e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c @@ -135,9 +135,7 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file) if (vma) { GEM_BUG_ON(vma->obj != obj); GEM_BUG_ON(!atomic_read(&vma->open_count)); - if (atomic_dec_and_test(&vma->open_count) && - !i915_vma_is_ggtt(vma)) - i915_vma_close(vma); + i915_vma_close(vma); } mutex_unlock(&ctx->mutex); @@ -164,9 +162,7 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, struct llist_node *freed) { struct drm_i915_gem_object *obj, *on; - intel_wakeref_t wakeref; - wakeref = intel_runtime_pm_get(&i915->runtime_pm); llist_for_each_entry_safe(obj, on, freed, freed) { struct i915_mmap_offset *mmo, *mn; @@ -206,7 +202,6 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, } obj->mmo.offsets = RB_ROOT; - GEM_BUG_ON(atomic_read(&obj->bind_count)); GEM_BUG_ON(obj->userfault_count); GEM_BUG_ON(!list_empty(&obj->lut_list)); @@ -225,8 +220,8 @@ static void __i915_gem_free_objects(struct drm_i915_private *i915, /* But keep the pointer alive for RCU-protected lookups */ call_rcu(&obj->rcu, __i915_gem_free_object_rcu); + cond_resched(); } - intel_runtime_pm_put(&i915->runtime_pm, wakeref); } void i915_gem_flush_free_objects(struct drm_i915_private *i915) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index e44a2f40b520..2faa481cc18f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -11,8 +11,6 @@ #include <drm/drm_file.h> #include <drm/drm_device.h> -#include <drm/i915_drm.h> - #include "display/intel_frontbuffer.h" #include "i915_gem_object_types.h" #include "i915_gem_gtt.h" diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c index 70809d8897cd..f457d7130491 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.c @@ -6,8 +6,8 @@ #include "i915_drv.h" #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" -#include "gt/intel_engine_pool.h" #include "gt/intel_gt.h" +#include "gt/intel_gt_buffer_pool.h" #include "gt/intel_ring.h" #include "i915_gem_clflush.h" #include "i915_gem_object_blt.h" @@ -18,7 +18,7 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, { struct drm_i915_private *i915 = ce->vm->i915; const u32 block_size = SZ_8M; /* ~1ms at 8GiB/s preemption delay */ - struct intel_engine_pool_node *pool; + struct intel_gt_buffer_pool_node *pool; struct i915_vma *batch; u64 offset; u64 count; @@ -33,7 +33,7 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, count = div_u64(round_up(vma->size, block_size), block_size); size = (1 + 8 * count) * sizeof(u32); size = round_up(size, PAGE_SIZE); - pool = intel_engine_get_pool(ce->engine, size); + pool = intel_gt_get_buffer_pool(ce->engine->gt, size); if (IS_ERR(pool)) { err = PTR_ERR(pool); goto out_pm; @@ -78,10 +78,12 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, } while (rem); *cmd = MI_BATCH_BUFFER_END; - intel_gt_chipset_flush(ce->vm->gt); + i915_gem_object_flush_map(pool->obj); i915_gem_object_unpin_map(pool->obj); + intel_gt_chipset_flush(ce->vm->gt); + batch = i915_vma_instance(pool->obj, ce->vm, NULL); if (IS_ERR(batch)) { err = PTR_ERR(batch); @@ -96,7 +98,7 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, return batch; out_put: - intel_engine_pool_put(pool); + intel_gt_buffer_pool_put(pool); out_pm: intel_engine_pm_put(ce->engine); return ERR_PTR(err); @@ -114,13 +116,13 @@ int intel_emit_vma_mark_active(struct i915_vma *vma, struct i915_request *rq) if (unlikely(err)) return err; - return intel_engine_pool_mark_active(vma->private, rq); + return intel_gt_buffer_pool_mark_active(vma->private, rq); } void intel_emit_vma_release(struct intel_context *ce, struct i915_vma *vma) { i915_vma_unpin(vma); - intel_engine_pool_put(vma->private); + intel_gt_buffer_pool_put(vma->private); intel_engine_pm_put(ce->engine); } @@ -186,7 +188,7 @@ int i915_gem_object_fill_blt(struct drm_i915_gem_object *obj, 0); out_request: if (unlikely(err)) - i915_request_skip(rq, err); + i915_request_set_error_once(rq, err); i915_request_add(rq); out_batch: @@ -196,13 +198,24 @@ out_unpin: return err; } +/* Wa_1209644611:icl,ehl */ +static bool wa_1209644611_applies(struct drm_i915_private *i915, u32 size) +{ + u32 height = size >> PAGE_SHIFT; + + if (!IS_GEN(i915, 11)) + return false; + + return height % 4 == 3 && height <= 8; +} + struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, struct i915_vma *src, struct i915_vma *dst) { struct drm_i915_private *i915 = ce->vm->i915; const u32 block_size = SZ_8M; /* ~1ms at 8GiB/s preemption delay */ - struct intel_engine_pool_node *pool; + struct intel_gt_buffer_pool_node *pool; struct i915_vma *batch; u64 src_offset, dst_offset; u64 count, rem; @@ -217,7 +230,7 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, count = div_u64(round_up(dst->size, block_size), block_size); size = (1 + 11 * count) * sizeof(u32); size = round_up(size, PAGE_SIZE); - pool = intel_engine_get_pool(ce->engine, size); + pool = intel_gt_get_buffer_pool(ce->engine->gt, size); if (IS_ERR(pool)) { err = PTR_ERR(pool); goto out_pm; @@ -237,7 +250,8 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, size = min_t(u64, rem, block_size); GEM_BUG_ON(size >> PAGE_SHIFT > S16_MAX); - if (INTEL_GEN(i915) >= 9) { + if (INTEL_GEN(i915) >= 9 && + !wa_1209644611_applies(i915, size)) { *cmd++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); *cmd++ = BLT_DEPTH_32 | PAGE_SIZE; *cmd++ = 0; @@ -277,10 +291,12 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, } while (rem); *cmd = MI_BATCH_BUFFER_END; - intel_gt_chipset_flush(ce->vm->gt); + i915_gem_object_flush_map(pool->obj); i915_gem_object_unpin_map(pool->obj); + intel_gt_chipset_flush(ce->vm->gt); + batch = i915_vma_instance(pool->obj, ce->vm, NULL); if (IS_ERR(batch)) { err = PTR_ERR(batch); @@ -295,7 +311,7 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, return batch; out_put: - intel_engine_pool_put(pool); + intel_gt_buffer_pool_put(pool); out_pm: intel_engine_pm_put(ce->engine); return ERR_PTR(err); @@ -385,7 +401,7 @@ out_unlock: drm_gem_unlock_reservations(objs, ARRAY_SIZE(objs), &acquire); out_request: if (unlikely(err)) - i915_request_skip(rq, err); + i915_request_set_error_once(rq, err); i915_request_add(rq); out_batch: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h index 243a43a87824..8bcd336a90dc 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_blt.h @@ -10,7 +10,6 @@ #include "gt/intel_context.h" #include "gt/intel_engine_pm.h" -#include "gt/intel_engine_pool.h" #include "i915_vma.h" struct drm_i915_gem_object; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h index a0b10bcd8d8a..54ee658bb168 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h @@ -179,9 +179,6 @@ struct drm_i915_gem_object { #define TILING_MASK (FENCE_MINIMUM_STRIDE - 1) #define STRIDE_MASK (~TILING_MASK) - /** Count of VMA actually bound by this object */ - atomic_t bind_count; - struct { /* * Protects the pages and their use. Do not use directly, but diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 24f4cadea114..af9e48ee4a33 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -199,8 +199,6 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) if (i915_gem_object_has_pinned_pages(obj)) return -EBUSY; - GEM_BUG_ON(atomic_read(&obj->bind_count)); - /* May be called by shrinker from within get_pages() (on another bo) */ mutex_lock(&obj->mm.lock); if (unlikely(atomic_read(&obj->mm.pages_pin_count))) { @@ -393,6 +391,7 @@ void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj, GEM_BUG_ON(range_overflows_t(typeof(obj->base.size), offset, size, obj->base.size)); + wmb(); /* let all previous writes be visible to coherent partners */ obj->mm.dirty = true; if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c index b07bb40edd5a..698e22420dc5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c @@ -194,10 +194,11 @@ int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) /* Perma-pin (until release) the physical set of pages */ __i915_gem_object_pin_pages(obj); - if (!IS_ERR_OR_NULL(pages)) { + if (!IS_ERR_OR_NULL(pages)) i915_gem_shmem_ops.put_pages(obj, pages); - i915_gem_object_release_memory_region(obj); - } + + i915_gem_object_release_memory_region(obj); + mutex_unlock(&obj->mm.lock); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index 7eaa2ab01de3..5b65ce738b16 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -12,7 +12,6 @@ #include <linux/pci.h> #include <linux/dma-buf.h> #include <linux/vmalloc.h> -#include <drm/i915_drm.h> #include "i915_trace.h" @@ -28,18 +27,6 @@ static bool can_release_pages(struct drm_i915_gem_object *obj) return false; /* - * Only report true if by unbinding the object and putting its pages - * we can actually make forward progress towards freeing physical - * pages. - * - * If the pages are pinned for any other reason than being bound - * to the GPU, simply unbinding from the GPU is not going to succeed - * in releasing our pin count on the pages themselves. - */ - if (atomic_read(&obj->mm.pages_pin_count) > atomic_read(&obj->bind_count)) - return false; - - /* * We can only return physical pages to the system if we can either * discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. @@ -55,6 +42,8 @@ static bool unsafe_drop_pages(struct drm_i915_gem_object *obj, flags = 0; if (shrink & I915_SHRINK_ACTIVE) flags = I915_GEM_OBJECT_UNBIND_ACTIVE; + if (!(shrink & I915_SHRINK_BOUND)) + flags = I915_GEM_OBJECT_UNBIND_TEST; if (i915_gem_object_unbind(obj, flags) == 0) __i915_gem_object_put_pages(obj); @@ -195,10 +184,6 @@ i915_gem_shrink(struct drm_i915_private *i915, i915_gem_object_is_framebuffer(obj)) continue; - if (!(shrink & I915_SHRINK_BOUND) && - atomic_read(&obj->bind_count)) - continue; - if (!can_release_pages(obj)) continue; @@ -256,8 +241,7 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *i915) with_intel_runtime_pm(&i915->runtime_pm, wakeref) { freed = i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_BOUND | - I915_SHRINK_UNBOUND | - I915_SHRINK_ACTIVE); + I915_SHRINK_UNBOUND); } return freed; @@ -336,7 +320,6 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) freed_pages = 0; with_intel_runtime_pm(&i915->runtime_pm, wakeref) freed_pages += i915_gem_shrink(i915, -1UL, NULL, - I915_SHRINK_ACTIVE | I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_WRITEBACK); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 491cfbaaa330..dc250278bd2c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -13,6 +13,7 @@ #include "gem/i915_gem_region.h" #include "i915_drv.h" #include "i915_gem_stolen.h" +#include "i915_vgpu.h" /* * The BIOS typically reserves some of the system's memory for the exclusive @@ -380,14 +381,14 @@ static int i915_gem_init_stolen(struct drm_i915_private *i915) mutex_init(&i915->mm.stolen_lock); if (intel_vgpu_active(i915)) { - dev_notice(i915->drm.dev, + drm_notice(&i915->drm, "%s, disabling use of stolen memory\n", "iGVT-g active"); return 0; } if (intel_vtd_active() && INTEL_GEN(i915) < 8) { - dev_notice(i915->drm.dev, + drm_notice(&i915->drm, "%s, disabling use of stolen memory\n", "DMAR active"); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c index 6c7825a2dc2a..0158e49bf9bb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c @@ -6,7 +6,6 @@ #include <linux/string.h> #include <linux/bitops.h> -#include <drm/i915_drm.h> #include "i915_drv.h" #include "i915_gem.h" @@ -183,21 +182,35 @@ i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, int tiling_mode, unsigned int stride) { struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt; - struct i915_vma *vma; + struct i915_vma *vma, *vn; + LIST_HEAD(unbind); int ret = 0; if (tiling_mode == I915_TILING_NONE) return 0; mutex_lock(&ggtt->vm.mutex); + + spin_lock(&obj->vma.lock); for_each_ggtt_vma(vma, obj) { + GEM_BUG_ON(vma->vm != &ggtt->vm); + if (i915_vma_fence_prepare(vma, tiling_mode, stride)) continue; + list_move(&vma->vm_link, &unbind); + } + spin_unlock(&obj->vma.lock); + + list_for_each_entry_safe(vma, vn, &unbind, vm_link) { ret = __i915_vma_unbind(vma); - if (ret) + if (ret) { + /* Restore the remaining vma on an error */ + list_splice(&unbind, &ggtt->vm.bound_list); break; + } } + mutex_unlock(&ggtt->vm.mutex); return ret; @@ -269,6 +282,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, } mutex_unlock(&obj->mm.lock); + spin_lock(&obj->vma.lock); for_each_ggtt_vma(vma, obj) { vma->fence_size = i915_gem_fence_size(i915, vma->size, tiling, stride); @@ -279,6 +293,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, if (vma->fence) vma->fence->dirty = true; } + spin_unlock(&obj->vma.lock); obj->tiling_and_stride = tiling | stride; i915_gem_object_unlock(obj); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 63ead7a2b64a..7ffd7afeb7a5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -10,8 +10,6 @@ #include <linux/swap.h> #include <linux/sched/mm.h> -#include <drm/i915_drm.h> - #include "i915_drv.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c index fa16f2c3f3ac..2b46c6530da9 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c @@ -88,8 +88,7 @@ static void huge_put_pages(struct drm_i915_gem_object *obj, } static const struct drm_i915_gem_object_ops huge_ops = { - .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | - I915_GEM_OBJECT_IS_SHRINKABLE, + .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE, .get_pages = huge_get_pages, .put_pages = huge_put_pages, }; diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index 2d0fd50c5312..c9988b6d5c88 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -421,7 +421,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) - goto out_close; + goto out_put; err = igt_check_page_sizes(vma); @@ -432,8 +432,6 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) } i915_vma_unpin(vma); - i915_vma_close(vma); - i915_gem_object_put(obj); if (err) @@ -443,8 +441,6 @@ static int igt_mock_exhaust_device_supported_pages(void *arg) goto out_device; -out_close: - i915_vma_close(vma); out_put: i915_gem_object_put(obj); out_device: @@ -492,7 +488,7 @@ static int igt_mock_memory_region_huge_pages(void *arg) err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) - goto out_close; + goto out_put; err = igt_check_page_sizes(vma); if (err) @@ -515,8 +511,6 @@ static int igt_mock_memory_region_huge_pages(void *arg) } i915_vma_unpin(vma); - i915_vma_close(vma); - __i915_gem_object_put_pages(obj); i915_gem_object_put(obj); } @@ -526,8 +520,6 @@ static int igt_mock_memory_region_huge_pages(void *arg) out_unpin: i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); out_put: i915_gem_object_put(obj); out_region: @@ -587,10 +579,8 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) } err = i915_vma_pin(vma, 0, 0, flags); - if (err) { - i915_vma_close(vma); + if (err) goto out_unpin; - } err = igt_check_page_sizes(vma); @@ -603,10 +593,8 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) i915_vma_unpin(vma); - if (err) { - i915_vma_close(vma); + if (err) goto out_unpin; - } /* * Try all the other valid offsets until the next @@ -615,16 +603,12 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) */ for (offset = 4096; offset < page_size; offset += 4096) { err = i915_vma_unbind(vma); - if (err) { - i915_vma_close(vma); + if (err) goto out_unpin; - } err = i915_vma_pin(vma, 0, 0, flags | offset); - if (err) { - i915_vma_close(vma); + if (err) goto out_unpin; - } err = igt_check_page_sizes(vma); @@ -636,10 +620,8 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) i915_vma_unpin(vma); - if (err) { - i915_vma_close(vma); + if (err) goto out_unpin; - } if (igt_timeout(end_time, "%s timed out at offset %x with page-size %x\n", @@ -647,8 +629,6 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) break; } - i915_vma_close(vma); - i915_gem_object_unpin_pages(obj); __i915_gem_object_put_pages(obj); i915_gem_object_put(obj); @@ -670,12 +650,6 @@ static void close_object_list(struct list_head *objects, struct drm_i915_gem_object *obj, *on; list_for_each_entry_safe(obj, on, objects, st_link) { - struct i915_vma *vma; - - vma = i915_vma_instance(obj, &ppgtt->vm, NULL); - if (!IS_ERR(vma)) - i915_vma_close(vma); - list_del(&obj->st_link); i915_gem_object_unpin_pages(obj); __i915_gem_object_put_pages(obj); @@ -912,7 +886,7 @@ static int igt_mock_ppgtt_64K(void *arg) err = i915_vma_pin(vma, 0, 0, flags); if (err) - goto out_vma_close; + goto out_object_unpin; err = igt_check_page_sizes(vma); if (err) @@ -945,8 +919,6 @@ static int igt_mock_ppgtt_64K(void *arg) } i915_vma_unpin(vma); - i915_vma_close(vma); - i915_gem_object_unpin_pages(obj); __i915_gem_object_put_pages(obj); i915_gem_object_put(obj); @@ -957,8 +929,6 @@ static int igt_mock_ppgtt_64K(void *arg) out_vma_unpin: i915_vma_unpin(vma); -out_vma_close: - i915_vma_close(vma); out_object_unpin: i915_gem_object_unpin_pages(obj); out_object_put: @@ -1070,7 +1040,7 @@ static int __igt_write_huge(struct intel_context *ce, err = i915_vma_unbind(vma); if (err) - goto out_vma_close; + return err; err = i915_vma_pin(vma, size, 0, flags | offset); if (err) { @@ -1081,7 +1051,7 @@ static int __igt_write_huge(struct intel_context *ce, if (err == -ENOSPC && i915_is_ggtt(ce->vm)) err = 0; - goto out_vma_close; + return err; } err = igt_check_page_sizes(vma); @@ -1102,8 +1072,6 @@ static int __igt_write_huge(struct intel_context *ce, out_vma_unpin: i915_vma_unpin(vma); -out_vma_close: - __i915_vma_put(vma); return err; } @@ -1477,8 +1445,10 @@ static int igt_ppgtt_pin_update(void *arg) unsigned int page_size = BIT(first); obj = i915_gem_object_create_internal(dev_priv, page_size); - if (IS_ERR(obj)) - return PTR_ERR(obj); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto out_vm; + } vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { @@ -1488,7 +1458,7 @@ static int igt_ppgtt_pin_update(void *arg) err = i915_vma_pin(vma, SZ_2M, 0, flags); if (err) - goto out_close; + goto out_put; if (vma->page_sizes.sg < page_size) { pr_info("Unable to allocate page-size %x, finishing test early\n", @@ -1525,14 +1495,14 @@ static int igt_ppgtt_pin_update(void *arg) goto out_unpin; i915_vma_unpin(vma); - i915_vma_close(vma); - i915_gem_object_put(obj); } obj = i915_gem_object_create_internal(dev_priv, PAGE_SIZE); - if (IS_ERR(obj)) - return PTR_ERR(obj); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + goto out_vm; + } vma = i915_vma_instance(obj, vm, NULL); if (IS_ERR(vma)) { @@ -1542,7 +1512,7 @@ static int igt_ppgtt_pin_update(void *arg) err = i915_vma_pin(vma, 0, 0, flags); if (err) - goto out_close; + goto out_put; /* * Make sure we don't end up with something like where the pde is still @@ -1572,8 +1542,6 @@ static int igt_ppgtt_pin_update(void *arg) out_unpin: i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); out_put: i915_gem_object_put(obj); out_vm: @@ -1625,13 +1593,11 @@ static int igt_tmpfs_fallback(void *arg) err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) - goto out_close; + goto out_put; err = igt_check_page_sizes(vma); i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); out_put: i915_gem_object_put(obj); out_restore: @@ -1678,7 +1644,7 @@ static int igt_shrink_thp(void *arg) err = i915_vma_pin(vma, 0, 0, flags); if (err) - goto out_close; + goto out_put; if (obj->mm.page_sizes.phys < I915_GTT_PAGE_SIZE_2M) { pr_info("failed to allocate THP, finishing test early\n"); @@ -1702,7 +1668,7 @@ static int igt_shrink_thp(void *arg) i915_gem_context_unlock_engines(ctx); i915_vma_unpin(vma); if (err) - goto out_close; + goto out_put; /* * Now that the pages are *unpinned* shrink-all should invoke @@ -1712,18 +1678,18 @@ static int igt_shrink_thp(void *arg) if (i915_gem_object_has_pages(obj)) { pr_err("shrink-all didn't truncate the pages\n"); err = -EINVAL; - goto out_close; + goto out_put; } if (obj->mm.page_sizes.sg || obj->mm.page_sizes.phys) { pr_err("residual page-size bits left\n"); err = -EINVAL; - goto out_close; + goto out_put; } err = i915_vma_pin(vma, 0, 0, flags); if (err) - goto out_close; + goto out_put; while (n--) { err = cpu_check(obj, n, 0xdeadbeaf); @@ -1733,8 +1699,6 @@ static int igt_shrink_thp(void *arg) out_unpin: i915_vma_unpin(vma); -out_close: - i915_vma_close(vma); out_put: i915_gem_object_put(obj); out_vm: @@ -1773,21 +1737,20 @@ int i915_gem_huge_page_mock_selftests(void) if (!i915_vm_is_4lvl(&ppgtt->vm)) { pr_err("failed to create 48b PPGTT\n"); err = -EINVAL; - goto out_close; + goto out_put; } /* If we were ever hit this then it's time to mock the 64K scratch */ if (!i915_vm_has_scratch_64K(&ppgtt->vm)) { pr_err("PPGTT missing 64K scratch page\n"); err = -EINVAL; - goto out_close; + goto out_put; } err = i915_subtests(tests, ppgtt); -out_close: +out_put: i915_vm_put(&ppgtt->vm); - out_unlock: drm_dev_put(&dev_priv->drm); return err; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index b972be165e85..8fe3ad2ee34e 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -7,9 +7,12 @@ #include "gt/intel_engine_user.h" #include "gt/intel_gt.h" +#include "gt/intel_gpu_commands.h" +#include "gem/i915_gem_lmem.h" #include "selftests/igt_flush_test.h" #include "selftests/mock_drm.h" +#include "selftests/i915_random.h" #include "huge_gem_object.h" #include "mock_context.h" @@ -127,10 +130,573 @@ static int igt_client_fill(void *arg) } while (1); } +#define WIDTH 512 +#define HEIGHT 32 + +struct blit_buffer { + struct i915_vma *vma; + u32 start_val; + u32 tiling; +}; + +struct tiled_blits { + struct intel_context *ce; + struct blit_buffer buffers[3]; + struct blit_buffer scratch; + struct i915_vma *batch; + u64 hole; + u32 width; + u32 height; +}; + +static int prepare_blit(const struct tiled_blits *t, + struct blit_buffer *dst, + struct blit_buffer *src, + struct drm_i915_gem_object *batch) +{ + const int gen = INTEL_GEN(to_i915(batch->base.dev)); + bool use_64b_reloc = gen >= 8; + u32 src_pitch, dst_pitch; + u32 cmd, *cs; + + cs = i915_gem_object_pin_map(batch, I915_MAP_WC); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + *cs++ = MI_LOAD_REGISTER_IMM(1); + *cs++ = i915_mmio_reg_offset(BCS_SWCTRL); + cmd = (BCS_SRC_Y | BCS_DST_Y) << 16; + if (src->tiling == I915_TILING_Y) + cmd |= BCS_SRC_Y; + if (dst->tiling == I915_TILING_Y) + cmd |= BCS_DST_Y; + *cs++ = cmd; + + cmd = MI_FLUSH_DW; + if (gen >= 8) + cmd++; + *cs++ = cmd; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + + cmd = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (8 - 2); + if (gen >= 8) + cmd += 2; + + src_pitch = t->width * 4; + if (src->tiling) { + cmd |= XY_SRC_COPY_BLT_SRC_TILED; + src_pitch /= 4; + } + + dst_pitch = t->width * 4; + if (dst->tiling) { + cmd |= XY_SRC_COPY_BLT_DST_TILED; + dst_pitch /= 4; + } + + *cs++ = cmd; + *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | dst_pitch; + *cs++ = 0; + *cs++ = t->height << 16 | t->width; + *cs++ = lower_32_bits(dst->vma->node.start); + if (use_64b_reloc) + *cs++ = upper_32_bits(dst->vma->node.start); + *cs++ = 0; + *cs++ = src_pitch; + *cs++ = lower_32_bits(src->vma->node.start); + if (use_64b_reloc) + *cs++ = upper_32_bits(src->vma->node.start); + + *cs++ = MI_BATCH_BUFFER_END; + + i915_gem_object_flush_map(batch); + i915_gem_object_unpin_map(batch); + + return 0; +} + +static void tiled_blits_destroy_buffers(struct tiled_blits *t) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(t->buffers); i++) + i915_vma_put(t->buffers[i].vma); + + i915_vma_put(t->scratch.vma); + i915_vma_put(t->batch); +} + +static struct i915_vma * +__create_vma(struct tiled_blits *t, size_t size, bool lmem) +{ + struct drm_i915_private *i915 = t->ce->vm->i915; + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + + if (lmem) + obj = i915_gem_object_create_lmem(i915, size, 0); + else + obj = i915_gem_object_create_shmem(i915, size); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + vma = i915_vma_instance(obj, t->ce->vm, NULL); + if (IS_ERR(vma)) + i915_gem_object_put(obj); + + return vma; +} + +static struct i915_vma *create_vma(struct tiled_blits *t, bool lmem) +{ + return __create_vma(t, PAGE_ALIGN(t->width * t->height * 4), lmem); +} + +static int tiled_blits_create_buffers(struct tiled_blits *t, + int width, int height, + struct rnd_state *prng) +{ + struct drm_i915_private *i915 = t->ce->engine->i915; + int i; + + t->width = width; + t->height = height; + + t->batch = __create_vma(t, PAGE_SIZE, false); + if (IS_ERR(t->batch)) + return PTR_ERR(t->batch); + + t->scratch.vma = create_vma(t, false); + if (IS_ERR(t->scratch.vma)) { + i915_vma_put(t->batch); + return PTR_ERR(t->scratch.vma); + } + + for (i = 0; i < ARRAY_SIZE(t->buffers); i++) { + struct i915_vma *vma; + + vma = create_vma(t, HAS_LMEM(i915) && i % 2); + if (IS_ERR(vma)) { + tiled_blits_destroy_buffers(t); + return PTR_ERR(vma); + } + + t->buffers[i].vma = vma; + t->buffers[i].tiling = + i915_prandom_u32_max_state(I915_TILING_Y + 1, prng); + } + + return 0; +} + +static void fill_scratch(struct tiled_blits *t, u32 *vaddr, u32 val) +{ + int i; + + t->scratch.start_val = val; + for (i = 0; i < t->width * t->height; i++) + vaddr[i] = val++; + + i915_gem_object_flush_map(t->scratch.vma->obj); +} + +static u64 swizzle_bit(unsigned int bit, u64 offset) +{ + return (offset & BIT_ULL(bit)) >> (bit - 6); +} + +static u64 tiled_offset(const struct intel_gt *gt, + u64 v, + unsigned int stride, + unsigned int tiling) +{ + unsigned int swizzle; + u64 x, y; + + if (tiling == I915_TILING_NONE) + return v; + + y = div64_u64_rem(v, stride, &x); + + if (tiling == I915_TILING_X) { + v = div64_u64_rem(y, 8, &y) * stride * 8; + v += y * 512; + v += div64_u64_rem(x, 512, &x) << 12; + v += x; + + swizzle = gt->ggtt->bit_6_swizzle_x; + } else { + const unsigned int ytile_span = 16; + const unsigned int ytile_height = 512; + + v = div64_u64_rem(y, 32, &y) * stride * 32; + v += y * ytile_span; + v += div64_u64_rem(x, ytile_span, &x) * ytile_height; + v += x; + + swizzle = gt->ggtt->bit_6_swizzle_y; + } + + switch (swizzle) { + case I915_BIT_6_SWIZZLE_9: + v ^= swizzle_bit(9, v); + break; + case I915_BIT_6_SWIZZLE_9_10: + v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v); + break; + case I915_BIT_6_SWIZZLE_9_11: + v ^= swizzle_bit(9, v) ^ swizzle_bit(11, v); + break; + case I915_BIT_6_SWIZZLE_9_10_11: + v ^= swizzle_bit(9, v) ^ swizzle_bit(10, v) ^ swizzle_bit(11, v); + break; + } + + return v; +} + +static const char *repr_tiling(int tiling) +{ + switch (tiling) { + case I915_TILING_NONE: return "linear"; + case I915_TILING_X: return "X"; + case I915_TILING_Y: return "Y"; + default: return "unknown"; + } +} + +static int verify_buffer(const struct tiled_blits *t, + struct blit_buffer *buf, + struct rnd_state *prng) +{ + const u32 *vaddr; + int ret = 0; + int x, y, p; + + x = i915_prandom_u32_max_state(t->width, prng); + y = i915_prandom_u32_max_state(t->height, prng); + p = y * t->width + x; + + vaddr = i915_gem_object_pin_map(buf->vma->obj, I915_MAP_WC); + if (IS_ERR(vaddr)) + return PTR_ERR(vaddr); + + if (vaddr[0] != buf->start_val) { + ret = -EINVAL; + } else { + u64 v = tiled_offset(buf->vma->vm->gt, + p * 4, t->width * 4, + buf->tiling); + + if (vaddr[v / sizeof(*vaddr)] != buf->start_val + p) + ret = -EINVAL; + } + if (ret) { + pr_err("Invalid %s tiling detected at (%d, %d), start_val %x\n", + repr_tiling(buf->tiling), + x, y, buf->start_val); + igt_hexdump(vaddr, 4096); + } + + i915_gem_object_unpin_map(buf->vma->obj); + return ret; +} + +static int move_to_active(struct i915_vma *vma, + struct i915_request *rq, + unsigned int flags) +{ + int err; + + i915_vma_lock(vma); + err = i915_request_await_object(rq, vma->obj, false); + if (err == 0) + err = i915_vma_move_to_active(vma, rq, flags); + i915_vma_unlock(vma); + + return err; +} + +static int pin_buffer(struct i915_vma *vma, u64 addr) +{ + int err; + + if (drm_mm_node_allocated(&vma->node) && vma->node.start != addr) { + err = i915_vma_unbind(vma); + if (err) + return err; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_OFFSET_FIXED | addr); + if (err) + return err; + + return 0; +} + +static int +tiled_blit(struct tiled_blits *t, + struct blit_buffer *dst, u64 dst_addr, + struct blit_buffer *src, u64 src_addr) +{ + struct i915_request *rq; + int err; + + err = pin_buffer(src->vma, src_addr); + if (err) { + pr_err("Cannot pin src @ %llx\n", src_addr); + return err; + } + + err = pin_buffer(dst->vma, dst_addr); + if (err) { + pr_err("Cannot pin dst @ %llx\n", dst_addr); + goto err_src; + } + + err = i915_vma_pin(t->batch, 0, 0, PIN_USER | PIN_HIGH); + if (err) { + pr_err("cannot pin batch\n"); + goto err_dst; + } + + err = prepare_blit(t, dst, src, t->batch->obj); + if (err) + goto err_bb; + + rq = intel_context_create_request(t->ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_bb; + } + + err = move_to_active(t->batch, rq, 0); + if (!err) + err = move_to_active(src->vma, rq, 0); + if (!err) + err = move_to_active(dst->vma, rq, 0); + if (!err) + err = rq->engine->emit_bb_start(rq, + t->batch->node.start, + t->batch->node.size, + 0); + i915_request_get(rq); + i915_request_add(rq); + if (i915_request_wait(rq, 0, HZ / 2) < 0) + err = -ETIME; + i915_request_put(rq); + + dst->start_val = src->start_val; +err_bb: + i915_vma_unpin(t->batch); +err_dst: + i915_vma_unpin(dst->vma); +err_src: + i915_vma_unpin(src->vma); + return err; +} + +static struct tiled_blits * +tiled_blits_create(struct intel_engine_cs *engine, struct rnd_state *prng) +{ + struct drm_mm_node hole; + struct tiled_blits *t; + u64 hole_size; + int err; + + t = kzalloc(sizeof(*t), GFP_KERNEL); + if (!t) + return ERR_PTR(-ENOMEM); + + t->ce = intel_context_create(engine); + if (IS_ERR(t->ce)) { + err = PTR_ERR(t->ce); + goto err_free; + } + + hole_size = 2 * PAGE_ALIGN(WIDTH * HEIGHT * 4); + hole_size *= 2; /* room to maneuver */ + hole_size += 2 * I915_GTT_MIN_ALIGNMENT; + + mutex_lock(&t->ce->vm->mutex); + memset(&hole, 0, sizeof(hole)); + err = drm_mm_insert_node_in_range(&t->ce->vm->mm, &hole, + hole_size, 0, I915_COLOR_UNEVICTABLE, + 0, U64_MAX, + DRM_MM_INSERT_BEST); + if (!err) + drm_mm_remove_node(&hole); + mutex_unlock(&t->ce->vm->mutex); + if (err) { + err = -ENODEV; + goto err_put; + } + + t->hole = hole.start + I915_GTT_MIN_ALIGNMENT; + pr_info("Using hole at %llx\n", t->hole); + + err = tiled_blits_create_buffers(t, WIDTH, HEIGHT, prng); + if (err) + goto err_put; + + return t; + +err_put: + intel_context_put(t->ce); +err_free: + kfree(t); + return ERR_PTR(err); +} + +static void tiled_blits_destroy(struct tiled_blits *t) +{ + tiled_blits_destroy_buffers(t); + + intel_context_put(t->ce); + kfree(t); +} + +static int tiled_blits_prepare(struct tiled_blits *t, + struct rnd_state *prng) +{ + u64 offset = PAGE_ALIGN(t->width * t->height * 4); + u32 *map; + int err; + int i; + + map = i915_gem_object_pin_map(t->scratch.vma->obj, I915_MAP_WC); + if (IS_ERR(map)) + return PTR_ERR(map); + + /* Use scratch to fill objects */ + for (i = 0; i < ARRAY_SIZE(t->buffers); i++) { + fill_scratch(t, map, prandom_u32_state(prng)); + GEM_BUG_ON(verify_buffer(t, &t->scratch, prng)); + + err = tiled_blit(t, + &t->buffers[i], t->hole + offset, + &t->scratch, t->hole); + if (err == 0) + err = verify_buffer(t, &t->buffers[i], prng); + if (err) { + pr_err("Failed to create buffer %d\n", i); + break; + } + } + + i915_gem_object_unpin_map(t->scratch.vma->obj); + return err; +} + +static int tiled_blits_bounce(struct tiled_blits *t, struct rnd_state *prng) +{ + u64 offset = + round_up(t->width * t->height * 4, 2 * I915_GTT_MIN_ALIGNMENT); + int err; + + /* We want to check position invariant tiling across GTT eviction */ + + err = tiled_blit(t, + &t->buffers[1], t->hole + offset / 2, + &t->buffers[0], t->hole + 2 * offset); + if (err) + return err; + + /* Reposition so that we overlap the old addresses, and slightly off */ + err = tiled_blit(t, + &t->buffers[2], t->hole + I915_GTT_MIN_ALIGNMENT, + &t->buffers[1], t->hole + 3 * offset / 2); + if (err) + return err; + + err = verify_buffer(t, &t->buffers[2], prng); + if (err) + return err; + + return 0; +} + +static int __igt_client_tiled_blits(struct intel_engine_cs *engine, + struct rnd_state *prng) +{ + struct tiled_blits *t; + int err; + + t = tiled_blits_create(engine, prng); + if (IS_ERR(t)) + return PTR_ERR(t); + + err = tiled_blits_prepare(t, prng); + if (err) + goto out; + + err = tiled_blits_bounce(t, prng); + if (err) + goto out; + +out: + tiled_blits_destroy(t); + return err; +} + +static bool has_bit17_swizzle(int sw) +{ + return (sw == I915_BIT_6_SWIZZLE_9_10_17 || + sw == I915_BIT_6_SWIZZLE_9_17); +} + +static bool bad_swizzling(struct drm_i915_private *i915) +{ + struct i915_ggtt *ggtt = &i915->ggtt; + + if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) + return true; + + if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) || + has_bit17_swizzle(ggtt->bit_6_swizzle_y)) + return true; + + return false; +} + +static int igt_client_tiled_blits(void *arg) +{ + struct drm_i915_private *i915 = arg; + I915_RND_STATE(prng); + int inst = 0; + + /* Test requires explicit BLT tiling controls */ + if (INTEL_GEN(i915) < 4) + return 0; + + if (bad_swizzling(i915)) /* Requires sane (sub-page) swizzling */ + return 0; + + do { + struct intel_engine_cs *engine; + int err; + + engine = intel_engine_lookup_user(i915, + I915_ENGINE_CLASS_COPY, + inst++); + if (!engine) + return 0; + + err = __igt_client_tiled_blits(engine, &prng); + if (err == -ENODEV) + err = 0; + if (err) + return err; + } while (1); +} + int i915_gem_client_blt_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(igt_client_fill), + SUBTEST(igt_client_tiled_blits), }; if (intel_gt_is_wedged(&i915->gt)) diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c index 3f6079e1dfb6..87d7d8aa080f 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c @@ -158,6 +158,8 @@ static int wc_set(struct context *ctx, unsigned long offset, u32 v) return PTR_ERR(map); map[offset / sizeof(*map)] = v; + + __i915_gem_object_flush_map(ctx->obj, offset, sizeof(*map)); i915_gem_object_unpin_map(ctx->obj); return 0; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 375d864736f3..b81978890641 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -972,12 +972,6 @@ emit_rpcs_query(struct drm_i915_gem_object *obj, goto err_batch; } - err = rq->engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - 0); - if (err) - goto err_request; - i915_vma_lock(batch); err = i915_request_await_object(rq, batch->obj, false); if (err == 0) @@ -994,6 +988,18 @@ emit_rpcs_query(struct drm_i915_gem_object *obj, if (err) goto skip_request; + if (rq->engine->emit_init_breadcrumb) { + err = rq->engine->emit_init_breadcrumb(rq); + if (err) + goto skip_request; + } + + err = rq->engine->emit_bb_start(rq, + batch->node.start, batch->node.size, + 0); + if (err) + goto skip_request; + i915_vma_unpin_and_release(&batch, 0); i915_vma_unpin(vma); @@ -1004,8 +1010,7 @@ emit_rpcs_query(struct drm_i915_gem_object *obj, return 0; skip_request: - i915_request_skip(rq, err); -err_request: + i915_request_set_error_once(rq, err); i915_request_add(rq); err_batch: i915_vma_unpin_and_release(&batch, 0); @@ -1541,10 +1546,6 @@ static int write_to_scratch(struct i915_gem_context *ctx, goto err_unpin; } - err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); - if (err) - goto err_request; - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, false); if (err == 0) @@ -1553,14 +1554,23 @@ static int write_to_scratch(struct i915_gem_context *ctx, if (err) goto skip_request; + if (rq->engine->emit_init_breadcrumb) { + err = rq->engine->emit_init_breadcrumb(rq); + if (err) + goto skip_request; + } + + err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, 0); + if (err) + goto skip_request; + i915_vma_unpin(vma); i915_request_add(rq); goto out_vm; skip_request: - i915_request_skip(rq, err); -err_request: + i915_request_set_error_once(rq, err); i915_request_add(rq); err_unpin: i915_vma_unpin(vma); @@ -1674,10 +1684,6 @@ static int read_from_scratch(struct i915_gem_context *ctx, goto err_unpin; } - err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, flags); - if (err) - goto err_request; - i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, true); if (err == 0) @@ -1686,8 +1692,17 @@ static int read_from_scratch(struct i915_gem_context *ctx, if (err) goto skip_request; + if (rq->engine->emit_init_breadcrumb) { + err = rq->engine->emit_init_breadcrumb(rq); + if (err) + goto skip_request; + } + + err = engine->emit_bb_start(rq, vma->node.start, vma->node.size, flags); + if (err) + goto skip_request; + i915_vma_unpin(vma); - i915_vma_close(vma); i915_request_add(rq); @@ -1708,8 +1723,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, goto out_vm; skip_request: - i915_request_skip(rq, err); -err_request: + i915_request_set_error_once(rq, err); i915_request_add(rq); err_unpin: i915_vma_unpin(vma); @@ -1809,7 +1823,6 @@ static int igt_vm_isolation(void *arg) vm_total = ctx_vm(ctx_a)->total; GEM_BUG_ON(ctx_vm(ctx_b)->total != vm_total); - vm_total -= I915_GTT_PAGE_SIZE; count = 0; num_engines = 0; @@ -1828,10 +1841,10 @@ static int igt_vm_isolation(void *arg) u32 value = 0xc5c5c5c5; u64 offset; - div64_u64_rem(i915_prandom_u64_state(&prng), - vm_total, &offset); - offset = round_down(offset, alignof_dword); - offset += I915_GTT_PAGE_SIZE; + /* Leave enough space at offset 0 for the batch */ + offset = igt_random_offset(&prng, + I915_GTT_PAGE_SIZE, vm_total, + sizeof(u32), alignof_dword); err = write_to_scratch(ctx_a, engine, offset, 0xdeadbeef); @@ -1926,7 +1939,7 @@ static int mock_context_barrier(void *arg) goto out; } - rq = igt_request_alloc(ctx, i915->engine[RCS0]); + rq = igt_request_alloc(ctx, i915->gt.engine[RCS0]); if (IS_ERR(rq)) { pr_err("Request allocation failed!\n"); goto out; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c new file mode 100644 index 000000000000..a49016f8ee0d --- /dev/null +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "i915_selftest.h" + +#include "gt/intel_engine_pm.h" +#include "selftests/igt_flush_test.h" + +static u64 read_reloc(const u32 *map, int x, const u64 mask) +{ + u64 reloc; + + memcpy(&reloc, &map[x], sizeof(reloc)); + return reloc & mask; +} + +static int __igt_gpu_reloc(struct i915_execbuffer *eb, + struct drm_i915_gem_object *obj) +{ + const unsigned int offsets[] = { 8, 3, 0 }; + const u64 mask = + GENMASK_ULL(eb->reloc_cache.use_64bit_reloc ? 63 : 31, 0); + const u32 *map = page_mask_bits(obj->mm.mapping); + struct i915_request *rq; + struct i915_vma *vma; + int err; + int i; + + vma = i915_vma_instance(obj, eb->context->vm, NULL); + if (IS_ERR(vma)) + return PTR_ERR(vma); + + err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH); + if (err) + return err; + + /* 8-Byte aligned */ + if (!__reloc_entry_gpu(eb, vma, + offsets[0] * sizeof(u32), + 0)) { + err = -EIO; + goto unpin_vma; + } + + /* !8-Byte aligned */ + if (!__reloc_entry_gpu(eb, vma, + offsets[1] * sizeof(u32), + 1)) { + err = -EIO; + goto unpin_vma; + } + + /* Skip to the end of the cmd page */ + i = PAGE_SIZE / sizeof(u32) - RELOC_TAIL - 1; + i -= eb->reloc_cache.rq_size; + memset32(eb->reloc_cache.rq_cmd + eb->reloc_cache.rq_size, + MI_NOOP, i); + eb->reloc_cache.rq_size += i; + + /* Force batch chaining */ + if (!__reloc_entry_gpu(eb, vma, + offsets[2] * sizeof(u32), + 2)) { + err = -EIO; + goto unpin_vma; + } + + GEM_BUG_ON(!eb->reloc_cache.rq); + rq = i915_request_get(eb->reloc_cache.rq); + err = reloc_gpu_flush(&eb->reloc_cache); + if (err) + goto put_rq; + GEM_BUG_ON(eb->reloc_cache.rq); + + err = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE, HZ / 2); + if (err) { + intel_gt_set_wedged(eb->engine->gt); + goto put_rq; + } + + if (!i915_request_completed(rq)) { + pr_err("%s: did not wait for relocations!\n", eb->engine->name); + err = -EINVAL; + goto put_rq; + } + + for (i = 0; i < ARRAY_SIZE(offsets); i++) { + u64 reloc = read_reloc(map, offsets[i], mask); + + if (reloc != i) { + pr_err("%s[%d]: map[%d] %llx != %x\n", + eb->engine->name, i, offsets[i], reloc, i); + err = -EINVAL; + } + } + if (err) + igt_hexdump(map, 4096); + +put_rq: + i915_request_put(rq); +unpin_vma: + i915_vma_unpin(vma); + return err; +} + +static int igt_gpu_reloc(void *arg) +{ + struct i915_execbuffer eb; + struct drm_i915_gem_object *scratch; + int err = 0; + u32 *map; + + eb.i915 = arg; + + scratch = i915_gem_object_create_internal(eb.i915, 4096); + if (IS_ERR(scratch)) + return PTR_ERR(scratch); + + map = i915_gem_object_pin_map(scratch, I915_MAP_WC); + if (IS_ERR(map)) { + err = PTR_ERR(map); + goto err_scratch; + } + + for_each_uabi_engine(eb.engine, eb.i915) { + reloc_cache_init(&eb.reloc_cache, eb.i915); + memset(map, POISON_INUSE, 4096); + + intel_engine_pm_get(eb.engine); + eb.context = intel_context_create(eb.engine); + if (IS_ERR(eb.context)) { + err = PTR_ERR(eb.context); + goto err_pm; + } + + err = intel_context_pin(eb.context); + if (err) + goto err_put; + + err = __igt_gpu_reloc(&eb, scratch); + + intel_context_unpin(eb.context); +err_put: + intel_context_put(eb.context); +err_pm: + intel_engine_pm_put(eb.engine); + if (err) + break; + } + + if (igt_flush_test(eb.i915)) + err = -EIO; + +err_scratch: + i915_gem_object_put(scratch); + return err; +} + +int i915_gem_execbuffer_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_gpu_reloc), + }; + + if (intel_gt_is_wedged(&i915->gt)) + return 0; + + return i915_live_subtests(tests, i915); +} diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index ef7c74cff28a..9c7402ce5bf9 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -570,7 +570,7 @@ static bool assert_mmap_offset(struct drm_i915_private *i915, obj = i915_gem_object_create_internal(i915, size); if (IS_ERR(obj)) - return PTR_ERR(obj); + return false; mmo = mmap_offset_attach(obj, I915_MMAP_OFFSET_GTT, NULL); i915_gem_object_put(obj); @@ -952,6 +952,129 @@ static int igt_mmap(void *arg) return 0; } +static const char *repr_mmap_type(enum i915_mmap_type type) +{ + switch (type) { + case I915_MMAP_TYPE_GTT: return "gtt"; + case I915_MMAP_TYPE_WB: return "wb"; + case I915_MMAP_TYPE_WC: return "wc"; + case I915_MMAP_TYPE_UC: return "uc"; + default: return "unknown"; + } +} + +static bool can_access(const struct drm_i915_gem_object *obj) +{ + unsigned int flags = + I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM; + + return i915_gem_object_type_has(obj, flags); +} + +static int __igt_mmap_access(struct drm_i915_private *i915, + struct drm_i915_gem_object *obj, + enum i915_mmap_type type) +{ + struct i915_mmap_offset *mmo; + unsigned long __user *ptr; + unsigned long A, B; + unsigned long x, y; + unsigned long addr; + int err; + + memset(&A, 0xAA, sizeof(A)); + memset(&B, 0xBB, sizeof(B)); + + if (!can_mmap(obj, type) || !can_access(obj)) + return 0; + + mmo = mmap_offset_attach(obj, type, NULL); + if (IS_ERR(mmo)) + return PTR_ERR(mmo); + + addr = igt_mmap_node(i915, &mmo->vma_node, 0, PROT_WRITE, MAP_SHARED); + if (IS_ERR_VALUE(addr)) + return addr; + ptr = (unsigned long __user *)addr; + + err = __put_user(A, ptr); + if (err) { + pr_err("%s(%s): failed to write into user mmap\n", + obj->mm.region->name, repr_mmap_type(type)); + goto out_unmap; + } + + intel_gt_flush_ggtt_writes(&i915->gt); + + err = access_process_vm(current, addr, &x, sizeof(x), 0); + if (err != sizeof(x)) { + pr_err("%s(%s): access_process_vm() read failed\n", + obj->mm.region->name, repr_mmap_type(type)); + goto out_unmap; + } + + err = access_process_vm(current, addr, &B, sizeof(B), FOLL_WRITE); + if (err != sizeof(B)) { + pr_err("%s(%s): access_process_vm() write failed\n", + obj->mm.region->name, repr_mmap_type(type)); + goto out_unmap; + } + + intel_gt_flush_ggtt_writes(&i915->gt); + + err = __get_user(y, ptr); + if (err) { + pr_err("%s(%s): failed to read from user mmap\n", + obj->mm.region->name, repr_mmap_type(type)); + goto out_unmap; + } + + if (x != A || y != B) { + pr_err("%s(%s): failed to read/write values, found (%lx, %lx)\n", + obj->mm.region->name, repr_mmap_type(type), + x, y); + err = -EINVAL; + goto out_unmap; + } + +out_unmap: + vm_munmap(addr, obj->base.size); + return err; +} + +static int igt_mmap_access(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_memory_region *mr; + enum intel_region_id id; + + for_each_memory_region(mr, i915, id) { + struct drm_i915_gem_object *obj; + int err; + + obj = i915_gem_object_create_region(mr, PAGE_SIZE, 0); + if (obj == ERR_PTR(-ENODEV)) + continue; + + if (IS_ERR(obj)) + return PTR_ERR(obj); + + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_GTT); + if (err == 0) + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WB); + if (err == 0) + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_WC); + if (err == 0) + err = __igt_mmap_access(i915, obj, I915_MMAP_TYPE_UC); + + i915_gem_object_put(obj); + if (err) + return err; + } + + return 0; +} + static int __igt_mmap_gpu(struct drm_i915_private *i915, struct drm_i915_gem_object *obj, enum i915_mmap_type type) @@ -1156,9 +1279,6 @@ static int __igt_mmap_revoke(struct drm_i915_private *i915, if (err) goto out_unmap; - GEM_BUG_ON(mmo->mmap_type == I915_MMAP_TYPE_GTT && - !atomic_read(&obj->bind_count)); - err = check_present(addr, obj->base.size); if (err) { pr_err("%s: was not present\n", obj->mm.region->name); @@ -1175,7 +1295,6 @@ static int __igt_mmap_revoke(struct drm_i915_private *i915, pr_err("Failed to unbind object!\n"); goto out_unmap; } - GEM_BUG_ON(atomic_read(&obj->bind_count)); if (type != I915_MMAP_TYPE_GTT) { __i915_gem_object_put_pages(obj); @@ -1233,6 +1352,7 @@ int i915_gem_mman_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_smoke_tiling), SUBTEST(igt_mmap_offset_exhaustion), SUBTEST(igt_mmap), + SUBTEST(igt_mmap_access), SUBTEST(igt_mmap_revoke), SUBTEST(igt_mmap_gpu), }; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c index 2b6db6f799de..faa5b6d91795 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c @@ -14,7 +14,7 @@ static int igt_gem_object(void *arg) { struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; - int err = -ENOMEM; + int err; /* Basic test to ensure we can create an object */ diff --git a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c index 6718da20f35d..e21b5023ca7d 100644 --- a/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c +++ b/drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c @@ -83,6 +83,8 @@ igt_emit_store_dw(struct i915_vma *vma, offset += PAGE_SIZE; } *cmd = MI_BATCH_BUFFER_END; + + i915_gem_object_flush_map(obj); i915_gem_object_unpin_map(obj); intel_gt_chipset_flush(vma->vm->gt); @@ -126,16 +128,6 @@ int igt_gpu_fill_dw(struct intel_context *ce, goto err_batch; } - flags = 0; - if (INTEL_GEN(ce->vm->i915) <= 5) - flags |= I915_DISPATCH_SECURE; - - err = rq->engine->emit_bb_start(rq, - batch->node.start, batch->node.size, - flags); - if (err) - goto err_request; - i915_vma_lock(batch); err = i915_request_await_object(rq, batch->obj, false); if (err == 0) @@ -152,15 +144,17 @@ int igt_gpu_fill_dw(struct intel_context *ce, if (err) goto skip_request; - i915_request_add(rq); - - i915_vma_unpin_and_release(&batch, 0); + flags = 0; + if (INTEL_GEN(ce->vm->i915) <= 5) + flags |= I915_DISPATCH_SECURE; - return 0; + err = rq->engine->emit_bb_start(rq, + batch->node.start, batch->node.size, + flags); skip_request: - i915_request_skip(rq, err); -err_request: + if (err) + i915_request_set_error_once(rq, err); i915_request_add(rq); err_batch: i915_vma_unpin_and_release(&batch, 0); diff --git a/drivers/gpu/drm/i915/gem/selftests/mock_context.c b/drivers/gpu/drm/i915/gem/selftests/mock_context.c index b12ea1daa29d..e7e3c620f542 100644 --- a/drivers/gpu/drm/i915/gem/selftests/mock_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/mock_context.c @@ -23,6 +23,9 @@ mock_context(struct drm_i915_private *i915, INIT_LIST_HEAD(&ctx->link); ctx->i915 = i915; + spin_lock_init(&ctx->stale.lock); + INIT_LIST_HEAD(&ctx->stale.engines); + i915_gem_context_set_persistence(ctx); mutex_init(&ctx->engines_mutex); diff --git a/drivers/gpu/drm/i915/gt/debugfs_engines.c b/drivers/gpu/drm/i915/gt/debugfs_engines.c index 6a5e9ab20b94..5e3725e62241 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_engines.c +++ b/drivers/gpu/drm/i915/gt/debugfs_engines.c @@ -32,5 +32,5 @@ void debugfs_engines_register(struct intel_gt *gt, struct dentry *root) { "engines", &engines_fops }, }; - debugfs_gt_register_files(gt, root, files, ARRAY_SIZE(files)); + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), gt); } diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt.c b/drivers/gpu/drm/i915/gt/debugfs_gt.c index 75255aaacaed..1de5fbaa1cf9 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_gt.c +++ b/drivers/gpu/drm/i915/gt/debugfs_gt.c @@ -9,6 +9,7 @@ #include "debugfs_engines.h" #include "debugfs_gt.h" #include "debugfs_gt_pm.h" +#include "uc/intel_uc_debugfs.h" #include "i915_drv.h" void debugfs_gt_register(struct intel_gt *gt) @@ -24,17 +25,19 @@ void debugfs_gt_register(struct intel_gt *gt) debugfs_engines_register(gt, root); debugfs_gt_pm_register(gt, root); + + intel_uc_debugfs_register(>->uc, root); } -void debugfs_gt_register_files(struct intel_gt *gt, - struct dentry *root, - const struct debugfs_gt_file *files, - unsigned long count) +void intel_gt_debugfs_register_files(struct dentry *root, + const struct debugfs_gt_file *files, + unsigned long count, void *data) { while (count--) { - if (!files->eval || files->eval(gt)) + umode_t mode = files->fops->write ? 0644 : 0444; + if (!files->eval || files->eval(data)) debugfs_create_file(files->name, - 0444, root, gt, + mode, root, data, files->fops); files++; diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt.h b/drivers/gpu/drm/i915/gt/debugfs_gt.h index 4ea0f06cda8f..f77540f727e9 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_gt.h +++ b/drivers/gpu/drm/i915/gt/debugfs_gt.h @@ -28,12 +28,11 @@ void debugfs_gt_register(struct intel_gt *gt); struct debugfs_gt_file { const char *name; const struct file_operations *fops; - bool (*eval)(const struct intel_gt *gt); + bool (*eval)(void *data); }; -void debugfs_gt_register_files(struct intel_gt *gt, - struct dentry *root, - const struct debugfs_gt_file *files, - unsigned long count); +void intel_gt_debugfs_register_files(struct dentry *root, + const struct debugfs_gt_file *files, + unsigned long count, void *data); #endif /* DEBUGFS_GT_H */ diff --git a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c b/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c index 059c9e5c002e..174a24553322 100644 --- a/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/debugfs_gt_pm.c @@ -10,6 +10,7 @@ #include "debugfs_gt_pm.h" #include "i915_drv.h" #include "intel_gt.h" +#include "intel_gt_clock_utils.h" #include "intel_llc.h" #include "intel_rc6.h" #include "intel_rps.h" @@ -268,7 +269,7 @@ static int frequency_show(struct seq_file *m, void *unused) yesno(rpmodectl & GEN6_RP_ENABLE)); seq_printf(m, "SW control enabled: %s\n", yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == - GEN6_RP_MEDIA_SW_MODE)); + GEN6_RP_MEDIA_SW_MODE)); vlv_punit_get(i915); freq_sts = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); @@ -300,8 +301,9 @@ static int frequency_show(struct seq_file *m, void *unused) u32 rp_state_cap; u32 rpmodectl, rpinclimit, rpdeclimit; u32 rpstat, cagf, reqf; - u32 rpupei, rpcurup, rpprevup; - u32 rpdownei, rpcurdown, rpprevdown; + u32 rpcurupei, rpcurup, rpprevup; + u32 rpcurdownei, rpcurdown, rpprevdown; + u32 rpupei, rpupt, rpdownei, rpdownt; u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask; int max_freq; @@ -334,12 +336,19 @@ static int frequency_show(struct seq_file *m, void *unused) rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); rpstat = intel_uncore_read(uncore, GEN6_RPSTAT1); - rpupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; + rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK; rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK; rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK; - rpdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; + rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK; rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK; rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK; + + rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI); + rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD); + + rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); + rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD); + cagf = intel_rps_read_actual_frequency(rps); intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); @@ -372,7 +381,7 @@ static int frequency_show(struct seq_file *m, void *unused) yesno(rpmodectl & GEN6_RP_ENABLE)); seq_printf(m, "SW control enabled: %s\n", yesno((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == - GEN6_RP_MEDIA_SW_MODE)); + GEN6_RP_MEDIA_SW_MODE)); seq_printf(m, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n", pm_ier, pm_imr, pm_mask); @@ -394,23 +403,35 @@ static int frequency_show(struct seq_file *m, void *unused) seq_printf(m, "RPDECLIMIT: 0x%08x\n", rpdeclimit); seq_printf(m, "RPNSWREQ: %dMHz\n", reqf); seq_printf(m, "CAGF: %dMHz\n", cagf); - seq_printf(m, "RP CUR UP EI: %d (%dus)\n", - rpupei, GT_PM_INTERVAL_TO_US(i915, rpupei)); - seq_printf(m, "RP CUR UP: %d (%dus)\n", - rpcurup, GT_PM_INTERVAL_TO_US(i915, rpcurup)); - seq_printf(m, "RP PREV UP: %d (%dus)\n", - rpprevup, GT_PM_INTERVAL_TO_US(i915, rpprevup)); + seq_printf(m, "RP CUR UP EI: %d (%dns)\n", + rpcurupei, + intel_gt_pm_interval_to_ns(gt, rpcurupei)); + seq_printf(m, "RP CUR UP: %d (%dns)\n", + rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup)); + seq_printf(m, "RP PREV UP: %d (%dns)\n", + rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup)); seq_printf(m, "Up threshold: %d%%\n", rps->power.up_threshold); - - seq_printf(m, "RP CUR DOWN EI: %d (%dus)\n", - rpdownei, GT_PM_INTERVAL_TO_US(i915, rpdownei)); - seq_printf(m, "RP CUR DOWN: %d (%dus)\n", - rpcurdown, GT_PM_INTERVAL_TO_US(i915, rpcurdown)); - seq_printf(m, "RP PREV DOWN: %d (%dus)\n", - rpprevdown, GT_PM_INTERVAL_TO_US(i915, rpprevdown)); + seq_printf(m, "RP UP EI: %d (%dns)\n", + rpupei, intel_gt_pm_interval_to_ns(gt, rpupei)); + seq_printf(m, "RP UP THRESHOLD: %d (%dns)\n", + rpupt, intel_gt_pm_interval_to_ns(gt, rpupt)); + + seq_printf(m, "RP CUR DOWN EI: %d (%dns)\n", + rpcurdownei, + intel_gt_pm_interval_to_ns(gt, rpcurdownei)); + seq_printf(m, "RP CUR DOWN: %d (%dns)\n", + rpcurdown, + intel_gt_pm_interval_to_ns(gt, rpcurdown)); + seq_printf(m, "RP PREV DOWN: %d (%dns)\n", + rpprevdown, + intel_gt_pm_interval_to_ns(gt, rpprevdown)); seq_printf(m, "Down threshold: %d%%\n", rps->power.down_threshold); + seq_printf(m, "RP DOWN EI: %d (%dns)\n", + rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei)); + seq_printf(m, "RP DOWN THRESHOLD: %d (%dns)\n", + rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt)); max_freq = (IS_GEN9_LP(i915) ? rp_state_cap >> 0 : rp_state_cap >> 16) & 0xff; @@ -506,8 +527,10 @@ static int llc_show(struct seq_file *m, void *data) return 0; } -static bool llc_eval(const struct intel_gt *gt) +static bool llc_eval(void *data) { + struct intel_gt *gt = data; + return HAS_LLC(gt->i915); } @@ -533,7 +556,8 @@ static int rps_boost_show(struct seq_file *m, void *data) struct drm_i915_private *i915 = gt->i915; struct intel_rps *rps = >->rps; - seq_printf(m, "RPS enabled? %d\n", rps->enabled); + seq_printf(m, "RPS enabled? %s\n", yesno(intel_rps_is_enabled(rps))); + seq_printf(m, "RPS active? %s\n", yesno(intel_rps_is_active(rps))); seq_printf(m, "GPU busy? %s\n", yesno(gt->awake)); seq_printf(m, "Boosts outstanding? %d\n", atomic_read(&rps->num_waiters)); @@ -553,7 +577,7 @@ static int rps_boost_show(struct seq_file *m, void *data) seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts)); - if (INTEL_GEN(i915) >= 6 && rps->enabled && gt->awake) { + if (INTEL_GEN(i915) >= 6 && intel_rps_is_active(rps)) { struct intel_uncore *uncore = gt->uncore; u32 rpup, rpupei; u32 rpdown, rpdownei; @@ -580,8 +604,10 @@ static int rps_boost_show(struct seq_file *m, void *data) return 0; } -static bool rps_eval(const struct intel_gt *gt) +static bool rps_eval(void *data) { + struct intel_gt *gt = data; + return HAS_RPS(gt->i915); } @@ -597,5 +623,5 @@ void debugfs_gt_pm_register(struct intel_gt *gt, struct dentry *root) { "rps_boost", &rps_boost_fops, rps_eval }, }; - debugfs_gt_register_files(gt, root, files, ARRAY_SIZE(files)); + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), gt); } diff --git a/drivers/gpu/drm/i915/gt/gen7_renderclear.c b/drivers/gpu/drm/i915/gt/gen7_renderclear.c new file mode 100644 index 000000000000..de595b66a746 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/gen7_renderclear.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include "gen7_renderclear.h" +#include "i915_drv.h" +#include "intel_gpu_commands.h" + +#define MAX_URB_ENTRIES 64 +#define STATE_SIZE (4 * 1024) +#define GT3_INLINE_DATA_DELAYS 0x1E00 +#define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS)) + +struct cb_kernel { + const void *data; + u32 size; +}; + +#define CB_KERNEL(name) { .data = (name), .size = sizeof(name) } + +#include "ivb_clear_kernel.c" +static const struct cb_kernel cb_kernel_ivb = CB_KERNEL(ivb_clear_kernel); + +#include "hsw_clear_kernel.c" +static const struct cb_kernel cb_kernel_hsw = CB_KERNEL(hsw_clear_kernel); + +struct batch_chunk { + struct i915_vma *vma; + u32 offset; + u32 *start; + u32 *end; + u32 max_items; +}; + +struct batch_vals { + u32 max_primitives; + u32 max_urb_entries; + u32 cmd_size; + u32 state_size; + u32 state_start; + u32 batch_size; + u32 surface_height; + u32 surface_width; + u32 scratch_size; + u32 max_size; +}; + +static void +batch_get_defaults(struct drm_i915_private *i915, struct batch_vals *bv) +{ + if (IS_HASWELL(i915)) { + bv->max_primitives = 280; + bv->max_urb_entries = MAX_URB_ENTRIES; + bv->surface_height = 16 * 16; + bv->surface_width = 32 * 2 * 16; + } else { + bv->max_primitives = 128; + bv->max_urb_entries = MAX_URB_ENTRIES / 2; + bv->surface_height = 16 * 8; + bv->surface_width = 32 * 16; + } + bv->cmd_size = bv->max_primitives * 4096; + bv->state_size = STATE_SIZE; + bv->state_start = bv->cmd_size; + bv->batch_size = bv->cmd_size + bv->state_size; + bv->scratch_size = bv->surface_height * bv->surface_width; + bv->max_size = bv->batch_size + bv->scratch_size; +} + +static void batch_init(struct batch_chunk *bc, + struct i915_vma *vma, + u32 *start, u32 offset, u32 max_bytes) +{ + bc->vma = vma; + bc->offset = offset; + bc->start = start + bc->offset / sizeof(*bc->start); + bc->end = bc->start; + bc->max_items = max_bytes / sizeof(*bc->start); +} + +static u32 batch_offset(const struct batch_chunk *bc, u32 *cs) +{ + return (cs - bc->start) * sizeof(*bc->start) + bc->offset; +} + +static u32 batch_addr(const struct batch_chunk *bc) +{ + return bc->vma->node.start; +} + +static void batch_add(struct batch_chunk *bc, const u32 d) +{ + GEM_BUG_ON((bc->end - bc->start) >= bc->max_items); + *bc->end++ = d; +} + +static u32 *batch_alloc_items(struct batch_chunk *bc, u32 align, u32 items) +{ + u32 *map; + + if (align) { + u32 *end = PTR_ALIGN(bc->end, align); + + memset32(bc->end, 0, end - bc->end); + bc->end = end; + } + + map = bc->end; + bc->end += items; + + return map; +} + +static u32 *batch_alloc_bytes(struct batch_chunk *bc, u32 align, u32 bytes) +{ + GEM_BUG_ON(!IS_ALIGNED(bytes, sizeof(*bc->start))); + return batch_alloc_items(bc, align, bytes / sizeof(*bc->start)); +} + +static u32 +gen7_fill_surface_state(struct batch_chunk *state, + const u32 dst_offset, + const struct batch_vals *bv) +{ + u32 surface_h = bv->surface_height; + u32 surface_w = bv->surface_width; + u32 *cs = batch_alloc_items(state, 32, 8); + u32 offset = batch_offset(state, cs); + +#define SURFACE_2D 1 +#define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0 +#define RENDER_CACHE_READ_WRITE 1 + + *cs++ = SURFACE_2D << 29 | + (SURFACEFORMAT_B8G8R8A8_UNORM << 18) | + (RENDER_CACHE_READ_WRITE << 8); + + *cs++ = batch_addr(state) + dst_offset; + + *cs++ = ((surface_h / 4 - 1) << 16) | (surface_w / 4 - 1); + *cs++ = surface_w; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; +#define SHADER_CHANNELS(r, g, b, a) \ + (((r) << 25) | ((g) << 22) | ((b) << 19) | ((a) << 16)) + *cs++ = SHADER_CHANNELS(4, 5, 6, 7); + batch_advance(state, cs); + + return offset; +} + +static u32 +gen7_fill_binding_table(struct batch_chunk *state, + const struct batch_vals *bv) +{ + u32 surface_start = gen7_fill_surface_state(state, bv->batch_size, bv); + u32 *cs = batch_alloc_items(state, 32, 8); + u32 offset = batch_offset(state, cs); + + *cs++ = surface_start - state->offset; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + batch_advance(state, cs); + + return offset; +} + +static u32 +gen7_fill_kernel_data(struct batch_chunk *state, + const u32 *data, + const u32 size) +{ + return batch_offset(state, + memcpy(batch_alloc_bytes(state, 64, size), + data, size)); +} + +static u32 +gen7_fill_interface_descriptor(struct batch_chunk *state, + const struct batch_vals *bv, + const struct cb_kernel *kernel, + unsigned int count) +{ + u32 kernel_offset = + gen7_fill_kernel_data(state, kernel->data, kernel->size); + u32 binding_table = gen7_fill_binding_table(state, bv); + u32 *cs = batch_alloc_items(state, 32, 8 * count); + u32 offset = batch_offset(state, cs); + + *cs++ = kernel_offset; + *cs++ = (1 << 7) | (1 << 13); + *cs++ = 0; + *cs++ = (binding_table - state->offset) | 1; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + + /* 1 - 63dummy idds */ + memset32(cs, 0x00, (count - 1) * 8); + batch_advance(state, cs + (count - 1) * 8); + + return offset; +} + +static void +gen7_emit_state_base_address(struct batch_chunk *batch, + u32 surface_state_base) +{ + u32 *cs = batch_alloc_items(batch, 0, 12); + + *cs++ = STATE_BASE_ADDRESS | (12 - 2); + /* general */ + *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; + /* surface */ + *cs++ = batch_addr(batch) | surface_state_base | BASE_ADDRESS_MODIFY; + /* dynamic */ + *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; + /* indirect */ + *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; + /* instruction */ + *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; + + /* general/dynamic/indirect/instruction access Bound */ + *cs++ = 0; + *cs++ = BASE_ADDRESS_MODIFY; + *cs++ = 0; + *cs++ = BASE_ADDRESS_MODIFY; + *cs++ = 0; + *cs++ = 0; + batch_advance(batch, cs); +} + +static void +gen7_emit_vfe_state(struct batch_chunk *batch, + const struct batch_vals *bv, + u32 urb_size, u32 curbe_size, + u32 mode) +{ + u32 urb_entries = bv->max_urb_entries; + u32 threads = bv->max_primitives - 1; + u32 *cs = batch_alloc_items(batch, 32, 8); + + *cs++ = MEDIA_VFE_STATE | (8 - 2); + + /* scratch buffer */ + *cs++ = 0; + + /* number of threads & urb entries for GPGPU vs Media Mode */ + *cs++ = threads << 16 | urb_entries << 8 | mode << 2; + + *cs++ = 0; + + /* urb entry size & curbe size in 256 bits unit */ + *cs++ = urb_size << 16 | curbe_size; + + /* scoreboard */ + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + batch_advance(batch, cs); +} + +static void +gen7_emit_interface_descriptor_load(struct batch_chunk *batch, + const u32 interface_descriptor, + unsigned int count) +{ + u32 *cs = batch_alloc_items(batch, 8, 4); + + *cs++ = MEDIA_INTERFACE_DESCRIPTOR_LOAD | (4 - 2); + *cs++ = 0; + *cs++ = count * 8 * sizeof(*cs); + + /* + * interface descriptor address - it is relative to the dynamics base + * address + */ + *cs++ = interface_descriptor; + batch_advance(batch, cs); +} + +static void +gen7_emit_media_object(struct batch_chunk *batch, + unsigned int media_object_index) +{ + unsigned int x_offset = (media_object_index % 16) * 64; + unsigned int y_offset = (media_object_index / 16) * 16; + unsigned int inline_data_size; + unsigned int media_batch_size; + unsigned int i; + u32 *cs; + + inline_data_size = 112 * 8; + media_batch_size = inline_data_size + 6; + + cs = batch_alloc_items(batch, 8, media_batch_size); + + *cs++ = MEDIA_OBJECT | (media_batch_size - 2); + + /* interface descriptor offset */ + *cs++ = 0; + + /* without indirect data */ + *cs++ = 0; + *cs++ = 0; + + /* scoreboard */ + *cs++ = 0; + *cs++ = 0; + + /* inline */ + *cs++ = (y_offset << 16) | (x_offset); + *cs++ = 0; + *cs++ = GT3_INLINE_DATA_DELAYS; + for (i = 3; i < inline_data_size; i++) + *cs++ = 0; + + batch_advance(batch, cs); +} + +static void gen7_emit_pipeline_flush(struct batch_chunk *batch) +{ + u32 *cs = batch_alloc_items(batch, 0, 5); + + *cs++ = GFX_OP_PIPE_CONTROL(5); + *cs++ = PIPE_CONTROL_STATE_CACHE_INVALIDATE | + PIPE_CONTROL_GLOBAL_GTT_IVB; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + batch_advance(batch, cs); +} + +static void emit_batch(struct i915_vma * const vma, + u32 *start, + const struct batch_vals *bv) +{ + struct drm_i915_private *i915 = vma->vm->i915; + unsigned int desc_count = 64; + const u32 urb_size = 112; + struct batch_chunk cmds, state; + u32 interface_descriptor; + unsigned int i; + + batch_init(&cmds, vma, start, 0, bv->cmd_size); + batch_init(&state, vma, start, bv->state_start, bv->state_size); + + interface_descriptor = + gen7_fill_interface_descriptor(&state, bv, + IS_HASWELL(i915) ? + &cb_kernel_hsw : + &cb_kernel_ivb, + desc_count); + gen7_emit_pipeline_flush(&cmds); + batch_add(&cmds, PIPELINE_SELECT | PIPELINE_SELECT_MEDIA); + batch_add(&cmds, MI_NOOP); + gen7_emit_state_base_address(&cmds, interface_descriptor); + gen7_emit_pipeline_flush(&cmds); + + gen7_emit_vfe_state(&cmds, bv, urb_size - 1, 0, 0); + + gen7_emit_interface_descriptor_load(&cmds, + interface_descriptor, + desc_count); + + for (i = 0; i < bv->max_primitives; i++) + gen7_emit_media_object(&cmds, i); + + batch_add(&cmds, MI_BATCH_BUFFER_END); +} + +int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine, + struct i915_vma * const vma) +{ + struct batch_vals bv; + u32 *batch; + + batch_get_defaults(engine->i915, &bv); + if (!vma) + return bv.max_size; + + GEM_BUG_ON(vma->obj->base.size < bv.max_size); + + batch = i915_gem_object_pin_map(vma->obj, I915_MAP_WC); + if (IS_ERR(batch)) + return PTR_ERR(batch); + + emit_batch(vma, memset(batch, 0, bv.max_size), &bv); + + i915_gem_object_flush_map(vma->obj); + i915_gem_object_unpin_map(vma->obj); + + return 0; +} diff --git a/drivers/gpu/drm/i915/gt/gen7_renderclear.h b/drivers/gpu/drm/i915/gt/gen7_renderclear.h new file mode 100644 index 000000000000..bb100748e2c6 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/gen7_renderclear.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef __GEN7_RENDERCLEAR_H__ +#define __GEN7_RENDERCLEAR_H__ + +struct intel_engine_cs; +struct i915_vma; + +int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine, + struct i915_vma * const vma); + +#endif /* __GEN7_RENDERCLEAR_H__ */ diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c index 4d1de2d97d5c..699125928272 100644 --- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c +++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c @@ -8,6 +8,7 @@ #include "gen8_ppgtt.h" #include "i915_scatterlist.h" #include "i915_trace.h" +#include "i915_pvinfo.h" #include "i915_vgpu.h" #include "intel_gt.h" #include "intel_gtt.h" @@ -25,6 +26,30 @@ static u64 gen8_pde_encode(const dma_addr_t addr, return pde; } +static u64 gen8_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) +{ + gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW; + + if (unlikely(flags & PTE_READ_ONLY)) + pte &= ~_PAGE_RW; + + switch (level) { + case I915_CACHE_NONE: + pte |= PPAT_UNCACHED; + break; + case I915_CACHE_WT: + pte |= PPAT_DISPLAY_ELLC; + break; + default: + pte |= PPAT_CACHED; + break; + } + + return pte; +} + static void gen8_ppgtt_notify_vgt(struct i915_ppgtt *ppgtt, bool create) { struct drm_i915_private *i915 = ppgtt->vm.i915; @@ -364,6 +389,16 @@ static int gen8_ppgtt_alloc(struct i915_address_space *vm, return err; } +static __always_inline void +write_pte(gen8_pte_t *pte, const gen8_pte_t val) +{ + /* Magic delays? Or can we refine these to flush all in one pass? */ + *pte = val; + wmb(); /* cpu to cache */ + clflush(pte); /* cache to memory */ + wmb(); /* visible to all */ +} + static __always_inline u64 gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, struct i915_page_directory *pdp, @@ -380,7 +415,8 @@ gen8_ppgtt_insert_pte(struct i915_ppgtt *ppgtt, vaddr = kmap_atomic_px(i915_pt_entry(pd, gen8_pd_index(idx, 1))); do { GEM_BUG_ON(iter->sg->length < I915_GTT_PAGE_SIZE); - vaddr[gen8_pd_index(idx, 0)] = pte_encode | iter->dma; + write_pte(&vaddr[gen8_pd_index(idx, 0)], + pte_encode | iter->dma); iter->dma += I915_GTT_PAGE_SIZE; if (iter->dma >= iter->max) { @@ -462,7 +498,7 @@ static void gen8_ppgtt_insert_huge(struct i915_vma *vma, do { GEM_BUG_ON(iter->sg->length < page_size); - vaddr[index++] = encode | iter->dma; + write_pte(&vaddr[index++], encode | iter->dma); start += page_size; iter->dma += page_size; @@ -706,6 +742,8 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt) ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc; ppgtt->vm.clear_range = gen8_ppgtt_clear; + ppgtt->vm.pte_encode = gen8_pte_encode; + if (intel_vgpu_active(gt->i915)) gen8_ppgtt_notify_vgt(ppgtt, true); diff --git a/drivers/gpu/drm/i915/gt/hsw_clear_kernel.c b/drivers/gpu/drm/i915/gt/hsw_clear_kernel.c new file mode 100644 index 000000000000..b47f9d4a0848 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/hsw_clear_kernel.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + * + * Generated by: IGT Gpu Tools on Fri 21 Feb 2020 05:30:13 AM UTC + */ + +static const u32 hsw_clear_kernel[] = { + 0x00000001, 0x26020128, 0x00000024, 0x00000000, + 0x00000040, 0x20280c21, 0x00000028, 0x00000001, + 0x01000010, 0x20000c20, 0x0000002c, 0x00000000, + 0x00010220, 0x34001c00, 0x00001400, 0x00000160, + 0x00600001, 0x20600061, 0x00000000, 0x00000000, + 0x00000008, 0x20601c85, 0x00000e00, 0x0000000c, + 0x00000005, 0x20601ca5, 0x00000060, 0x00000001, + 0x00000008, 0x20641c85, 0x00000e00, 0x0000000d, + 0x00000005, 0x20641ca5, 0x00000064, 0x00000003, + 0x00000041, 0x207424a5, 0x00000064, 0x00000034, + 0x00000040, 0x206014a5, 0x00000060, 0x00000074, + 0x00000008, 0x20681c85, 0x00000e00, 0x00000008, + 0x00000005, 0x20681ca5, 0x00000068, 0x0000000f, + 0x00000041, 0x20701ca5, 0x00000060, 0x00000010, + 0x00000040, 0x206814a5, 0x00000068, 0x00000070, + 0x00600001, 0x20a00061, 0x00000000, 0x00000000, + 0x00000005, 0x206c1c85, 0x00000e00, 0x00000007, + 0x00000041, 0x206c1ca5, 0x0000006c, 0x00000004, + 0x00600001, 0x20800021, 0x008d0000, 0x00000000, + 0x00000001, 0x20800021, 0x0000006c, 0x00000000, + 0x00000001, 0x20840021, 0x00000068, 0x00000000, + 0x00000001, 0x20880061, 0x00000000, 0x00000003, + 0x00000005, 0x208c0d21, 0x00000086, 0xffffffff, + 0x05600032, 0x20a00fa1, 0x008d0080, 0x02190001, + 0x00000040, 0x20a01ca5, 0x000000a0, 0x00000001, + 0x05600032, 0x20a00fa1, 0x008d0080, 0x040a8001, + 0x02000040, 0x20281c21, 0x00000028, 0xffffffff, + 0x00010220, 0x34001c00, 0x00001400, 0xffffffe0, + 0x00000001, 0x26020128, 0x00000024, 0x00000000, + 0x00000001, 0x220010e4, 0x00000000, 0x00000000, + 0x00000001, 0x220831ec, 0x00000000, 0x007f007f, + 0x00600001, 0x20400021, 0x008d0000, 0x00000000, + 0x00600001, 0x2fe00021, 0x008d0000, 0x00000000, + 0x00200001, 0x20400121, 0x00450020, 0x00000000, + 0x00000001, 0x20480061, 0x00000000, 0x000f000f, + 0x00000005, 0x204c0d21, 0x00000046, 0xffffffef, + 0x00800001, 0x20600061, 0x00000000, 0x00000000, + 0x00800001, 0x20800061, 0x00000000, 0x00000000, + 0x00800001, 0x20a00061, 0x00000000, 0x00000000, + 0x00800001, 0x20c00061, 0x00000000, 0x00000000, + 0x00800001, 0x20e00061, 0x00000000, 0x00000000, + 0x00800001, 0x21000061, 0x00000000, 0x00000000, + 0x00800001, 0x21200061, 0x00000000, 0x00000000, + 0x00800001, 0x21400061, 0x00000000, 0x00000000, + 0x05600032, 0x20000fa0, 0x008d0040, 0x120a8000, + 0x00000040, 0x20402d21, 0x00000020, 0x00100010, + 0x05600032, 0x20000fa0, 0x008d0040, 0x120a8000, + 0x02000040, 0x22083d8c, 0x00000208, 0xffffffff, + 0x00800001, 0xa0000109, 0x00000602, 0x00000000, + 0x00000040, 0x22001c84, 0x00000200, 0x00000020, + 0x00010220, 0x34001c00, 0x00001400, 0xffffffc0, + 0x07600032, 0x20000fa0, 0x008d0fe0, 0x82000010, +}; diff --git a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c index cbad7fe722ce..d907d538176e 100644 --- a/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c +++ b/drivers/gpu/drm/i915/gt/intel_breadcrumbs.c @@ -64,7 +64,7 @@ static void __intel_breadcrumbs_disarm_irq(struct intel_breadcrumbs *b) if (!--b->irq_enabled) irq_disable(engine); - b->irq_armed = false; + WRITE_ONCE(b->irq_armed, false); intel_gt_pm_put_async(engine->gt); } @@ -73,7 +73,7 @@ void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine) struct intel_breadcrumbs *b = &engine->breadcrumbs; unsigned long flags; - if (!b->irq_armed) + if (!READ_ONCE(b->irq_armed)) return; spin_lock_irqsave(&b->irq_lock, flags); @@ -142,6 +142,18 @@ static void add_retire(struct intel_breadcrumbs *b, struct intel_timeline *tl) intel_engine_add_retire(engine, tl); } +static void __signal_request(struct i915_request *rq, struct list_head *signals) +{ + GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags)); + clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); + + if (!__dma_fence_signal(&rq->fence)) + return; + + i915_request_get(rq); + list_add_tail(&rq->signal_link, signals); +} + static void signal_irq_work(struct irq_work *work) { struct intel_breadcrumbs *b = container_of(work, typeof(*b), irq_work); @@ -155,6 +167,8 @@ static void signal_irq_work(struct irq_work *work) if (b->irq_armed && list_empty(&b->signalers)) __intel_breadcrumbs_disarm_irq(b); + list_splice_init(&b->signaled_requests, &signal); + list_for_each_entry_safe(ce, cn, &b->signalers, signal_link) { GEM_BUG_ON(list_empty(&ce->signals)); @@ -163,24 +177,15 @@ static void signal_irq_work(struct irq_work *work) list_entry(pos, typeof(*rq), signal_link); GEM_BUG_ON(!check_signal_order(ce, rq)); - if (!__request_completed(rq)) break; - GEM_BUG_ON(!test_bit(I915_FENCE_FLAG_SIGNAL, - &rq->fence.flags)); - clear_bit(I915_FENCE_FLAG_SIGNAL, &rq->fence.flags); - - if (!__dma_fence_signal(&rq->fence)) - continue; - /* * Queue for execution after dropping the signaling * spinlock as the callback chain may end up adding * more signalers to the same context or engine. */ - i915_request_get(rq); - list_add_tail(&rq->signal_link, &signal); + __signal_request(rq, &signal); } /* @@ -233,7 +238,7 @@ static bool __intel_breadcrumbs_arm_irq(struct intel_breadcrumbs *b) * which we can add a new waiter and avoid the cost of re-enabling * the irq. */ - b->irq_armed = true; + WRITE_ONCE(b->irq_armed, true); /* * Since we are waiting on a request, the GPU should be busy @@ -255,6 +260,7 @@ void intel_engine_init_breadcrumbs(struct intel_engine_cs *engine) spin_lock_init(&b->irq_lock); INIT_LIST_HEAD(&b->signalers); + INIT_LIST_HEAD(&b->signaled_requests); init_irq_work(&b->irq_work, signal_irq_work); } @@ -274,6 +280,32 @@ void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine) spin_unlock_irqrestore(&b->irq_lock, flags); } +void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine, + struct intel_context *ce) +{ + struct intel_breadcrumbs *b = &engine->breadcrumbs; + unsigned long flags; + + spin_lock_irqsave(&b->irq_lock, flags); + if (!list_empty(&ce->signals)) { + struct i915_request *rq, *next; + + /* Queue for executing the signal callbacks in the irq_work */ + list_for_each_entry_safe(rq, next, &ce->signals, signal_link) { + GEM_BUG_ON(rq->engine != engine); + GEM_BUG_ON(!__request_completed(rq)); + + __signal_request(rq, &b->signaled_requests); + } + + INIT_LIST_HEAD(&ce->signals); + list_del_init(&ce->signal_link); + + irq_work_queue(&b->irq_work); + } + spin_unlock_irqrestore(&b->irq_lock, flags); +} + void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine) { } diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index 8bb444cda14f..74ddb49b2941 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -51,6 +51,11 @@ int intel_context_alloc_state(struct intel_context *ce) return -EINTR; if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { + if (intel_context_is_banned(ce)) { + err = -EIO; + goto unlock; + } + err = ce->ops->alloc(ce); if (unlikely(err)) goto unlock; @@ -92,6 +97,8 @@ int __intel_context_do_pin(struct intel_context *ce) { int err; + GEM_BUG_ON(intel_context_is_closed(ce)); + if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) { err = intel_context_alloc_state(ce); if (err) @@ -107,6 +114,11 @@ int __intel_context_do_pin(struct intel_context *ce) goto out_release; } + if (unlikely(intel_context_is_closed(ce))) { + err = -ENOENT; + goto out_unlock; + } + if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) { err = intel_context_active_acquire(ce); if (unlikely(err)) diff --git a/drivers/gpu/drm/i915/gt/intel_context.h b/drivers/gpu/drm/i915/gt/intel_context.h index 18efad255124..07be021882cc 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.h +++ b/drivers/gpu/drm/i915/gt/intel_context.h @@ -173,6 +173,11 @@ static inline bool intel_context_is_barrier(const struct intel_context *ce) return test_bit(CONTEXT_BARRIER_BIT, &ce->flags); } +static inline bool intel_context_is_closed(const struct intel_context *ce) +{ + return test_bit(CONTEXT_CLOSED_BIT, &ce->flags); +} + static inline bool intel_context_use_semaphores(const struct intel_context *ce) { return test_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); diff --git a/drivers/gpu/drm/i915/gt/intel_context_param.c b/drivers/gpu/drm/i915/gt/intel_context_param.c new file mode 100644 index 000000000000..65dcd090245d --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_context_param.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include "i915_active.h" +#include "intel_context.h" +#include "intel_context_param.h" +#include "intel_ring.h" + +int intel_context_set_ring_size(struct intel_context *ce, long sz) +{ + int err; + + if (intel_context_lock_pinned(ce)) + return -EINTR; + + err = i915_active_wait(&ce->active); + if (err < 0) + goto unlock; + + if (intel_context_is_pinned(ce)) { + err = -EBUSY; /* In active use, come back later! */ + goto unlock; + } + + if (test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { + struct intel_ring *ring; + + /* Replace the existing ringbuffer */ + ring = intel_engine_create_ring(ce->engine, sz); + if (IS_ERR(ring)) { + err = PTR_ERR(ring); + goto unlock; + } + + intel_ring_put(ce->ring); + ce->ring = ring; + + /* Context image will be updated on next pin */ + } else { + ce->ring = __intel_context_ring_size(sz); + } + +unlock: + intel_context_unlock_pinned(ce); + return err; +} + +long intel_context_get_ring_size(struct intel_context *ce) +{ + long sz = (unsigned long)READ_ONCE(ce->ring); + + if (test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { + if (intel_context_lock_pinned(ce)) + return -EINTR; + + sz = ce->ring->size; + intel_context_unlock_pinned(ce); + } + + return sz; +} diff --git a/drivers/gpu/drm/i915/gt/intel_context_param.h b/drivers/gpu/drm/i915/gt/intel_context_param.h new file mode 100644 index 000000000000..f053d8633fe2 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_context_param.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef INTEL_CONTEXT_PARAM_H +#define INTEL_CONTEXT_PARAM_H + +struct intel_context; + +int intel_context_set_ring_size(struct intel_context *ce, long sz); +long intel_context_get_ring_size(struct intel_context *ce); + +#endif /* INTEL_CONTEXT_PARAM_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_context_sseu.c b/drivers/gpu/drm/i915/gt/intel_context_sseu.c index 57a30956c922..487299cb91f2 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_context_sseu.c @@ -25,8 +25,7 @@ static int gen8_emit_rpcs_config(struct i915_request *rq, return PTR_ERR(cs); offset = i915_ggtt_offset(ce->state) + - LRC_STATE_PN * PAGE_SIZE + - CTX_R_PWR_CLK_STATE * 4; + LRC_STATE_OFFSET + CTX_R_PWR_CLK_STATE * 4; *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *cs++ = lower_32_bits(offset); diff --git a/drivers/gpu/drm/i915/gt/intel_context_types.h b/drivers/gpu/drm/i915/gt/intel_context_types.h index 11278343b9b5..4954b0df4864 100644 --- a/drivers/gpu/drm/i915/gt/intel_context_types.h +++ b/drivers/gpu/drm/i915/gt/intel_context_types.h @@ -45,8 +45,8 @@ struct intel_context { struct intel_engine_cs *engine; struct intel_engine_cs *inflight; -#define intel_context_inflight(ce) ptr_mask_bits((ce)->inflight, 2) -#define intel_context_inflight_count(ce) ptr_unmask_bits((ce)->inflight, 2) +#define intel_context_inflight(ce) ptr_mask_bits(READ_ONCE((ce)->inflight), 2) +#define intel_context_inflight_count(ce) ptr_unmask_bits(READ_ONCE((ce)->inflight), 2) struct i915_address_space *vm; struct i915_gem_context __rcu *gem_context; @@ -62,13 +62,20 @@ struct intel_context { #define CONTEXT_BARRIER_BIT 0 #define CONTEXT_ALLOC_BIT 1 #define CONTEXT_VALID_BIT 2 -#define CONTEXT_USE_SEMAPHORES 3 -#define CONTEXT_BANNED 4 -#define CONTEXT_FORCE_SINGLE_SUBMISSION 5 -#define CONTEXT_NOPREEMPT 6 +#define CONTEXT_CLOSED_BIT 3 +#define CONTEXT_USE_SEMAPHORES 4 +#define CONTEXT_BANNED 5 +#define CONTEXT_FORCE_SINGLE_SUBMISSION 6 +#define CONTEXT_NOPREEMPT 7 u32 *lrc_reg_state; - u64 lrc_desc; + union { + struct { + u32 lrca; + u32 ccid; + }; + u64 desc; + } lrc; u32 tag; /* cookie passed to HW to track this context on submission */ /* Time on GPU as tracked by the hw. */ @@ -95,6 +102,8 @@ struct intel_context { /** sseu: Control eu/slice partitioning */ struct intel_sseu sseu; + + u8 wa_bb_page; /* if set, page num reserved for context workarounds */ }; #endif /* __INTEL_CONTEXT_TYPES__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine.h b/drivers/gpu/drm/i915/gt/intel_engine.h index 29c8c03c5caa..9bf6d4989968 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine.h +++ b/drivers/gpu/drm/i915/gt/intel_engine.h @@ -107,7 +107,20 @@ execlists_num_ports(const struct intel_engine_execlists * const execlists) static inline struct i915_request * execlists_active(const struct intel_engine_execlists *execlists) { - return *READ_ONCE(execlists->active); + struct i915_request * const *cur, * const *old, *active; + + cur = READ_ONCE(execlists->active); + smp_rmb(); /* pairs with overwrite protection in process_csb() */ + do { + old = cur; + + active = READ_ONCE(*cur); + cur = READ_ONCE(execlists->active); + + smp_rmb(); /* and complete the seqlock retry */ + } while (unlikely(cur != old)); + + return active; } static inline void @@ -186,6 +199,8 @@ void intel_engine_cleanup(struct intel_engine_cs *engine); int intel_engines_init_mmio(struct intel_gt *gt); int intel_engines_init(struct intel_gt *gt); +void intel_engine_free_request_pool(struct intel_engine_cs *engine); + void intel_engines_release(struct intel_gt *gt); void intel_engines_free(struct intel_gt *gt); @@ -223,22 +238,35 @@ intel_engine_signal_breadcrumbs(struct intel_engine_cs *engine) void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine); void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine); +void intel_engine_transfer_stale_breadcrumbs(struct intel_engine_cs *engine, + struct intel_context *ce); + void intel_engine_print_breadcrumbs(struct intel_engine_cs *engine, struct drm_printer *p); -static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) +static inline u32 *__gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset) { memset(batch, 0, 6 * sizeof(u32)); - batch[0] = GFX_OP_PIPE_CONTROL(6); - batch[1] = flags; + batch[0] = GFX_OP_PIPE_CONTROL(6) | flags0; + batch[1] = flags1; batch[2] = offset; return batch + 6; } +static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset) +{ + return __gen8_emit_pipe_control(batch, 0, flags, offset); +} + +static inline u32 *gen12_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset) +{ + return __gen8_emit_pipe_control(batch, flags0, flags1, offset); +} + static inline u32 * -gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags) +__gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags0, u32 flags1) { /* We're using qword write, offset should be aligned to 8 bytes. */ GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8)); @@ -247,8 +275,8 @@ gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags) * need a prior CS_STALL, which is emitted by the flush * following the batch. */ - *cs++ = GFX_OP_PIPE_CONTROL(6); - *cs++ = flags | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_GLOBAL_GTT_IVB; + *cs++ = GFX_OP_PIPE_CONTROL(6) | flags0; + *cs++ = flags1 | PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_GLOBAL_GTT_IVB; *cs++ = gtt_offset; *cs++ = 0; *cs++ = value; @@ -258,6 +286,18 @@ gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags) return cs; } +static inline u32* +gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags) +{ + return __gen8_emit_ggtt_write_rcs(cs, value, gtt_offset, 0, flags); +} + +static inline u32* +gen12_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset, u32 flags0, u32 flags1) +{ + return __gen8_emit_ggtt_write_rcs(cs, value, gtt_offset, flags0, flags1); +} + static inline u32 * gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset, u32 flags) { @@ -295,9 +335,6 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m, const char *header, ...); -int intel_enable_engine_stats(struct intel_engine_cs *engine); -void intel_disable_engine_stats(struct intel_engine_cs *engine); - ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine); struct i915_request * @@ -320,13 +357,4 @@ intel_engine_has_preempt_reset(const struct intel_engine_cs *engine) return intel_engine_has_preemption(engine); } -static inline bool -intel_engine_has_timeslices(const struct intel_engine_cs *engine) -{ - if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) - return false; - - return intel_engine_has_semaphores(engine); -} - #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index e46e55354e95..da5b61085257 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -31,7 +31,6 @@ #include "intel_context.h" #include "intel_engine.h" #include "intel_engine_pm.h" -#include "intel_engine_pool.h" #include "intel_engine_user.h" #include "intel_gt.h" #include "intel_gt_requests.h" @@ -275,6 +274,7 @@ static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) { const struct engine_info *info = &intel_engines[id]; + struct drm_i915_private *i915 = gt->i915; struct intel_engine_cs *engine; BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH)); @@ -301,11 +301,11 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->id = id; engine->legacy_idx = INVALID_ENGINE; engine->mask = BIT(id); - engine->i915 = gt->i915; + engine->i915 = i915; engine->gt = gt; engine->uncore = gt->uncore; engine->hw_id = engine->guc_id = info->hw_id; - engine->mmio_base = __engine_mmio_base(gt->i915, info->mmio_bases); + engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); engine->class = info->class; engine->instance = info->instance; @@ -313,6 +313,8 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->props.heartbeat_interval_ms = CONFIG_DRM_I915_HEARTBEAT_INTERVAL; + engine->props.max_busywait_duration_ns = + CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT; engine->props.preempt_timeout_ms = CONFIG_DRM_I915_PREEMPT_TIMEOUT; engine->props.stop_timeout_ms = @@ -320,11 +322,17 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) engine->props.timeslice_duration_ms = CONFIG_DRM_I915_TIMESLICE_DURATION; + /* Override to uninterruptible for OpenCL workloads. */ + if (INTEL_GEN(i915) == 12 && engine->class == RENDER_CLASS) + engine->props.preempt_timeout_ms = 0; + + engine->defaults = engine->props; /* never to change again */ + engine->context_size = intel_engine_context_size(gt, engine->class); if (WARN_ON(engine->context_size > BIT(20))) engine->context_size = 0; if (engine->context_size) - DRIVER_CAPS(gt->i915)->has_logical_contexts = true; + DRIVER_CAPS(i915)->has_logical_contexts = true; /* Nothing to do here, execute in order of dependencies */ engine->schedule = NULL; @@ -340,8 +348,6 @@ static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id) gt->engine_class[info->class][info->instance] = engine; gt->engine[id] = engine; - gt->i915->engine[id] = engine; - return 0; } @@ -418,17 +424,27 @@ void intel_engines_release(struct intel_gt *gt) engine->release = NULL; memset(&engine->reset, 0, sizeof(engine->reset)); - - gt->i915->engine[id] = NULL; } } +void intel_engine_free_request_pool(struct intel_engine_cs *engine) +{ + if (!engine->request_pool) + return; + + kmem_cache_free(i915_request_slab_cache(), engine->request_pool); +} + void intel_engines_free(struct intel_gt *gt) { struct intel_engine_cs *engine; enum intel_engine_id id; + /* Free the requests! dma-resv keeps fences around for an eternity */ + rcu_barrier(); + for_each_engine(engine, gt, id) { + intel_engine_free_request_pool(engine); kfree(engine); gt->engine[id] = NULL; } @@ -616,8 +632,6 @@ static int engine_setup_common(struct intel_engine_cs *engine) intel_engine_init__pm(engine); intel_engine_init_retire(engine); - intel_engine_pool_init(&engine->pool); - /* Use the whole device by default */ engine->sseu = intel_sseu_from_device_info(&RUNTIME_INFO(engine->i915)->sseu); @@ -639,7 +653,7 @@ static int measure_breadcrumb_dw(struct intel_context *ce) { struct intel_engine_cs *engine = ce->engine; struct measure_breadcrumb *frame; - int dw = -ENOMEM; + int dw; GEM_BUG_ON(!engine->gt->scratch); @@ -814,12 +828,11 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) cleanup_status_page(engine); intel_engine_fini_retire(engine); - intel_engine_pool_fini(&engine->pool); intel_engine_fini_breadcrumbs(engine); intel_engine_cleanup_cmd_parser(engine); if (engine->default_state) - i915_gem_object_put(engine->default_state); + fput(engine->default_state); if (engine->kernel_context) { intel_context_unpin(engine->kernel_context); @@ -1218,6 +1231,49 @@ static void print_request(struct drm_printer *m, name); } +static struct intel_timeline *get_timeline(struct i915_request *rq) +{ + struct intel_timeline *tl; + + /* + * Even though we are holding the engine->active.lock here, there + * is no control over the submission queue per-se and we are + * inspecting the active state at a random point in time, with an + * unknown queue. Play safe and make sure the timeline remains valid. + * (Only being used for pretty printing, one extra kref shouldn't + * cause a camel stampede!) + */ + rcu_read_lock(); + tl = rcu_dereference(rq->timeline); + if (!kref_get_unless_zero(&tl->kref)) + tl = NULL; + rcu_read_unlock(); + + return tl; +} + +static int print_ring(char *buf, int sz, struct i915_request *rq) +{ + int len = 0; + + if (!i915_request_signaled(rq)) { + struct intel_timeline *tl = get_timeline(rq); + + len = scnprintf(buf, sz, + "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ", + i915_ggtt_offset(rq->ring->vma), + tl ? tl->hwsp_offset : 0, + hwsp_seqno(rq), + DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context), + 1000 * 1000)); + + if (tl) + intel_timeline_put(tl); + } + + return len; +} + static void hexdump(struct drm_printer *m, const void *buf, size_t len) { const size_t rowsize = 8 * sizeof(u32); @@ -1247,27 +1303,6 @@ static void hexdump(struct drm_printer *m, const void *buf, size_t len) } } -static struct intel_timeline *get_timeline(struct i915_request *rq) -{ - struct intel_timeline *tl; - - /* - * Even though we are holding the engine->active.lock here, there - * is no control over the submission queue per-se and we are - * inspecting the active state at a random point in time, with an - * unknown queue. Play safe and make sure the timeline remains valid. - * (Only being used for pretty printing, one extra kref shouldn't - * cause a camel stampede!) - */ - rcu_read_lock(); - tl = rcu_dereference(rq->timeline); - if (!kref_get_unless_zero(&tl->kref)) - tl = NULL; - rcu_read_unlock(); - - return tl; -} - static const char *repr_timer(const struct timer_list *t) { if (!READ_ONCE(t->expires)) @@ -1288,6 +1323,12 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, if (engine->id == RENDER_CLASS && IS_GEN_RANGE(dev_priv, 4, 7)) drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); + if (HAS_EXECLISTS(dev_priv)) { + drm_printf(m, "\tEL_STAT_HI: 0x%08x\n", + ENGINE_READ(engine, RING_EXECLIST_STATUS_HI)); + drm_printf(m, "\tEL_STAT_LO: 0x%08x\n", + ENGINE_READ(engine, RING_EXECLIST_STATUS_LO)); + } drm_printf(m, "\tRING_START: 0x%08x\n", ENGINE_READ(engine, RING_START)); drm_printf(m, "\tRING_HEAD: 0x%08x\n", @@ -1379,40 +1420,25 @@ static void intel_engine_print_registers(struct intel_engine_cs *engine, char hdr[160]; int len; - len = snprintf(hdr, sizeof(hdr), - "\t\tActive[%d]: ", - (int)(port - execlists->active)); - if (!i915_request_signaled(rq)) { - struct intel_timeline *tl = get_timeline(rq); - - len += snprintf(hdr + len, sizeof(hdr) - len, - "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ", - i915_ggtt_offset(rq->ring->vma), - tl ? tl->hwsp_offset : 0, - hwsp_seqno(rq), - DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context), - 1000 * 1000)); - - if (tl) - intel_timeline_put(tl); - } - snprintf(hdr + len, sizeof(hdr) - len, "rq: "); + len = scnprintf(hdr, sizeof(hdr), + "\t\tActive[%d]: ccid:%08x, ", + (int)(port - execlists->active), + rq->context->lrc.ccid); + len += print_ring(hdr + len, sizeof(hdr) - len, rq); + scnprintf(hdr + len, sizeof(hdr) - len, "rq: "); print_request(m, rq, hdr); } for (port = execlists->pending; (rq = *port); port++) { - struct intel_timeline *tl = get_timeline(rq); - char hdr[80]; - - snprintf(hdr, sizeof(hdr), - "\t\tPending[%d] ring:{start:%08x, hwsp:%08x, seqno:%08x}, rq: ", - (int)(port - execlists->pending), - i915_ggtt_offset(rq->ring->vma), - tl ? tl->hwsp_offset : 0, - hwsp_seqno(rq)); - print_request(m, rq, hdr); + char hdr[160]; + int len; - if (tl) - intel_timeline_put(tl); + len = scnprintf(hdr, sizeof(hdr), + "\t\tPending[%d]: ccid:%08x, ", + (int)(port - execlists->pending), + rq->context->lrc.ccid); + len += print_ring(hdr + len, sizeof(hdr) - len, rq); + scnprintf(hdr + len, sizeof(hdr) - len, "rq: "); + print_request(m, rq, hdr); } rcu_read_unlock(); execlists_active_unlock_bh(execlists); @@ -1561,58 +1587,6 @@ void intel_engine_dump(struct intel_engine_cs *engine, intel_engine_print_breadcrumbs(engine, m); } -/** - * intel_enable_engine_stats() - Enable engine busy tracking on engine - * @engine: engine to enable stats collection - * - * Start collecting the engine busyness data for @engine. - * - * Returns 0 on success or a negative error code. - */ -int intel_enable_engine_stats(struct intel_engine_cs *engine) -{ - struct intel_engine_execlists *execlists = &engine->execlists; - unsigned long flags; - int err = 0; - - if (!intel_engine_supports_stats(engine)) - return -ENODEV; - - execlists_active_lock_bh(execlists); - write_seqlock_irqsave(&engine->stats.lock, flags); - - if (unlikely(engine->stats.enabled == ~0)) { - err = -EBUSY; - goto unlock; - } - - if (engine->stats.enabled++ == 0) { - struct i915_request * const *port; - struct i915_request *rq; - - engine->stats.enabled_at = ktime_get(); - - /* XXX submission method oblivious? */ - for (port = execlists->active; (rq = *port); port++) - engine->stats.active++; - - for (port = execlists->pending; (rq = *port); port++) { - /* Exclude any contexts already counted in active */ - if (!intel_context_inflight_count(rq->context)) - engine->stats.active++; - } - - if (engine->stats.active) - engine->stats.start = engine->stats.enabled_at; - } - -unlock: - write_sequnlock_irqrestore(&engine->stats.lock, flags); - execlists_active_unlock_bh(execlists); - - return err; -} - static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine) { ktime_t total = engine->stats.total; @@ -1621,7 +1595,7 @@ static ktime_t __intel_engine_get_busy_time(struct intel_engine_cs *engine) * If the engine is executing something at the moment * add it to the total. */ - if (engine->stats.active) + if (atomic_read(&engine->stats.active)) total = ktime_add(total, ktime_sub(ktime_get(), engine->stats.start)); @@ -1647,28 +1621,6 @@ ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine) return total; } -/** - * intel_disable_engine_stats() - Disable engine busy tracking on engine - * @engine: engine to disable stats collection - * - * Stops collecting the engine busyness data for @engine. - */ -void intel_disable_engine_stats(struct intel_engine_cs *engine) -{ - unsigned long flags; - - if (!intel_engine_supports_stats(engine)) - return; - - write_seqlock_irqsave(&engine->stats.lock, flags); - WARN_ON_ONCE(engine->stats.enabled == 0); - if (--engine->stats.enabled == 0) { - engine->stats.total = __intel_engine_get_busy_time(engine); - engine->stats.active = 0; - } - write_sequnlock_irqrestore(&engine->stats.lock, flags); -} - static bool match_ring(struct i915_request *rq) { u32 ring = ENGINE_READ(rq->engine, RING_START); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c index dd825718e4e5..5136c8bf112d 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c @@ -31,7 +31,7 @@ static bool next_heartbeat(struct intel_engine_cs *engine) delay = msecs_to_jiffies_timeout(delay); if (delay >= HZ) delay = round_jiffies_up_relative(delay); - schedule_delayed_work(&engine->heartbeat.work, delay); + mod_delayed_work(system_wq, &engine->heartbeat.work, delay); return true; } diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.c b/drivers/gpu/drm/i915/gt/intel_engine_pm.c index b6cf284e3a2d..d0a1078ef632 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.c @@ -10,31 +10,22 @@ #include "intel_engine.h" #include "intel_engine_heartbeat.h" #include "intel_engine_pm.h" -#include "intel_engine_pool.h" #include "intel_gt.h" #include "intel_gt_pm.h" #include "intel_rc6.h" #include "intel_ring.h" +#include "shmem_utils.h" static int __engine_unpark(struct intel_wakeref *wf) { struct intel_engine_cs *engine = container_of(wf, typeof(*engine), wakeref); struct intel_context *ce; - void *map; ENGINE_TRACE(engine, "\n"); intel_gt_pm_get(engine->gt); - /* Pin the default state for fast resets from atomic context. */ - map = NULL; - if (engine->default_state) - map = i915_gem_object_pin_map(engine->default_state, - I915_MAP_WB); - if (!IS_ERR_OR_NULL(map)) - engine->pinned_default_state = map; - /* Discard stale context state from across idling */ ce = engine->kernel_context; if (ce) { @@ -44,6 +35,7 @@ static int __engine_unpark(struct intel_wakeref *wf) if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) && ce->state) { struct drm_i915_gem_object *obj = ce->state->obj; int type = i915_coherent_map_type(engine->i915); + void *map; map = i915_gem_object_pin_map(obj, type); if (!IS_ERR(map)) { @@ -181,7 +173,7 @@ static bool switch_to_kernel_context(struct intel_engine_cs *engine) * Ergo, if we put ourselves on the timelines.active_list * (se intel_timeline_enter()) before we increment the * engine->wakeref.count, we may see the request completion and retire - * it causing an undeflow of the engine->wakeref. + * it causing an underflow of the engine->wakeref. */ flags = __timeline_mark_lock(ce); GEM_BUG_ON(atomic_read(&ce->timeline->active_count) < 0); @@ -255,7 +247,6 @@ static int __engine_park(struct intel_wakeref *wf) intel_engine_park_heartbeat(engine); intel_engine_disarm_breadcrumbs(engine); - intel_engine_pool_park(&engine->pool); /* Must be reset upon idling, or we may miss the busy wakeup. */ GEM_BUG_ON(engine->execlists.queue_priority_hint != INT_MIN); @@ -263,11 +254,6 @@ static int __engine_park(struct intel_wakeref *wf) if (engine->park) engine->park(engine); - if (engine->pinned_default_state) { - i915_gem_object_unpin_map(engine->default_state); - engine->pinned_default_state = NULL; - } - engine->execlists.no_priolist = false; /* While gt calls i915_vma_parked(), we have to break the lock cycle */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pm.h b/drivers/gpu/drm/i915/gt/intel_engine_pm.h index e52c2b0cb245..418df0a13145 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pm.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_pm.h @@ -37,6 +37,12 @@ static inline void intel_engine_pm_put_async(struct intel_engine_cs *engine) intel_wakeref_put_async(&engine->wakeref); } +static inline void intel_engine_pm_put_delay(struct intel_engine_cs *engine, + unsigned long delay) +{ + intel_wakeref_put_delay(&engine->wakeref, delay); +} + static inline void intel_engine_pm_flush(struct intel_engine_cs *engine) { intel_wakeref_unlock_wait(&engine->wakeref); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pool.h b/drivers/gpu/drm/i915/gt/intel_engine_pool.h deleted file mode 100644 index 1bd89cadc3b7..000000000000 --- a/drivers/gpu/drm/i915/gt/intel_engine_pool.h +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SPDX-License-Identifier: MIT - * - * Copyright © 2014-2018 Intel Corporation - */ - -#ifndef INTEL_ENGINE_POOL_H -#define INTEL_ENGINE_POOL_H - -#include "intel_engine_pool_types.h" -#include "i915_active.h" -#include "i915_request.h" - -struct intel_engine_pool_node * -intel_engine_get_pool(struct intel_engine_cs *engine, size_t size); - -static inline int -intel_engine_pool_mark_active(struct intel_engine_pool_node *node, - struct i915_request *rq) -{ - return i915_active_add_request(&node->active, rq); -} - -static inline void -intel_engine_pool_put(struct intel_engine_pool_node *node) -{ - i915_active_release(&node->active); -} - -void intel_engine_pool_init(struct intel_engine_pool *pool); -void intel_engine_pool_park(struct intel_engine_pool *pool); -void intel_engine_pool_fini(struct intel_engine_pool *pool); - -#endif /* INTEL_ENGINE_POOL_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index b23366a81048..2b6cdf47d428 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -22,7 +22,6 @@ #include "i915_pmu.h" #include "i915_priolist_types.h" #include "i915_selftest.h" -#include "intel_engine_pool_types.h" #include "intel_sseu.h" #include "intel_timeline_types.h" #include "intel_wakeref.h" @@ -157,6 +156,20 @@ struct intel_engine_execlists { struct i915_priolist default_priolist; /** + * @ccid: identifier for contexts submitted to this engine + */ + u32 ccid; + + /** + * @yield: CCID at the time of the last semaphore-wait interrupt. + * + * Instead of leaving a semaphore busy-spinning on an engine, we would + * like to switch to another ready context, i.e. yielding the semaphore + * timeslice. + */ + u32 yield; + + /** * @error_interrupt: CS Master EIR * * The CS generates an interrupt when it detects an error. We capture @@ -167,6 +180,11 @@ struct intel_engine_execlists { u32 error_interrupt; /** + * @reset_ccid: Active CCID [EXECLISTS_STATUS_HI] at the time of reset + */ + u32 reset_ccid; + + /** * @no_priolist: priority lists disabled */ bool no_priolist; @@ -295,8 +313,7 @@ struct intel_engine_cs { u32 context_size; u32 mmio_base; - unsigned int context_tag; -#define NUM_CONTEXT_TAG roundup_pow_of_two(2 * EXECLIST_MAX_PORTS) + unsigned long context_tag; struct rb_node uabi_node; @@ -308,6 +325,9 @@ struct intel_engine_cs { struct list_head hold; /* ready requests, but on hold */ } active; + /* keep a request in reserve for a [pm] barrier under oom */ + struct i915_request *request_pool; + struct llist_head barrier_tasks; struct intel_context *kernel_context; /* pinned */ @@ -323,8 +343,7 @@ struct intel_engine_cs { unsigned long wakeref_serial; struct intel_wakeref wakeref; - struct drm_i915_gem_object *default_state; - void *pinned_default_state; + struct file *default_state; struct { struct intel_ring *ring; @@ -358,6 +377,8 @@ struct intel_engine_cs { spinlock_t irq_lock; struct list_head signalers; + struct list_head signaled_requests; + struct irq_work irq_work; /* for use from inside irq_lock */ unsigned int irq_enabled; @@ -389,13 +410,6 @@ struct intel_engine_cs { struct i915_pmu_sample sample[I915_ENGINE_SAMPLE_COUNT]; } pmu; - /* - * A pool of objects to use as shadow copies of client batch buffers - * when the command parser is enabled. Prevents the client from - * modifying the batch contents after software parsing. - */ - struct intel_engine_pool pool; - struct intel_hw_status_page status_page; struct i915_ctx_workarounds wa_ctx; struct i915_wa_list ctx_wa_list; @@ -407,6 +421,7 @@ struct intel_engine_cs { void (*irq_enable)(struct intel_engine_cs *engine); void (*irq_disable)(struct intel_engine_cs *engine); + void (*sanitize)(struct intel_engine_cs *engine); int (*resume)(struct intel_engine_cs *engine); struct { @@ -483,10 +498,11 @@ struct intel_engine_cs { #define I915_ENGINE_SUPPORTS_STATS BIT(1) #define I915_ENGINE_HAS_PREEMPTION BIT(2) #define I915_ENGINE_HAS_SEMAPHORES BIT(3) -#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4) -#define I915_ENGINE_IS_VIRTUAL BIT(5) -#define I915_ENGINE_HAS_RELATIVE_MMIO BIT(6) -#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(7) +#define I915_ENGINE_HAS_TIMESLICES BIT(4) +#define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(5) +#define I915_ENGINE_IS_VIRTUAL BIT(6) +#define I915_ENGINE_HAS_RELATIVE_MMIO BIT(7) +#define I915_ENGINE_REQUIRES_CMD_PARSER BIT(8) unsigned int flags; /* @@ -515,42 +531,43 @@ struct intel_engine_cs { struct { /** - * @lock: Lock protecting the below fields. - */ - seqlock_t lock; - /** - * @enabled: Reference count indicating number of listeners. + * @active: Number of contexts currently scheduled in. */ - unsigned int enabled; + atomic_t active; + /** - * @active: Number of contexts currently scheduled in. + * @lock: Lock protecting the below fields. */ - unsigned int active; + seqlock_t lock; + /** - * @enabled_at: Timestamp when busy stats were enabled. + * @total: Total time this engine was busy. + * + * Accumulated time not counting the most recent block in cases + * where engine is currently busy (active > 0). */ - ktime_t enabled_at; + ktime_t total; + /** * @start: Timestamp of the last idle to active transition. * * Idle is defined as active == 0, active is active > 0. */ ktime_t start; + /** - * @total: Total time this engine was busy. - * - * Accumulated time not counting the most recent block in cases - * where engine is currently busy (active > 0). + * @rps: Utilisation at last RPS sampling. */ - ktime_t total; + ktime_t rps; } stats; struct { unsigned long heartbeat_interval_ms; + unsigned long max_busywait_duration_ns; unsigned long preempt_timeout_ms; unsigned long stop_timeout_ms; unsigned long timeslice_duration_ms; - } props; + } props, defaults; }; static inline bool @@ -584,6 +601,15 @@ intel_engine_has_semaphores(const struct intel_engine_cs *engine) } static inline bool +intel_engine_has_timeslices(const struct intel_engine_cs *engine) +{ + if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + return false; + + return engine->flags & I915_ENGINE_HAS_TIMESLICES; +} + +static inline bool intel_engine_needs_breadcrumb_tasklet(const struct intel_engine_cs *engine) { return engine->flags & I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 7dae91e0d002..66165b10256e 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -8,6 +8,8 @@ #include <asm/set_memory.h> #include <asm/smp.h> +#include <drm/i915_drm.h> + #include "intel_gt.h" #include "i915_drv.h" #include "i915_scatterlist.h" @@ -63,7 +65,7 @@ static int ggtt_init_hw(struct i915_ggtt *ggtt) ggtt->mappable_end); } - i915_ggtt_init_fences(ggtt); + intel_ggtt_init_fences(ggtt); return 0; } @@ -157,6 +159,13 @@ static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt) intel_gtt_chipset_flush(); } +static u64 gen8_ggtt_pte_encode(dma_addr_t addr, + enum i915_cache_level level, + u32 flags) +{ + return addr | _PAGE_PRESENT; +} + static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) { writeq(pte, addr); @@ -172,7 +181,7 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm, gen8_pte_t __iomem *pte = (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; - gen8_set_pte(pte, gen8_pte_encode(addr, level, 0)); + gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, 0)); ggtt->invalidate(ggtt); } @@ -182,10 +191,11 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, enum i915_cache_level level, u32 flags) { + const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, 0); struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - struct sgt_iter sgt_iter; - gen8_pte_t __iomem *gtt_entries; - const gen8_pte_t pte_encode = gen8_pte_encode(0, level, 0); + gen8_pte_t __iomem *gte; + gen8_pte_t __iomem *end; + struct sgt_iter iter; dma_addr_t addr; /* @@ -193,10 +203,17 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm, * not to allow the user to override access to a read only page. */ - gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm; - gtt_entries += vma->node.start / I915_GTT_PAGE_SIZE; - for_each_sgt_daddr(addr, sgt_iter, vma->pages) - gen8_set_pte(gtt_entries++, pte_encode | addr); + gte = (gen8_pte_t __iomem *)ggtt->gsm; + gte += vma->node.start / I915_GTT_PAGE_SIZE; + end = gte + vma->node.size / I915_GTT_PAGE_SIZE; + + for_each_sgt_daddr(addr, iter, vma->pages) + gen8_set_pte(gte++, pte_encode | addr); + GEM_BUG_ON(gte > end); + + /* Fill the allocated but "unused" space beyond the end of the buffer */ + while (gte < end) + gen8_set_pte(gte++, vm->scratch[0].encode); /* * We want to flush the TLBs only after we're certain all the PTE @@ -232,13 +249,22 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm, u32 flags) { struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); - gen6_pte_t __iomem *entries = (gen6_pte_t __iomem *)ggtt->gsm; - unsigned int i = vma->node.start / I915_GTT_PAGE_SIZE; + gen6_pte_t __iomem *gte; + gen6_pte_t __iomem *end; struct sgt_iter iter; dma_addr_t addr; + gte = (gen6_pte_t __iomem *)ggtt->gsm; + gte += vma->node.start / I915_GTT_PAGE_SIZE; + end = gte + vma->node.size / I915_GTT_PAGE_SIZE; + for_each_sgt_daddr(addr, iter, vma->pages) - iowrite32(vm->pte_encode(addr, level, flags), &entries[i++]); + iowrite32(vm->pte_encode(addr, level, flags), gte++); + GEM_BUG_ON(gte > end); + + /* Fill the allocated but "unused" space beyond the end of the buffer */ + while (gte < end) + iowrite32(vm->scratch[0].encode, gte++); /* * We want to flush the TLBs only after we're certain all the PTE @@ -427,7 +453,7 @@ static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) u64 size; int ret; - if (!USES_GUC(ggtt->vm.i915)) + if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) return 0; GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); @@ -689,11 +715,13 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) */ void i915_ggtt_driver_release(struct drm_i915_private *i915) { + struct i915_ggtt *ggtt = &i915->ggtt; struct pagevec *pvec; - fini_aliasing_ppgtt(&i915->ggtt); + fini_aliasing_ppgtt(ggtt); - ggtt_cleanup_hw(&i915->ggtt); + intel_ggtt_fini_fences(ggtt); + ggtt_cleanup_hw(ggtt); pvec = &i915->mm.wc_stash.pvec; if (pvec->nr) { @@ -754,17 +782,17 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) * readback check when writing GTT PTE entries. */ if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10) - ggtt->gsm = ioremap_nocache(phys_addr, size); + ggtt->gsm = ioremap(phys_addr, size); else ggtt->gsm = ioremap_wc(phys_addr, size); if (!ggtt->gsm) { - DRM_ERROR("Failed to map the ggtt page table\n"); + drm_err(&i915->drm, "Failed to map the ggtt page table\n"); return -ENOMEM; } ret = setup_scratch_page(&ggtt->vm, GFP_DMA32); if (ret) { - DRM_ERROR("Scratch setup failed\n"); + drm_err(&i915->drm, "Scratch setup failed\n"); /* iounmap will also get called at remove, but meh */ iounmap(ggtt->gsm); return ret; @@ -812,7 +840,6 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) struct pci_dev *pdev = i915->drm.pdev; unsigned int size; u16 snb_gmch_ctl; - int err; /* TODO: We're not aware of mappable constraints on gen8 yet */ if (!IS_DGFX(i915)) { @@ -820,12 +847,6 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->mappable_end = resource_size(&ggtt->gmadr); } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(39)); - if (!err) - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(39)); - if (err) - DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err); - pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); if (IS_CHERRYVIEW(i915)) size = chv_get_total_gtt_size(snb_gmch_ctl); @@ -857,7 +878,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.vma_ops.set_pages = ggtt_set_pages; ggtt->vm.vma_ops.clear_pages = clear_pages; - ggtt->vm.pte_encode = gen8_pte_encode; + ggtt->vm.pte_encode = gen8_ggtt_pte_encode; setup_private_pat(ggtt->vm.gt->uncore); @@ -961,7 +982,6 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt) struct pci_dev *pdev = i915->drm.pdev; unsigned int size; u16 snb_gmch_ctl; - int err; ggtt->gmadr = pci_resource(pdev, 2); ggtt->mappable_end = resource_size(&ggtt->gmadr); @@ -971,15 +991,11 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt) * just a coarse sanity check. */ if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { - DRM_ERROR("Unknown GMADR size (%pa)\n", &ggtt->mappable_end); + drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", + &ggtt->mappable_end); return -ENXIO; } - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(40)); - if (!err) - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(40)); - if (err) - DRM_ERROR("Can't set DMA mask/consistent mask (%d)\n", err); pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); size = gen6_get_total_gtt_size(snb_gmch_ctl); @@ -1026,7 +1042,7 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt) ret = intel_gmch_probe(i915->bridge_dev, i915->drm.pdev, NULL); if (!ret) { - DRM_ERROR("failed to set up gmch\n"); + drm_err(&i915->drm, "failed to set up gmch\n"); return -EIO; } @@ -1049,7 +1065,7 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.vma_ops.clear_pages = clear_pages; if (unlikely(ggtt->do_idle_maps)) - dev_notice(i915->drm.dev, + drm_notice(&i915->drm, "Applying Ironlake quirks for intel_iommu\n"); return 0; @@ -1074,26 +1090,29 @@ static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) return ret; if ((ggtt->vm.total - 1) >> 32) { - DRM_ERROR("We never expected a Global GTT with more than 32bits" - " of address space! Found %lldM!\n", - ggtt->vm.total >> 20); + drm_err(&i915->drm, + "We never expected a Global GTT with more than 32bits" + " of address space! Found %lldM!\n", + ggtt->vm.total >> 20); ggtt->vm.total = 1ULL << 32; ggtt->mappable_end = min_t(u64, ggtt->mappable_end, ggtt->vm.total); } if (ggtt->mappable_end > ggtt->vm.total) { - DRM_ERROR("mappable aperture extends past end of GGTT," - " aperture=%pa, total=%llx\n", - &ggtt->mappable_end, ggtt->vm.total); + drm_err(&i915->drm, + "mappable aperture extends past end of GGTT," + " aperture=%pa, total=%llx\n", + &ggtt->mappable_end, ggtt->vm.total); ggtt->mappable_end = ggtt->vm.total; } /* GMADR is the PCI mmio aperture into the global GTT. */ - DRM_DEBUG_DRIVER("GGTT size = %lluM\n", ggtt->vm.total >> 20); - DRM_DEBUG_DRIVER("GMADR size = %lluM\n", (u64)ggtt->mappable_end >> 20); - DRM_DEBUG_DRIVER("DSM size = %lluM\n", - (u64)resource_size(&intel_graphics_stolen_res) >> 20); + drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); + drm_dbg(&i915->drm, "GMADR size = %lluM\n", + (u64)ggtt->mappable_end >> 20); + drm_dbg(&i915->drm, "DSM size = %lluM\n", + (u64)resource_size(&intel_graphics_stolen_res) >> 20); return 0; } @@ -1111,7 +1130,7 @@ int i915_ggtt_probe_hw(struct drm_i915_private *i915) return ret; if (intel_vtd_active()) - dev_info(i915->drm.dev, "VT-d active for gfx access\n"); + drm_info(&i915->drm, "VT-d active for gfx access\n"); return 0; } @@ -1186,6 +1205,8 @@ void i915_ggtt_resume(struct i915_ggtt *ggtt) if (INTEL_GEN(ggtt->vm.i915) >= 8) setup_private_pat(ggtt->vm.gt->uncore); + + intel_ggtt_restore_fences(ggtt); } static struct scatterlist * diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index 049cd3785347..7fb36b12fe7a 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence_reg.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -21,10 +21,9 @@ * IN THE SOFTWARE. */ -#include <drm/i915_drm.h> - #include "i915_drv.h" #include "i915_scatterlist.h" +#include "i915_pvinfo.h" #include "i915_vgpu.h" /** @@ -69,8 +68,7 @@ static struct intel_uncore *fence_to_uncore(struct i915_fence_reg *fence) return fence->ggtt->vm.gt->uncore; } -static void i965_write_fence_reg(struct i915_fence_reg *fence, - struct i915_vma *vma) +static void i965_write_fence_reg(struct i915_fence_reg *fence) { i915_reg_t fence_reg_lo, fence_reg_hi; int fence_pitch_shift; @@ -88,18 +86,16 @@ static void i965_write_fence_reg(struct i915_fence_reg *fence, } val = 0; - if (vma) { - unsigned int stride = i915_gem_object_get_stride(vma->obj); + if (fence->tiling) { + unsigned int stride = fence->stride; - GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); - GEM_BUG_ON(!IS_ALIGNED(vma->node.start, I965_FENCE_PAGE)); - GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I965_FENCE_PAGE)); GEM_BUG_ON(!IS_ALIGNED(stride, 128)); - val = (vma->node.start + vma->fence_size - I965_FENCE_PAGE) << 32; - val |= vma->node.start; + val = fence->start + fence->size - I965_FENCE_PAGE; + val <<= 32; + val |= fence->start; val |= (u64)((stride / 128) - 1) << fence_pitch_shift; - if (i915_gem_object_get_tiling(vma->obj) == I915_TILING_Y) + if (fence->tiling == I915_TILING_Y) val |= BIT(I965_FENCE_TILING_Y_SHIFT); val |= I965_FENCE_REG_VALID; } @@ -126,21 +122,15 @@ static void i965_write_fence_reg(struct i915_fence_reg *fence, } } -static void i915_write_fence_reg(struct i915_fence_reg *fence, - struct i915_vma *vma) +static void i915_write_fence_reg(struct i915_fence_reg *fence) { u32 val; val = 0; - if (vma) { - unsigned int tiling = i915_gem_object_get_tiling(vma->obj); + if (fence->tiling) { + unsigned int stride = fence->stride; + unsigned int tiling = fence->tiling; bool is_y_tiled = tiling == I915_TILING_Y; - unsigned int stride = i915_gem_object_get_stride(vma->obj); - - GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); - GEM_BUG_ON(vma->node.start & ~I915_FENCE_START_MASK); - GEM_BUG_ON(!is_power_of_2(vma->fence_size)); - GEM_BUG_ON(!IS_ALIGNED(vma->node.start, vma->fence_size)); if (is_y_tiled && HAS_128_BYTE_Y_TILING(fence_to_i915(fence))) stride /= 128; @@ -148,10 +138,10 @@ static void i915_write_fence_reg(struct i915_fence_reg *fence, stride /= 512; GEM_BUG_ON(!is_power_of_2(stride)); - val = vma->node.start; + val = fence->start; if (is_y_tiled) val |= BIT(I830_FENCE_TILING_Y_SHIFT); - val |= I915_FENCE_SIZE_BITS(vma->fence_size); + val |= I915_FENCE_SIZE_BITS(fence->size); val |= ilog2(stride) << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; @@ -166,25 +156,18 @@ static void i915_write_fence_reg(struct i915_fence_reg *fence, } } -static void i830_write_fence_reg(struct i915_fence_reg *fence, - struct i915_vma *vma) +static void i830_write_fence_reg(struct i915_fence_reg *fence) { u32 val; val = 0; - if (vma) { - unsigned int stride = i915_gem_object_get_stride(vma->obj); + if (fence->tiling) { + unsigned int stride = fence->stride; - GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); - GEM_BUG_ON(vma->node.start & ~I830_FENCE_START_MASK); - GEM_BUG_ON(!is_power_of_2(vma->fence_size)); - GEM_BUG_ON(!is_power_of_2(stride / 128)); - GEM_BUG_ON(!IS_ALIGNED(vma->node.start, vma->fence_size)); - - val = vma->node.start; - if (i915_gem_object_get_tiling(vma->obj) == I915_TILING_Y) + val = fence->start; + if (fence->tiling == I915_TILING_Y) val |= BIT(I830_FENCE_TILING_Y_SHIFT); - val |= I830_FENCE_SIZE_BITS(vma->fence_size); + val |= I830_FENCE_SIZE_BITS(fence->size); val |= ilog2(stride / 128) << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; } @@ -198,8 +181,7 @@ static void i830_write_fence_reg(struct i915_fence_reg *fence, } } -static void fence_write(struct i915_fence_reg *fence, - struct i915_vma *vma) +static void fence_write(struct i915_fence_reg *fence) { struct drm_i915_private *i915 = fence_to_i915(fence); @@ -210,18 +192,21 @@ static void fence_write(struct i915_fence_reg *fence, */ if (IS_GEN(i915, 2)) - i830_write_fence_reg(fence, vma); + i830_write_fence_reg(fence); else if (IS_GEN(i915, 3)) - i915_write_fence_reg(fence, vma); + i915_write_fence_reg(fence); else - i965_write_fence_reg(fence, vma); + i965_write_fence_reg(fence); /* * Access through the fenced region afterwards is * ordered by the posting reads whilst writing the registers. */ +} - fence->dirty = false; +static bool gpu_uses_fence_registers(struct i915_fence_reg *fence) +{ + return INTEL_GEN(fence_to_i915(fence)) < 4; } static int fence_update(struct i915_fence_reg *fence, @@ -233,27 +218,32 @@ static int fence_update(struct i915_fence_reg *fence, struct i915_vma *old; int ret; + fence->tiling = 0; if (vma) { + GEM_BUG_ON(!i915_gem_object_get_stride(vma->obj) || + !i915_gem_object_get_tiling(vma->obj)); + if (!i915_vma_is_map_and_fenceable(vma)) return -EINVAL; - if (drm_WARN(&uncore->i915->drm, - !i915_gem_object_get_stride(vma->obj) || - !i915_gem_object_get_tiling(vma->obj), - "bogus fence setup with stride: 0x%x, tiling mode: %i\n", - i915_gem_object_get_stride(vma->obj), - i915_gem_object_get_tiling(vma->obj))) - return -EINVAL; + if (gpu_uses_fence_registers(fence)) { + /* implicit 'unfenced' GPU blits */ + ret = i915_vma_sync(vma); + if (ret) + return ret; + } - ret = i915_vma_sync(vma); - if (ret) - return ret; + fence->start = vma->node.start; + fence->size = vma->fence_size; + fence->stride = i915_gem_object_get_stride(vma->obj); + fence->tiling = i915_gem_object_get_tiling(vma->obj); } + WRITE_ONCE(fence->dirty, false); old = xchg(&fence->vma, NULL); if (old) { /* XXX Ideally we would move the waiting to outside the mutex */ - ret = i915_vma_sync(old); + ret = i915_active_wait(&fence->active); if (ret) { fence->vma = old; return ret; @@ -277,7 +267,7 @@ static int fence_update(struct i915_fence_reg *fence, /* * We only need to update the register itself if the device is awake. * If the device is currently powered down, we will defer the write - * to the runtime resume, see i915_gem_restore_fences(). + * to the runtime resume, see intel_ggtt_restore_fences(). * * This only works for removing the fence register, on acquisition * the caller must hold the rpm wakeref. The fence register must @@ -291,7 +281,7 @@ static int fence_update(struct i915_fence_reg *fence, } WRITE_ONCE(fence->vma, vma); - fence_write(fence, vma); + fence_write(fence); if (vma) { vma->fence = fence; @@ -308,23 +298,26 @@ static int fence_update(struct i915_fence_reg *fence, * * This function force-removes any fence from the given object, which is useful * if the kernel wants to do untiled GTT access. - * - * Returns: - * - * 0 on success, negative error code on failure. */ -int i915_vma_revoke_fence(struct i915_vma *vma) +void i915_vma_revoke_fence(struct i915_vma *vma) { struct i915_fence_reg *fence = vma->fence; + intel_wakeref_t wakeref; lockdep_assert_held(&vma->vm->mutex); if (!fence) - return 0; + return; + + GEM_BUG_ON(fence->vma != vma); + GEM_BUG_ON(!i915_active_is_idle(&fence->active)); + GEM_BUG_ON(atomic_read(&fence->pin_count)); - if (atomic_read(&fence->pin_count)) - return -EBUSY; + fence->tiling = 0; + WRITE_ONCE(fence->vma, NULL); + vma->fence = NULL; - return fence_update(fence, NULL); + with_intel_runtime_pm_if_in_use(fence_to_uncore(fence)->rpm, wakeref) + fence_write(fence); } static struct i915_fence_reg *fence_find(struct i915_ggtt *ggtt) @@ -488,34 +481,19 @@ void i915_unreserve_fence(struct i915_fence_reg *fence) } /** - * i915_gem_restore_fences - restore fence state + * intel_ggtt_restore_fences - restore fence state * @ggtt: Global GTT * * Restore the hw fence state to match the software tracking again, to be called * after a gpu reset and on resume. Note that on runtime suspend we only cancel * the fences, to be reacquired by the user later. */ -void i915_gem_restore_fences(struct i915_ggtt *ggtt) +void intel_ggtt_restore_fences(struct i915_ggtt *ggtt) { int i; - rcu_read_lock(); /* keep obj alive as we dereference */ - for (i = 0; i < ggtt->num_fences; i++) { - struct i915_fence_reg *reg = &ggtt->fence_regs[i]; - struct i915_vma *vma = READ_ONCE(reg->vma); - - GEM_BUG_ON(vma && vma->fence != reg); - - /* - * Commit delayed tiling changes if we have an object still - * attached to the fence, otherwise just clear the fence. - */ - if (vma && !i915_gem_object_is_tiled(vma->obj)) - vma = NULL; - - fence_write(reg, vma); - } - rcu_read_unlock(); + for (i = 0; i < ggtt->num_fences; i++) + fence_write(&ggtt->fence_regs[i]); } /** @@ -747,7 +725,7 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) * bit 17 of its physical address and therefore being interpreted differently * by the GPU. */ -static void i915_gem_swizzle_page(struct page *page) +static void swizzle_page(struct page *page) { char temp[64]; char *vaddr; @@ -792,7 +770,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, for_each_sgt_page(page, sgt_iter, pages) { char new_bit_17 = page_to_phys(page) >> 17; if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) { - i915_gem_swizzle_page(page); + swizzle_page(page); set_page_dirty(page); } i++; @@ -837,7 +815,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, } } -void i915_ggtt_init_fences(struct i915_ggtt *ggtt) +void intel_ggtt_init_fences(struct i915_ggtt *ggtt) { struct drm_i915_private *i915 = ggtt->vm.i915; struct intel_uncore *uncore = ggtt->vm.gt->uncore; @@ -865,18 +843,37 @@ void i915_ggtt_init_fences(struct i915_ggtt *ggtt) if (intel_vgpu_active(i915)) num_fences = intel_uncore_read(uncore, vgtif_reg(avail_rs.fence_num)); + ggtt->fence_regs = kcalloc(num_fences, + sizeof(*ggtt->fence_regs), + GFP_KERNEL); + if (!ggtt->fence_regs) + num_fences = 0; /* Initialize fence registers to zero */ for (i = 0; i < num_fences; i++) { struct i915_fence_reg *fence = &ggtt->fence_regs[i]; + i915_active_init(&fence->active, NULL, NULL); fence->ggtt = ggtt; fence->id = i; list_add_tail(&fence->link, &ggtt->fence_list); } ggtt->num_fences = num_fences; - i915_gem_restore_fences(ggtt); + intel_ggtt_restore_fences(ggtt); +} + +void intel_ggtt_fini_fences(struct i915_ggtt *ggtt) +{ + int i; + + for (i = 0; i < ggtt->num_fences; i++) { + struct i915_fence_reg *fence = &ggtt->fence_regs[i]; + + i915_active_fini(&fence->active); + } + + kfree(ggtt->fence_regs); } void intel_gt_init_swizzling(struct intel_gt *gt) diff --git a/drivers/gpu/drm/i915/i915_gem_fence_reg.h b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.h index 7bd521cd7cd7..9eef679e1311 100644 --- a/drivers/gpu/drm/i915/i915_gem_fence_reg.h +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.h @@ -22,12 +22,14 @@ * */ -#ifndef __I915_FENCE_REG_H__ -#define __I915_FENCE_REG_H__ +#ifndef __INTEL_GGTT_FENCING_H__ +#define __INTEL_GGTT_FENCING_H__ #include <linux/list.h> #include <linux/types.h> +#include "i915_active.h" + struct drm_i915_gem_object; struct i915_ggtt; struct i915_vma; @@ -41,6 +43,7 @@ struct i915_fence_reg { struct i915_ggtt *ggtt; struct i915_vma *vma; atomic_t pin_count; + struct i915_active active; int id; /** * Whether the tiling parameters for the currently @@ -51,20 +54,24 @@ struct i915_fence_reg { * command (such as BLT on gen2/3), as a "fence". */ bool dirty; + u32 start; + u32 size; + u32 tiling; + u32 stride; }; -/* i915_gem_fence_reg.c */ struct i915_fence_reg *i915_reserve_fence(struct i915_ggtt *ggtt); void i915_unreserve_fence(struct i915_fence_reg *fence); -void i915_gem_restore_fences(struct i915_ggtt *ggtt); +void intel_ggtt_restore_fences(struct i915_ggtt *ggtt); void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, struct sg_table *pages); void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, struct sg_table *pages); -void i915_ggtt_init_fences(struct i915_ggtt *ggtt); +void intel_ggtt_init_fences(struct i915_ggtt *ggtt); +void intel_ggtt_fini_fences(struct i915_ggtt *ggtt); void intel_gt_init_swizzling(struct intel_gt *gt); diff --git a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h index 51b8718513bc..534e435f20bc 100644 --- a/drivers/gpu/drm/i915/gt/intel_gpu_commands.h +++ b/drivers/gpu/drm/i915/gt/intel_gpu_commands.h @@ -138,7 +138,7 @@ */ #define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*(x)-1) /* Gen11+. addr = base + (ctx_restore ? offset & GENMASK(12,2) : offset) */ -#define MI_LRI_CS_MMIO (1<<19) +#define MI_LRI_LRM_CS_MMIO REG_BIT(19) #define MI_LRI_FORCE_POSTED (1<<12) #define MI_LOAD_REGISTER_IMM_MAX_REGS (126) #define MI_STORE_REGISTER_MEM MI_INSTR(0x24, 1) @@ -156,6 +156,7 @@ #define MI_LOAD_REGISTER_MEM MI_INSTR(0x29, 1) #define MI_LOAD_REGISTER_MEM_GEN8 MI_INSTR(0x29, 2) #define MI_LOAD_REGISTER_REG MI_INSTR(0x2A, 1) +#define MI_LRR_SOURCE_CS_MMIO REG_BIT(18) #define MI_BATCH_BUFFER MI_INSTR(0x30, 1) #define MI_BATCH_NON_SECURE (1) /* for snb/ivb/vlv this also means "batch in ppgtt" when ppgtt is enabled. */ @@ -235,9 +236,8 @@ #define PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH (1<<12) /* gen6+ */ #define PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE (1<<11) /* MBZ on ILK */ #define PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE (1<<10) /* GM45+ only */ -#define PIPE_CONTROL_L3_RO_CACHE_INVALIDATE REG_BIT(10) /* gen12 */ #define PIPE_CONTROL_INDIRECT_STATE_DISABLE (1<<9) -#define PIPE_CONTROL_HDC_PIPELINE_FLUSH REG_BIT(9) /* gen12 */ +#define PIPE_CONTROL0_HDC_PIPELINE_FLUSH REG_BIT(9) /* gen12 */ #define PIPE_CONTROL_NOTIFY (1<<8) #define PIPE_CONTROL_FLUSH_ENABLE (1<<7) /* gen7+ */ #define PIPE_CONTROL_DC_FLUSH_ENABLE (1<<5) @@ -292,10 +292,21 @@ #define MI_STORE_URB_MEM MI_INSTR(0x2D, 0) #define MI_CONDITIONAL_BATCH_BUFFER_END MI_INSTR(0x36, 0) -#define PIPELINE_SELECT ((0x3<<29)|(0x1<<27)|(0x1<<24)|(0x4<<16)) -#define GFX_OP_3DSTATE_VF_STATISTICS ((0x3<<29)|(0x1<<27)|(0x0<<24)|(0xB<<16)) -#define MEDIA_VFE_STATE ((0x3<<29)|(0x2<<27)|(0x0<<24)|(0x0<<16)) +#define STATE_BASE_ADDRESS \ + ((0x3 << 29) | (0x0 << 27) | (0x1 << 24) | (0x1 << 16)) +#define BASE_ADDRESS_MODIFY REG_BIT(0) +#define PIPELINE_SELECT \ + ((0x3 << 29) | (0x1 << 27) | (0x1 << 24) | (0x4 << 16)) +#define PIPELINE_SELECT_MEDIA REG_BIT(0) +#define GFX_OP_3DSTATE_VF_STATISTICS \ + ((0x3 << 29) | (0x1 << 27) | (0x0 << 24) | (0xB << 16)) +#define MEDIA_VFE_STATE \ + ((0x3 << 29) | (0x2 << 27) | (0x0 << 24) | (0x0 << 16)) #define MEDIA_VFE_STATE_MMIO_ACCESS_MASK (0x18) +#define MEDIA_INTERFACE_DESCRIPTOR_LOAD \ + ((0x3 << 29) | (0x2 << 27) | (0x0 << 24) | (0x2 << 16)) +#define MEDIA_OBJECT \ + ((0x3 << 29) | (0x2 << 27) | (0x1 << 24) | (0x0 << 16)) #define GPGPU_OBJECT ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x4<<16)) #define GPGPU_WALKER ((0x3<<29)|(0x2<<27)|(0x1<<24)|(0x5<<16)) #define GFX_OP_3DSTATE_DX9_CONSTANTF_VS \ diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index f1f1b306e0af..f069551e412f 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -7,6 +7,8 @@ #include "i915_drv.h" #include "intel_context.h" #include "intel_gt.h" +#include "intel_gt_buffer_pool.h" +#include "intel_gt_clock_utils.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" #include "intel_mocs.h" @@ -15,6 +17,7 @@ #include "intel_rps.h" #include "intel_uncore.h" #include "intel_pm.h" +#include "shmem_utils.h" void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) { @@ -26,6 +29,7 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) INIT_LIST_HEAD(>->closed_vma); spin_lock_init(>->closed_lock); + intel_gt_init_buffer_pool(gt); intel_gt_init_reset(gt); intel_gt_init_requests(gt); intel_gt_init_timelines(gt); @@ -370,18 +374,6 @@ static struct i915_address_space *kernel_vm(struct intel_gt *gt) return i915_vm_get(>->ggtt->vm); } -static int __intel_context_flush_retire(struct intel_context *ce) -{ - struct intel_timeline *tl; - - tl = intel_context_timeline_lock(ce); - if (IS_ERR(tl)) - return PTR_ERR(tl); - - intel_context_timeline_unlock(tl); - return 0; -} - static int __engines_record_defaults(struct intel_gt *gt) { struct i915_request *requests[I915_NUM_ENGINES] = {}; @@ -447,8 +439,7 @@ err_rq: for (id = 0; id < ARRAY_SIZE(requests); id++) { struct i915_request *rq; - struct i915_vma *state; - void *vaddr; + struct file *state; rq = requests[id]; if (!rq) @@ -460,48 +451,16 @@ err_rq: } GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags)); - state = rq->context->state; - if (!state) + if (!rq->context->state) continue; - /* Serialise with retirement on another CPU */ - GEM_BUG_ON(!i915_request_completed(rq)); - err = __intel_context_flush_retire(rq->context); - if (err) - goto out; - - /* We want to be able to unbind the state from the GGTT */ - GEM_BUG_ON(intel_context_is_pinned(rq->context)); - - /* - * As we will hold a reference to the logical state, it will - * not be torn down with the context, and importantly the - * object will hold onto its vma (making it possible for a - * stray GTT write to corrupt our defaults). Unmap the vma - * from the GTT to prevent such accidents and reclaim the - * space. - */ - err = i915_vma_unbind(state); - if (err) - goto out; - - i915_gem_object_lock(state->obj); - err = i915_gem_object_set_to_cpu_domain(state->obj, false); - i915_gem_object_unlock(state->obj); - if (err) - goto out; - - i915_gem_object_set_cache_coherency(state->obj, I915_CACHE_LLC); - - /* Check we can acquire the image of the context state */ - vaddr = i915_gem_object_pin_map(state->obj, I915_MAP_FORCE_WB); - if (IS_ERR(vaddr)) { - err = PTR_ERR(vaddr); + /* Keep a copy of the state's backing pages; free the obj */ + state = shmem_create_from_object(rq->context->state->obj); + if (IS_ERR(state)) { + err = PTR_ERR(state); goto out; } - - rq->engine->default_state = i915_gem_object_get(state->obj); - i915_gem_object_unpin_map(state->obj); + rq->engine->default_state = state; } out: @@ -576,6 +535,8 @@ int intel_gt_init(struct intel_gt *gt) */ intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); + intel_gt_init_clock_frequency(gt); + err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K); if (err) goto out_fw; @@ -592,7 +553,9 @@ int intel_gt_init(struct intel_gt *gt) if (err) goto err_engines; - intel_uc_init(>->uc); + err = intel_uc_init(>->uc); + if (err) + goto err_engines; err = intel_gt_resume(gt); if (err) @@ -633,8 +596,7 @@ void intel_gt_driver_remove(struct intel_gt *gt) { __intel_gt_disable(gt); - intel_uc_fini_hw(>->uc); - intel_uc_fini(>->uc); + intel_uc_driver_remove(>->uc); intel_engines_release(gt); } @@ -642,6 +604,13 @@ void intel_gt_driver_remove(struct intel_gt *gt) void intel_gt_driver_unregister(struct intel_gt *gt) { intel_rps_driver_unregister(>->rps); + + /* + * Upon unregistering the device to prevent any new users, cancel + * all in-flight requests so that we can quickly unbind the active + * resources. + */ + intel_gt_set_wedged(gt); } void intel_gt_driver_release(struct intel_gt *gt) @@ -654,10 +623,14 @@ void intel_gt_driver_release(struct intel_gt *gt) intel_gt_pm_fini(gt); intel_gt_fini_scratch(gt); + intel_gt_fini_buffer_pool(gt); } void intel_gt_driver_late_release(struct intel_gt *gt) { + /* We need to wait for inflight RCU frees to release their grip */ + rcu_barrier(); + intel_uc_driver_late_release(>->uc); intel_gt_fini_requests(gt); intel_gt_fini_reset(gt); diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pool.c b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c index 397186818305..1495054a4305 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pool.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c @@ -1,6 +1,5 @@ +// SPDX-License-Identifier: MIT /* - * SPDX-License-Identifier: MIT - * * Copyright © 2014-2018 Intel Corporation */ @@ -8,15 +7,15 @@ #include "i915_drv.h" #include "intel_engine_pm.h" -#include "intel_engine_pool.h" +#include "intel_gt_buffer_pool.h" -static struct intel_engine_cs *to_engine(struct intel_engine_pool *pool) +static struct intel_gt *to_gt(struct intel_gt_buffer_pool *pool) { - return container_of(pool, struct intel_engine_cs, pool); + return container_of(pool, struct intel_gt, buffer_pool); } static struct list_head * -bucket_for_size(struct intel_engine_pool *pool, size_t sz) +bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz) { int n; @@ -32,16 +31,50 @@ bucket_for_size(struct intel_engine_pool *pool, size_t sz) return &pool->cache_list[n]; } -static void node_free(struct intel_engine_pool_node *node) +static void node_free(struct intel_gt_buffer_pool_node *node) { i915_gem_object_put(node->obj); i915_active_fini(&node->active); kfree(node); } +static void pool_free_work(struct work_struct *wrk) +{ + struct intel_gt_buffer_pool *pool = + container_of(wrk, typeof(*pool), work.work); + struct intel_gt_buffer_pool_node *node, *next; + unsigned long old = jiffies - HZ; + bool active = false; + LIST_HEAD(stale); + int n; + + /* Free buffers that have not been used in the past second */ + spin_lock_irq(&pool->lock); + for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { + struct list_head *list = &pool->cache_list[n]; + + /* Most recent at head; oldest at tail */ + list_for_each_entry_safe_reverse(node, next, list, link) { + if (time_before(node->age, old)) + break; + + list_move(&node->link, &stale); + } + active |= !list_empty(list); + } + spin_unlock_irq(&pool->lock); + + list_for_each_entry_safe(node, next, &stale, link) + node_free(node); + + if (active) + schedule_delayed_work(&pool->work, + round_jiffies_up_relative(HZ)); +} + static int pool_active(struct i915_active *ref) { - struct intel_engine_pool_node *node = + struct intel_gt_buffer_pool_node *node = container_of(ref, typeof(*node), active); struct dma_resv *resv = node->obj->base.resv; int err; @@ -64,29 +97,31 @@ static int pool_active(struct i915_active *ref) __i915_active_call static void pool_retire(struct i915_active *ref) { - struct intel_engine_pool_node *node = + struct intel_gt_buffer_pool_node *node = container_of(ref, typeof(*node), active); - struct intel_engine_pool *pool = node->pool; + struct intel_gt_buffer_pool *pool = node->pool; struct list_head *list = bucket_for_size(pool, node->obj->base.size); unsigned long flags; - GEM_BUG_ON(!intel_engine_pm_is_awake(to_engine(pool))); - i915_gem_object_unpin_pages(node->obj); /* Return this object to the shrinker pool */ i915_gem_object_make_purgeable(node->obj); spin_lock_irqsave(&pool->lock, flags); + node->age = jiffies; list_add(&node->link, list); spin_unlock_irqrestore(&pool->lock, flags); + + schedule_delayed_work(&pool->work, + round_jiffies_up_relative(HZ)); } -static struct intel_engine_pool_node * -node_create(struct intel_engine_pool *pool, size_t sz) +static struct intel_gt_buffer_pool_node * +node_create(struct intel_gt_buffer_pool *pool, size_t sz) { - struct intel_engine_cs *engine = to_engine(pool); - struct intel_engine_pool_node *node; + struct intel_gt *gt = to_gt(pool); + struct intel_gt_buffer_pool_node *node; struct drm_i915_gem_object *obj; node = kmalloc(sizeof(*node), @@ -97,7 +132,7 @@ node_create(struct intel_engine_pool *pool, size_t sz) node->pool = pool; i915_active_init(&node->active, pool_active, pool_retire); - obj = i915_gem_object_create_internal(engine->i915, sz); + obj = i915_gem_object_create_internal(gt->i915, sz); if (IS_ERR(obj)) { i915_active_fini(&node->active); kfree(node); @@ -110,26 +145,15 @@ node_create(struct intel_engine_pool *pool, size_t sz) return node; } -static struct intel_engine_pool *lookup_pool(struct intel_engine_cs *engine) +struct intel_gt_buffer_pool_node * +intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size) { - if (intel_engine_is_virtual(engine)) - engine = intel_virtual_engine_get_sibling(engine, 0); - - GEM_BUG_ON(!engine); - return &engine->pool; -} - -struct intel_engine_pool_node * -intel_engine_get_pool(struct intel_engine_cs *engine, size_t size) -{ - struct intel_engine_pool *pool = lookup_pool(engine); - struct intel_engine_pool_node *node; + struct intel_gt_buffer_pool *pool = >->buffer_pool; + struct intel_gt_buffer_pool_node *node; struct list_head *list; unsigned long flags; int ret; - GEM_BUG_ON(!intel_engine_pm_is_awake(to_engine(pool))); - size = PAGE_ALIGN(size); list = bucket_for_size(pool, size); @@ -157,34 +181,48 @@ intel_engine_get_pool(struct intel_engine_cs *engine, size_t size) return node; } -void intel_engine_pool_init(struct intel_engine_pool *pool) +void intel_gt_init_buffer_pool(struct intel_gt *gt) { + struct intel_gt_buffer_pool *pool = >->buffer_pool; int n; spin_lock_init(&pool->lock); for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) INIT_LIST_HEAD(&pool->cache_list[n]); + INIT_DELAYED_WORK(&pool->work, pool_free_work); } -void intel_engine_pool_park(struct intel_engine_pool *pool) +static void pool_free_imm(struct intel_gt_buffer_pool *pool) { int n; + spin_lock_irq(&pool->lock); for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { + struct intel_gt_buffer_pool_node *node, *next; struct list_head *list = &pool->cache_list[n]; - struct intel_engine_pool_node *node, *nn; - list_for_each_entry_safe(node, nn, list, link) + list_for_each_entry_safe(node, next, list, link) node_free(node); - INIT_LIST_HEAD(list); } + spin_unlock_irq(&pool->lock); +} + +void intel_gt_flush_buffer_pool(struct intel_gt *gt) +{ + struct intel_gt_buffer_pool *pool = >->buffer_pool; + + if (cancel_delayed_work_sync(&pool->work)) + pool_free_imm(pool); } -void intel_engine_pool_fini(struct intel_engine_pool *pool) +void intel_gt_fini_buffer_pool(struct intel_gt *gt) { + struct intel_gt_buffer_pool *pool = >->buffer_pool; int n; + intel_gt_flush_buffer_pool(gt); + for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) GEM_BUG_ON(!list_empty(&pool->cache_list[n])); } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.h b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.h new file mode 100644 index 000000000000..42cbac003e8a --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2018 Intel Corporation + */ + +#ifndef INTEL_GT_BUFFER_POOL_H +#define INTEL_GT_BUFFER_POOL_H + +#include <linux/types.h> + +#include "i915_active.h" +#include "intel_gt_buffer_pool_types.h" + +struct intel_gt; +struct i915_request; + +struct intel_gt_buffer_pool_node * +intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size); + +static inline int +intel_gt_buffer_pool_mark_active(struct intel_gt_buffer_pool_node *node, + struct i915_request *rq) +{ + return i915_active_add_request(&node->active, rq); +} + +static inline void +intel_gt_buffer_pool_put(struct intel_gt_buffer_pool_node *node) +{ + i915_active_release(&node->active); +} + +void intel_gt_init_buffer_pool(struct intel_gt *gt); +void intel_gt_flush_buffer_pool(struct intel_gt *gt); +void intel_gt_fini_buffer_pool(struct intel_gt *gt); + +#endif /* INTEL_GT_BUFFER_POOL_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_engine_pool_types.h b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h index e31ee361b76f..e28bdda771ed 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_pool_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h @@ -4,26 +4,29 @@ * Copyright © 2014-2018 Intel Corporation */ -#ifndef INTEL_ENGINE_POOL_TYPES_H -#define INTEL_ENGINE_POOL_TYPES_H +#ifndef INTEL_GT_BUFFER_POOL_TYPES_H +#define INTEL_GT_BUFFER_POOL_TYPES_H #include <linux/list.h> #include <linux/spinlock.h> +#include <linux/workqueue.h> #include "i915_active_types.h" struct drm_i915_gem_object; -struct intel_engine_pool { +struct intel_gt_buffer_pool { spinlock_t lock; struct list_head cache_list[4]; + struct delayed_work work; }; -struct intel_engine_pool_node { +struct intel_gt_buffer_pool_node { struct i915_active active; struct drm_i915_gem_object *obj; struct list_head link; - struct intel_engine_pool *pool; + struct intel_gt_buffer_pool *pool; + unsigned long age; }; -#endif /* INTEL_ENGINE_POOL_TYPES_H */ +#endif /* INTEL_GT_BUFFER_POOL_TYPES_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c new file mode 100644 index 000000000000..999079686846 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_gt.h" +#include "intel_gt_clock_utils.h" + +#define MHZ_12 12000000 /* 12MHz (24MHz/2), 83.333ns */ +#define MHZ_12_5 12500000 /* 12.5MHz (25MHz/2), 80ns */ +#define MHZ_19_2 19200000 /* 19.2MHz, 52.083ns */ + +static u32 read_clock_frequency(const struct intel_gt *gt) +{ + if (INTEL_GEN(gt->i915) >= 11) { + u32 config; + + config = intel_uncore_read(gt->uncore, RPM_CONFIG0); + config &= GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK; + config >>= GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT; + + switch (config) { + case 0: return MHZ_12; + case 1: + case 2: return MHZ_19_2; + default: + case 3: return MHZ_12_5; + } + } else if (INTEL_GEN(gt->i915) >= 9) { + if (IS_GEN9_LP(gt->i915)) + return MHZ_19_2; + else + return MHZ_12; + } else { + return MHZ_12_5; + } +} + +void intel_gt_init_clock_frequency(struct intel_gt *gt) +{ + /* + * Note that on gen11+, the clock frequency may be reconfigured. + * We do not, and we assume nobody else does. + */ + gt->clock_frequency = read_clock_frequency(gt); + GT_TRACE(gt, + "Using clock frequency: %dkHz\n", + gt->clock_frequency / 1000); +} + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +void intel_gt_check_clock_frequency(const struct intel_gt *gt) +{ + if (gt->clock_frequency != read_clock_frequency(gt)) { + dev_err(gt->i915->drm.dev, + "GT clock frequency changed, was %uHz, now %uHz!\n", + gt->clock_frequency, + read_clock_frequency(gt)); + } +} +#endif + +static u64 div_u64_roundup(u64 nom, u32 den) +{ + return div_u64(nom + den - 1, den); +} + +u32 intel_gt_clock_interval_to_ns(const struct intel_gt *gt, u32 count) +{ + return div_u64_roundup(mul_u32_u32(count, 1000 * 1000 * 1000), + gt->clock_frequency); +} + +u32 intel_gt_pm_interval_to_ns(const struct intel_gt *gt, u32 count) +{ + return intel_gt_clock_interval_to_ns(gt, 16 * count); +} + +u32 intel_gt_ns_to_clock_interval(const struct intel_gt *gt, u32 ns) +{ + return div_u64_roundup(mul_u32_u32(gt->clock_frequency, ns), + 1000 * 1000 * 1000); +} + +u32 intel_gt_ns_to_pm_interval(const struct intel_gt *gt, u32 ns) +{ + u32 val; + + /* + * Make these a multiple of magic 25 to avoid SNB (eg. Dell XPS + * 8300) freezing up around GPU hangs. Looks as if even + * scheduling/timer interrupts start misbehaving if the RPS + * EI/thresholds are "bad", leading to a very sluggish or even + * frozen machine. + */ + val = DIV_ROUND_UP(intel_gt_ns_to_clock_interval(gt, ns), 16); + if (IS_GEN(gt->i915, 6)) + val = roundup(val, 25); + + return val; +} diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.h b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.h new file mode 100644 index 000000000000..f793c89f2cbd --- /dev/null +++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __INTEL_GT_CLOCK_UTILS_H__ +#define __INTEL_GT_CLOCK_UTILS_H__ + +#include <linux/types.h> + +struct intel_gt; + +void intel_gt_init_clock_frequency(struct intel_gt *gt); + +#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) +void intel_gt_check_clock_frequency(const struct intel_gt *gt); +#else +static inline void intel_gt_check_clock_frequency(const struct intel_gt *gt) {} +#endif + +u32 intel_gt_clock_interval_to_ns(const struct intel_gt *gt, u32 count); +u32 intel_gt_pm_interval_to_ns(const struct intel_gt *gt, u32 count); + +u32 intel_gt_ns_to_clock_interval(const struct intel_gt *gt, u32 ns); +u32 intel_gt_ns_to_pm_interval(const struct intel_gt *gt, u32 ns); + +#endif /* __INTEL_GT_CLOCK_UTILS_H__ */ diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index f0e7fd95165a..0cc7dd54f4f9 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -39,6 +39,15 @@ cs_irq_handler(struct intel_engine_cs *engine, u32 iir) } } + if (iir & GT_WAIT_SEMAPHORE_INTERRUPT) { + WRITE_ONCE(engine->execlists.yield, + ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI)); + ENGINE_TRACE(engine, "semaphore yield: %08x\n", + engine->execlists.yield); + if (del_timer(&engine->execlists.timer)) + tasklet = true; + } + if (iir & GT_CONTEXT_SWITCH_INTERRUPT) tasklet = true; @@ -228,7 +237,8 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) const u32 irqs = GT_CS_MASTER_ERROR_INTERRUPT | GT_RENDER_USER_INTERRUPT | - GT_CONTEXT_SWITCH_INTERRUPT; + GT_CONTEXT_SWITCH_INTERRUPT | + GT_WAIT_SEMAPHORE_INTERRUPT; struct intel_uncore *uncore = gt->uncore; const u32 dmask = irqs << 16 | irqs; const u32 smask = irqs << 16; @@ -366,7 +376,8 @@ void gen8_gt_irq_postinstall(struct intel_gt *gt) const u32 irqs = GT_CS_MASTER_ERROR_INTERRUPT | GT_RENDER_USER_INTERRUPT | - GT_CONTEXT_SWITCH_INTERRUPT; + GT_CONTEXT_SWITCH_INTERRUPT | + GT_WAIT_SEMAPHORE_INTERRUPT; const u32 gt_interrupts[] = { irqs << GEN8_RCS_IRQ_SHIFT | irqs << GEN8_BCS_IRQ_SHIFT, irqs << GEN8_VCS0_IRQ_SHIFT | irqs << GEN8_VCS1_IRQ_SHIFT, diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_pm.c index 8b653c0f5e5f..6bdb434a442d 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.c @@ -12,6 +12,7 @@ #include "intel_context.h" #include "intel_engine_pm.h" #include "intel_gt.h" +#include "intel_gt_clock_utils.h" #include "intel_gt_pm.h" #include "intel_gt_requests.h" #include "intel_llc.h" @@ -138,6 +139,8 @@ static void gt_sanitize(struct intel_gt *gt, bool force) wakeref = intel_runtime_pm_get(gt->uncore->rpm); intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); + intel_gt_check_clock_frequency(gt); + /* * As we have just resumed the machine and woken the device up from * deep PCI sleep (presumably D3_cold), assume the HW has been reset @@ -155,6 +158,10 @@ static void gt_sanitize(struct intel_gt *gt, bool force) intel_uc_reset_prepare(>->uc); + for_each_engine(engine, gt, id) + if (engine->sanitize) + engine->sanitize(engine); + if (reset_engines(gt) || force) { for_each_engine(engine, gt, id) __intel_engine_reset(engine, false); @@ -164,6 +171,8 @@ static void gt_sanitize(struct intel_gt *gt, bool force) if (engine->reset.finish) engine->reset.finish(engine); + intel_rps_sanitize(>->rps); + intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); intel_runtime_pm_put(gt->uncore->rpm, wakeref); } @@ -191,11 +200,12 @@ int intel_gt_resume(struct intel_gt *gt) * Only the kernel contexts should remain pinned over suspend, * allowing us to fixup the user contexts on their first pin. */ + gt_sanitize(gt, true); + intel_gt_pm_get(gt); intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); intel_rc6_sanitize(>->rc6); - gt_sanitize(gt, true); if (intel_gt_is_wedged(gt)) { err = -EIO; goto out_fw; @@ -204,7 +214,7 @@ int intel_gt_resume(struct intel_gt *gt) /* Only when the HW is re-initialised, can we replay the requests */ err = intel_gt_init_hw(gt); if (err) { - dev_err(gt->i915->drm.dev, + drm_err(>->i915->drm, "Failed to initialize GPU, declaring it wedged!\n"); goto err_wedged; } @@ -220,7 +230,7 @@ int intel_gt_resume(struct intel_gt *gt) intel_engine_pm_put(engine); if (err) { - dev_err(gt->i915->drm.dev, + drm_err(>->i915->drm, "Failed to restart %s (%d)\n", engine->name, err); goto err_wedged; @@ -324,6 +334,7 @@ int intel_gt_runtime_resume(struct intel_gt *gt) { GT_TRACE(gt, "\n"); intel_gt_init_swizzling(gt); + intel_ggtt_restore_fences(gt->ggtt); return intel_uc_runtime_resume(>->uc); } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_requests.c b/drivers/gpu/drm/i915/gt/intel_gt_requests.c index 8a5054f21bf8..16ff47c83bd5 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_requests.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_requests.c @@ -26,6 +26,11 @@ static bool retire_requests(struct intel_timeline *tl) return !i915_active_fence_isset(&tl->last_request); } +static bool engine_active(const struct intel_engine_cs *engine) +{ + return !list_empty(&engine->kernel_context->timeline->requests); +} + static bool flush_submission(struct intel_gt *gt) { struct intel_engine_cs *engine; @@ -37,8 +42,13 @@ static bool flush_submission(struct intel_gt *gt) for_each_engine(engine, gt, id) { intel_engine_flush_submission(engine); - active |= flush_work(&engine->retire_work); - active |= flush_work(&engine->wakeref.work); + + /* Flush the background retirement and idle barriers */ + flush_work(&engine->retire_work); + flush_delayed_work(&engine->wakeref.work); + + /* Is the idle barrier still outstanding? */ + active |= engine_active(engine); } return active; @@ -147,25 +157,32 @@ long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout) fence = i915_active_fence_get(&tl->last_request); if (fence) { + mutex_unlock(&tl->mutex); + timeout = dma_fence_wait_timeout(fence, interruptible, timeout); dma_fence_put(fence); + + /* Retirement is best effort */ + if (!mutex_trylock(&tl->mutex)) { + active_count++; + goto out_active; + } } } - if (!retire_requests(tl) || flush_submission(gt)) + if (!retire_requests(tl)) active_count++; + mutex_unlock(&tl->mutex); - spin_lock(&timelines->lock); +out_active: spin_lock(&timelines->lock); - /* Resume iteration after dropping lock */ + /* Resume list iteration after reacquiring spinlock */ list_safe_reset_next(tl, tn, link); if (atomic_dec_and_test(&tl->active_count)) list_del(&tl->link); - mutex_unlock(&tl->mutex); - /* Defer the final release to after the spinlock */ if (refcount_dec_and_test(&tl->kref.refcount)) { GEM_BUG_ON(atomic_read(&tl->active_count)); @@ -177,6 +194,9 @@ long intel_gt_retire_requests_timeout(struct intel_gt *gt, long timeout) list_for_each_entry_safe(tl, tn, &free, link) __intel_timeline_free(&tl->kref); + if (flush_submission(gt)) /* Wait, there's more! */ + active_count++; + return active_count ? timeout : 0; } diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h index 96890dd12b5f..0cc1d6b185dc 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_types.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h @@ -17,6 +17,7 @@ #include "i915_vma.h" #include "intel_engine_types.h" +#include "intel_gt_buffer_pool_types.h" #include "intel_llc_types.h" #include "intel_reset_types.h" #include "intel_rc6_types.h" @@ -61,6 +62,7 @@ struct intel_gt { struct list_head closed_vma; spinlock_t closed_lock; /* guards the list of closed_vma */ + ktime_t last_init_time; struct intel_reset reset; /** @@ -72,14 +74,12 @@ struct intel_gt { */ intel_wakeref_t awake; + u32 clock_frequency; + struct intel_llc llc; struct intel_rc6 rc6; struct intel_rps rps; - ktime_t last_init_time; - - struct i915_vma *scratch; - spinlock_t irq_lock; u32 gt_imr; u32 pm_ier; @@ -97,6 +97,18 @@ struct intel_gt { * Reserved for exclusive use by the kernel. */ struct i915_address_space *vm; + + /* + * A pool of objects to use as shadow copies of client batch buffers + * when the command parser is enabled. Prevents the client from + * modifying the batch contents after software parsing. + * + * Buffers older than 1s are periodically reaped from the pool, + * or may be reclaimed by the shrinker before then. + */ + struct intel_gt_buffer_pool buffer_pool; + + struct i915_vma *scratch; }; enum intel_gt_scratch_field { diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c index bb9a6e638175..2a72cce63fd9 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.c +++ b/drivers/gpu/drm/i915/gt/intel_gtt.c @@ -171,7 +171,9 @@ void __i915_vm_close(struct i915_address_space *vm) { struct i915_vma *vma, *vn; - mutex_lock(&vm->mutex); + if (!atomic_dec_and_mutex_lock(&vm->open, &vm->mutex)) + return; + list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { struct drm_i915_gem_object *obj = vma->obj; @@ -186,6 +188,7 @@ void __i915_vm_close(struct i915_address_space *vm) i915_gem_object_put(obj); } GEM_BUG_ON(!list_empty(&vm->bound_list)); + mutex_unlock(&vm->mutex); } @@ -484,30 +487,6 @@ void gtt_write_workarounds(struct intel_gt *gt) } } -u64 gen8_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 flags) -{ - gen8_pte_t pte = addr | _PAGE_PRESENT | _PAGE_RW; - - if (unlikely(flags & PTE_READ_ONLY)) - pte &= ~_PAGE_RW; - - switch (level) { - case I915_CACHE_NONE: - pte |= PPAT_UNCACHED; - break; - case I915_CACHE_WT: - pte |= PPAT_DISPLAY_ELLC; - break; - default: - pte |= PPAT_CACHED; - break; - } - - return pte; -} - static void tgl_setup_private_ppat(struct intel_uncore *uncore) { /* TGL doesn't support LLC or AGE settings */ diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h index 23004445806a..d93ebdf3fa0e 100644 --- a/drivers/gpu/drm/i915/gt/intel_gtt.h +++ b/drivers/gpu/drm/i915/gt/intel_gtt.h @@ -26,7 +26,6 @@ #include <drm/drm_mm.h> #include "gt/intel_reset.h" -#include "i915_gem_fence_reg.h" #include "i915_selftest.h" #include "i915_vma_types.h" @@ -135,6 +134,8 @@ typedef u64 gen8_pte_t; #define GEN8_PDE_IPS_64K BIT(11) #define GEN8_PDE_PS_2M BIT(7) +struct i915_fence_reg; + #define for_each_sgt_daddr(__dp, __iter, __sgt) \ __for_each_sgt_daddr(__dp, __iter, __sgt, I915_GTT_PAGE_SIZE) @@ -333,7 +334,7 @@ struct i915_ggtt { u32 pin_bias; unsigned int num_fences; - struct i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; + struct i915_fence_reg *fence_regs; struct list_head fence_list; /** @@ -429,8 +430,7 @@ static inline void i915_vm_close(struct i915_address_space *vm) { GEM_BUG_ON(!atomic_read(&vm->open)); - if (atomic_dec_and_test(&vm->open)) - __i915_vm_close(vm); + __i915_vm_close(vm); i915_vm_put(vm); } @@ -515,10 +515,6 @@ struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt); void i915_ggtt_suspend(struct i915_ggtt *gtt); void i915_ggtt_resume(struct i915_ggtt *ggtt); -u64 gen8_pte_encode(dma_addr_t addr, - enum i915_cache_level level, - u32 flags); - int setup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p); void cleanup_page_dma(struct i915_address_space *vm, struct i915_page_dma *p); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index ba31cbe8c68e..87e6c5bdd2dc 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -147,6 +147,7 @@ #include "intel_reset.h" #include "intel_ring.h" #include "intel_workarounds.h" +#include "shmem_utils.h" #define RING_EXECLIST_QFULL (1 << 0x2) #define RING_EXECLIST1_VALID (1 << 0x3) @@ -216,7 +217,7 @@ struct virtual_engine { /* And finally, which physical engines this virtual engine maps onto. */ unsigned int num_siblings; - struct intel_engine_cs *siblings[0]; + struct intel_engine_cs *siblings[]; }; static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine) @@ -238,6 +239,123 @@ __execlists_update_reg_state(const struct intel_context *ce, const struct intel_engine_cs *engine, u32 head); +static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) +{ + if (INTEL_GEN(engine->i915) >= 12) + return 0x60; + else if (INTEL_GEN(engine->i915) >= 9) + return 0x54; + else if (engine->class == RENDER_CLASS) + return 0x58; + else + return -1; +} + +static int lrc_ring_gpr0(const struct intel_engine_cs *engine) +{ + if (INTEL_GEN(engine->i915) >= 12) + return 0x74; + else if (INTEL_GEN(engine->i915) >= 9) + return 0x68; + else if (engine->class == RENDER_CLASS) + return 0xd8; + else + return -1; +} + +static int lrc_ring_wa_bb_per_ctx(const struct intel_engine_cs *engine) +{ + if (INTEL_GEN(engine->i915) >= 12) + return 0x12; + else if (INTEL_GEN(engine->i915) >= 9 || engine->class == RENDER_CLASS) + return 0x18; + else + return -1; +} + +static int lrc_ring_indirect_ptr(const struct intel_engine_cs *engine) +{ + int x; + + x = lrc_ring_wa_bb_per_ctx(engine); + if (x < 0) + return x; + + return x + 2; +} + +static int lrc_ring_indirect_offset(const struct intel_engine_cs *engine) +{ + int x; + + x = lrc_ring_indirect_ptr(engine); + if (x < 0) + return x; + + return x + 2; +} + +static int lrc_ring_cmd_buf_cctl(const struct intel_engine_cs *engine) +{ + if (engine->class != RENDER_CLASS) + return -1; + + if (INTEL_GEN(engine->i915) >= 12) + return 0xb6; + else if (INTEL_GEN(engine->i915) >= 11) + return 0xaa; + else + return -1; +} + +static u32 +lrc_ring_indirect_offset_default(const struct intel_engine_cs *engine) +{ + switch (INTEL_GEN(engine->i915)) { + default: + MISSING_CASE(INTEL_GEN(engine->i915)); + fallthrough; + case 12: + return GEN12_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; + case 11: + return GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; + case 10: + return GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; + case 9: + return GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; + case 8: + return GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; + } +} + +static void +lrc_ring_setup_indirect_ctx(u32 *regs, + const struct intel_engine_cs *engine, + u32 ctx_bb_ggtt_addr, + u32 size) +{ + GEM_BUG_ON(!size); + GEM_BUG_ON(!IS_ALIGNED(size, CACHELINE_BYTES)); + GEM_BUG_ON(lrc_ring_indirect_ptr(engine) == -1); + regs[lrc_ring_indirect_ptr(engine) + 1] = + ctx_bb_ggtt_addr | (size / CACHELINE_BYTES); + + GEM_BUG_ON(lrc_ring_indirect_offset(engine) == -1); + regs[lrc_ring_indirect_offset(engine) + 1] = + lrc_ring_indirect_offset_default(engine) << 6; +} + +static u32 intel_context_get_runtime(const struct intel_context *ce) +{ + /* + * We can use either ppHWSP[16] which is recorded before the context + * switch (and so excludes the cost of context switches) or use the + * value from the context image itself, which is saved/restored earlier + * and so includes the cost of the save. + */ + return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); +} + static void mark_eio(struct i915_request *rq) { if (i915_request_completed(rq)) @@ -245,7 +363,7 @@ static void mark_eio(struct i915_request *rq) GEM_BUG_ON(i915_request_signaled(rq)); - dma_fence_set_error(&rq->fence, -EIO); + i915_request_set_error_once(rq, -EIO); i915_request_mark_complete(rq); } @@ -293,7 +411,7 @@ static inline struct i915_priolist *to_priolist(struct rb_node *rb) static inline int rq_prio(const struct i915_request *rq) { - return rq->sched.attr.priority; + return READ_ONCE(rq->sched.attr.priority); } static int effective_prio(const struct i915_request *rq) @@ -311,18 +429,7 @@ static int effective_prio(const struct i915_request *rq) if (i915_request_has_nopreempt(rq)) prio = I915_PRIORITY_UNPREEMPTABLE; - /* - * On unwinding the active request, we give it a priority bump - * if it has completed waiting on any semaphore. If we know that - * the request has already started, we can prevent an unwanted - * preempt-to-idle cycle by taking that into account now. - */ - if (__i915_request_has_started(rq)) - prio |= I915_PRIORITY_NOSEMAPHORE; - - /* Restrict mere WAIT boosts from triggering preemption */ - BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); /* only internal */ - return prio | __NO_PREEMPTION; + return prio; } static int queue_prio(const struct intel_engine_execlists *execlists) @@ -456,10 +563,10 @@ assert_priority_queue(const struct i915_request *prev, * engine info, SW context ID and SW counter need to form a unique number * (Context ID) per lrc. */ -static u64 +static u32 lrc_descriptor(struct intel_context *ce, struct intel_engine_cs *engine) { - u64 desc; + u32 desc; desc = INTEL_LEGACY_32B_CONTEXT; if (i915_vm_is_4lvl(ce->vm)) @@ -470,21 +577,7 @@ lrc_descriptor(struct intel_context *ce, struct intel_engine_cs *engine) if (IS_GEN(engine->i915, 8)) desc |= GEN8_CTX_L3LLC_COHERENT; - desc |= i915_ggtt_offset(ce->state); /* bits 12-31 */ - /* - * The following 32bits are copied into the OA reports (dword 2). - * Consider updating oa_get_render_ctx_id in i915_perf.c when changing - * anything below. - */ - if (INTEL_GEN(engine->i915) >= 11) { - desc |= (u64)engine->instance << GEN11_ENGINE_INSTANCE_SHIFT; - /* bits 48-53 */ - - desc |= (u64)engine->class << GEN11_ENGINE_CLASS_SHIFT; - /* bits 61-63 */ - } - - return desc; + return i915_ggtt_offset(ce->state) | desc; } static inline unsigned int dword_in_page(void *addr) @@ -503,7 +596,7 @@ static void set_offsets(u32 *regs, #define REG16(x) \ (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ (((x) >> 2) & 0x7f) -#define END(x) 0, (x) +#define END(total_state_size) 0, (total_state_size) { const u32 base = engine->mmio_base; @@ -526,7 +619,7 @@ static void set_offsets(u32 *regs, if (flags & POSTED) *regs |= MI_LRI_FORCE_POSTED; if (INTEL_GEN(engine->i915) >= 11) - *regs |= MI_LRI_CS_MMIO; + *regs |= MI_LRI_LRM_CS_MMIO; regs++; GEM_BUG_ON(!count); @@ -911,8 +1004,63 @@ static const u8 gen12_rcs_offsets[] = { NOP(6), LRI(1, 0), REG(0x0c8), + NOP(3 + 9 + 1), + + LRI(51, POSTED), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG(0x028), + REG(0x09c), + REG(0x0c0), + REG(0x178), + REG(0x17c), + REG16(0x358), + REG(0x170), + REG(0x150), + REG(0x154), + REG(0x158), + REG16(0x41c), + REG16(0x600), + REG16(0x604), + REG16(0x608), + REG16(0x60c), + REG16(0x610), + REG16(0x614), + REG16(0x618), + REG16(0x61c), + REG16(0x620), + REG16(0x624), + REG16(0x628), + REG16(0x62c), + REG16(0x630), + REG16(0x634), + REG16(0x638), + REG16(0x63c), + REG16(0x640), + REG16(0x644), + REG16(0x648), + REG16(0x64c), + REG16(0x650), + REG16(0x654), + REG16(0x658), + REG16(0x65c), + REG16(0x660), + REG16(0x664), + REG16(0x668), + REG16(0x66c), + REG16(0x670), + REG16(0x674), + REG16(0x678), + REG16(0x67c), + REG(0x068), + REG(0x084), + NOP(1), - END(80) + END(192) }; #undef END @@ -1004,7 +1152,7 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) i915_request_cancel_breadcrumb(rq); spin_unlock(&rq->lock); } - rq->engine = owner; + WRITE_ONCE(rq->engine, owner); owner->submit_request(rq); active = NULL; } @@ -1040,17 +1188,14 @@ static void intel_engine_context_in(struct intel_engine_cs *engine) { unsigned long flags; - if (READ_ONCE(engine->stats.enabled) == 0) + if (atomic_add_unless(&engine->stats.active, 1, 0)) return; write_seqlock_irqsave(&engine->stats.lock, flags); - - if (engine->stats.enabled > 0) { - if (engine->stats.active++ == 0) - engine->stats.start = ktime_get(); - GEM_BUG_ON(engine->stats.active == 0); + if (!atomic_add_unless(&engine->stats.active, 1, 0)) { + engine->stats.start = ktime_get(); + atomic_inc(&engine->stats.active); } - write_sequnlock_irqrestore(&engine->stats.lock, flags); } @@ -1058,51 +1203,20 @@ static void intel_engine_context_out(struct intel_engine_cs *engine) { unsigned long flags; - if (READ_ONCE(engine->stats.enabled) == 0) + GEM_BUG_ON(!atomic_read(&engine->stats.active)); + + if (atomic_add_unless(&engine->stats.active, -1, 1)) return; write_seqlock_irqsave(&engine->stats.lock, flags); - - if (engine->stats.enabled > 0) { - ktime_t last; - - if (engine->stats.active && --engine->stats.active == 0) { - /* - * Decrement the active context count and in case GPU - * is now idle add up to the running total. - */ - last = ktime_sub(ktime_get(), engine->stats.start); - - engine->stats.total = ktime_add(engine->stats.total, - last); - } else if (engine->stats.active == 0) { - /* - * After turning on engine stats, context out might be - * the first event in which case we account from the - * time stats gathering was turned on. - */ - last = ktime_sub(ktime_get(), engine->stats.enabled_at); - - engine->stats.total = ktime_add(engine->stats.total, - last); - } + if (atomic_dec_and_test(&engine->stats.active)) { + engine->stats.total = + ktime_add(engine->stats.total, + ktime_sub(ktime_get(), engine->stats.start)); } - write_sequnlock_irqrestore(&engine->stats.lock, flags); } -static int lrc_ring_mi_mode(const struct intel_engine_cs *engine) -{ - if (INTEL_GEN(engine->i915) >= 12) - return 0x60; - else if (INTEL_GEN(engine->i915) >= 9) - return 0x54; - else if (engine->class == RENDER_CLASS) - return 0x58; - else - return -1; -} - static void execlists_check_context(const struct intel_context *ce, const struct intel_engine_cs *engine) @@ -1146,14 +1260,12 @@ execlists_check_context(const struct intel_context *ce, static void restore_default_state(struct intel_context *ce, struct intel_engine_cs *engine) { - u32 *regs = ce->lrc_reg_state; + u32 *regs; - if (engine->pinned_default_state) - memcpy(regs, /* skip restoring the vanilla PPHWSP */ - engine->pinned_default_state + LRC_STATE_PN * PAGE_SIZE, - engine->context_size - PAGE_SIZE); + regs = memset(ce->lrc_reg_state, 0, engine->context_size - PAGE_SIZE); + execlists_init_reg_state(regs, ce, engine, ce->ring, true); - execlists_init_reg_state(regs, ce, engine, ce->ring, false); + ce->runtime.last = intel_context_get_runtime(ce); } static void reset_active(struct i915_request *rq, @@ -1192,18 +1304,7 @@ static void reset_active(struct i915_request *rq, __execlists_update_reg_state(ce, engine, head); /* We've switched away, so this should be a no-op, but intent matters */ - ce->lrc_desc |= CTX_DESC_FORCE_RESTORE; -} - -static u32 intel_context_get_runtime(const struct intel_context *ce) -{ - /* - * We can use either ppHWSP[16] which is recorded before the context - * switch (and so excludes the cost of context switches) or use the - * value from the context image itself, which is saved/restored earlier - * and so includes the cost of the save. - */ - return READ_ONCE(ce->lrc_reg_state[CTX_TIMESTAMP]); + ce->lrc.desc |= CTX_DESC_FORCE_RESTORE; } static void st_update_runtime_underflow(struct intel_context *ce, s32 dt) @@ -1251,18 +1352,23 @@ __execlists_schedule_in(struct i915_request *rq) if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) execlists_check_context(ce, engine); - ce->lrc_desc &= ~GENMASK_ULL(47, 37); if (ce->tag) { /* Use a fixed tag for OA and friends */ - ce->lrc_desc |= (u64)ce->tag << 32; + GEM_BUG_ON(ce->tag <= BITS_PER_LONG); + ce->lrc.ccid = ce->tag; } else { /* We don't need a strict matching tag, just different values */ - ce->lrc_desc |= - (u64)(++engine->context_tag % NUM_CONTEXT_TAG) << - GEN11_SW_CTX_ID_SHIFT; - BUILD_BUG_ON(NUM_CONTEXT_TAG > GEN12_MAX_CONTEXT_HW_ID); + unsigned int tag = ffs(READ_ONCE(engine->context_tag)); + + GEM_BUG_ON(tag == 0 || tag >= BITS_PER_LONG); + clear_bit(tag - 1, &engine->context_tag); + ce->lrc.ccid = tag << (GEN11_SW_CTX_ID_SHIFT - 32); + + BUILD_BUG_ON(BITS_PER_LONG > GEN12_MAX_CONTEXT_HW_ID); } + ce->lrc.ccid |= engine->execlists.ccid; + __intel_gt_pm_get(engine->gt); execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_IN); intel_engine_context_in(engine); @@ -1302,7 +1408,8 @@ static void kick_siblings(struct i915_request *rq, struct intel_context *ce) static inline void __execlists_schedule_out(struct i915_request *rq, - struct intel_engine_cs * const engine) + struct intel_engine_cs * const engine, + unsigned int ccid) { struct intel_context * const ce = rq->context; @@ -1316,10 +1423,18 @@ __execlists_schedule_out(struct i915_request *rq, * If we have just completed this context, the engine may now be * idle and we want to re-enter powersaving. */ - if (list_is_last(&rq->link, &ce->timeline->requests) && + if (list_is_last_rcu(&rq->link, &ce->timeline->requests) && i915_request_completed(rq)) intel_engine_add_retire(engine, ce->timeline); + ccid >>= GEN11_SW_CTX_ID_SHIFT - 32; + ccid &= GEN12_MAX_CONTEXT_HW_ID; + if (ccid < BITS_PER_LONG) { + GEM_BUG_ON(ccid == 0); + GEM_BUG_ON(test_bit(ccid - 1, &engine->context_tag)); + set_bit(ccid - 1, &engine->context_tag); + } + intel_context_update_runtime(ce); intel_engine_context_out(engine); execlists_context_status_change(rq, INTEL_CONTEXT_SCHEDULE_OUT); @@ -1345,15 +1460,17 @@ execlists_schedule_out(struct i915_request *rq) { struct intel_context * const ce = rq->context; struct intel_engine_cs *cur, *old; + u32 ccid; trace_i915_request_out(rq); + ccid = rq->context->lrc.ccid; old = READ_ONCE(ce->inflight); do cur = ptr_unmask_bits(old, 2) ? ptr_dec(old) : NULL; while (!try_cmpxchg(&ce->inflight, &old, cur)); if (!cur) - __execlists_schedule_out(rq, old); + __execlists_schedule_out(rq, old, ccid); i915_request_put(rq); } @@ -1361,7 +1478,7 @@ execlists_schedule_out(struct i915_request *rq) static u64 execlists_update_context(struct i915_request *rq) { struct intel_context *ce = rq->context; - u64 desc = ce->lrc_desc; + u64 desc = ce->lrc.desc; u32 tail, prev; /* @@ -1400,7 +1517,7 @@ static u64 execlists_update_context(struct i915_request *rq) */ wmb(); - ce->lrc_desc &= ~CTX_DESC_FORCE_RESTORE; + ce->lrc.desc &= ~CTX_DESC_FORCE_RESTORE; return desc; } @@ -1415,6 +1532,24 @@ static inline void write_desc(struct intel_engine_execlists *execlists, u64 desc } } +static __maybe_unused char * +dump_port(char *buf, int buflen, const char *prefix, struct i915_request *rq) +{ + if (!rq) + return ""; + + snprintf(buf, buflen, "%sccid:%x %llx:%lld%s prio %d", + prefix, + rq->context->lrc.ccid, + rq->fence.context, rq->fence.seqno, + i915_request_completed(rq) ? "!" : + i915_request_started(rq) ? "*" : + "", + rq_prio(rq)); + + return buf; +} + static __maybe_unused void trace_ports(const struct intel_engine_execlists *execlists, const char *msg, @@ -1422,18 +1557,14 @@ trace_ports(const struct intel_engine_execlists *execlists, { const struct intel_engine_cs *engine = container_of(execlists, typeof(*engine), execlists); + char __maybe_unused p0[40], p1[40]; if (!ports[0]) return; - ENGINE_TRACE(engine, "%s { %llx:%lld%s, %llx:%lld }\n", msg, - ports[0]->fence.context, - ports[0]->fence.seqno, - i915_request_completed(ports[0]) ? "!" : - i915_request_started(ports[0]) ? "*" : - "", - ports[1] ? ports[1]->fence.context : 0, - ports[1] ? ports[1]->fence.seqno : 0); + ENGINE_TRACE(engine, "%s { %s%s }\n", msg, + dump_port(p0, sizeof(p0), "", ports[0]), + dump_port(p1, sizeof(p1), ", ", ports[1])); } static inline bool @@ -1446,8 +1577,12 @@ static __maybe_unused bool assert_pending_valid(const struct intel_engine_execlists *execlists, const char *msg) { + struct intel_engine_cs *engine = + container_of(execlists, typeof(*engine), execlists); struct i915_request * const *port, *rq; struct intel_context *ce = NULL; + bool sentinel = false; + u32 ccid = -1; trace_ports(execlists, msg, execlists->pending); @@ -1456,13 +1591,14 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, return true; if (!execlists->pending[0]) { - GEM_TRACE_ERR("Nothing pending for promotion!\n"); + GEM_TRACE_ERR("%s: Nothing pending for promotion!\n", + engine->name); return false; } if (execlists->pending[execlists_num_ports(execlists)]) { - GEM_TRACE_ERR("Excess pending[%d] for promotion!\n", - execlists_num_ports(execlists)); + GEM_TRACE_ERR("%s: Excess pending[%d] for promotion!\n", + engine->name, execlists_num_ports(execlists)); return false; } @@ -1474,13 +1610,45 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, GEM_BUG_ON(!i915_request_is_active(rq)); if (ce == rq->context) { - GEM_TRACE_ERR("Dup context:%llx in pending[%zd]\n", + GEM_TRACE_ERR("%s: Dup context:%llx in pending[%zd]\n", + engine->name, ce->timeline->fence_context, port - execlists->pending); return false; } ce = rq->context; + if (ccid == ce->lrc.ccid) { + GEM_TRACE_ERR("%s: Dup ccid:%x context:%llx in pending[%zd]\n", + engine->name, + ccid, ce->timeline->fence_context, + port - execlists->pending); + return false; + } + ccid = ce->lrc.ccid; + + /* + * Sentinels are supposed to be lonely so they flush the + * current exection off the HW. Check that they are the + * only request in the pending submission. + */ + if (sentinel) { + GEM_TRACE_ERR("%s: context:%llx after sentinel in pending[%zd]\n", + engine->name, + ce->timeline->fence_context, + port - execlists->pending); + return false; + } + + sentinel = i915_request_has_sentinel(rq); + if (sentinel && port != execlists->pending) { + GEM_TRACE_ERR("%s: sentinel context:%llx not in prime position[%zd]\n", + engine->name, + ce->timeline->fence_context, + port - execlists->pending); + return false; + } + /* Hold tightly onto the lock to prevent concurrent retires! */ if (!spin_trylock_irqsave(&rq->lock, flags)) continue; @@ -1490,7 +1658,8 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, if (i915_active_is_idle(&ce->active) && !intel_context_is_barrier(ce)) { - GEM_TRACE_ERR("Inactive context:%llx in pending[%zd]\n", + GEM_TRACE_ERR("%s: Inactive context:%llx in pending[%zd]\n", + engine->name, ce->timeline->fence_context, port - execlists->pending); ok = false; @@ -1498,7 +1667,8 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, } if (!i915_vma_is_pinned(ce->state)) { - GEM_TRACE_ERR("Unpinned context:%llx in pending[%zd]\n", + GEM_TRACE_ERR("%s: Unpinned context:%llx in pending[%zd]\n", + engine->name, ce->timeline->fence_context, port - execlists->pending); ok = false; @@ -1506,7 +1676,8 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, } if (!i915_vma_is_pinned(ce->ring->vma)) { - GEM_TRACE_ERR("Unpinned ring:%llx in pending[%zd]\n", + GEM_TRACE_ERR("%s: Unpinned ring:%llx in pending[%zd]\n", + engine->name, ce->timeline->fence_context, port - execlists->pending); ok = false; @@ -1576,6 +1747,11 @@ static bool can_merge_ctx(const struct intel_context *prev, return true; } +static unsigned long i915_request_flags(const struct i915_request *rq) +{ + return READ_ONCE(rq->fence.flags); +} + static bool can_merge_rq(const struct i915_request *prev, const struct i915_request *next) { @@ -1593,7 +1769,7 @@ static bool can_merge_rq(const struct i915_request *prev, if (i915_request_completed(next)) return true; - if (unlikely((prev->fence.flags ^ next->fence.flags) & + if (unlikely((i915_request_flags(prev) ^ i915_request_flags(next)) & (BIT(I915_FENCE_FLAG_NOPREEMPT) | BIT(I915_FENCE_FLAG_SENTINEL)))) return false; @@ -1601,6 +1777,7 @@ static bool can_merge_rq(const struct i915_request *prev, if (!can_merge_ctx(prev->context, next->context)) return false; + GEM_BUG_ON(i915_seqno_passed(prev->fence.seqno, next->fence.seqno)); return true; } @@ -1635,31 +1812,16 @@ static bool virtual_matches(const struct virtual_engine *ve, return true; } -static void virtual_xfer_breadcrumbs(struct virtual_engine *ve, - struct intel_engine_cs *engine) +static void virtual_xfer_breadcrumbs(struct virtual_engine *ve) { - struct intel_engine_cs *old = ve->siblings[0]; - - /* All unattached (rq->engine == old) must already be completed */ - - spin_lock(&old->breadcrumbs.irq_lock); - if (!list_empty(&ve->context.signal_link)) { - list_move_tail(&ve->context.signal_link, - &engine->breadcrumbs.signalers); - intel_engine_signal_breadcrumbs(engine); - } - spin_unlock(&old->breadcrumbs.irq_lock); -} - -static struct i915_request * -last_active(const struct intel_engine_execlists *execlists) -{ - struct i915_request * const *last = READ_ONCE(execlists->active); - - while (*last && i915_request_completed(*last)) - last++; - - return *last; + /* + * All the outstanding signals on ve->siblings[0] must have + * been completed, just pending the interrupt handler. As those + * signals still refer to the old sibling (via rq->engine), we must + * transfer those to the old irq_worker to keep our locking + * consistent. + */ + intel_engine_transfer_stale_breadcrumbs(ve->siblings[0], &ve->context); } #define for_each_waiter(p__, rq__) \ @@ -1668,9 +1830,9 @@ last_active(const struct intel_engine_execlists *execlists) wait_link) #define for_each_signaler(p__, rq__) \ - list_for_each_entry_lockless(p__, \ - &(rq__)->sched.signalers_list, \ - signal_link) + list_for_each_entry_rcu(p__, \ + &(rq__)->sched.signalers_list, \ + signal_link) static void defer_request(struct i915_request *rq, struct list_head * const pl) { @@ -1693,12 +1855,16 @@ static void defer_request(struct i915_request *rq, struct list_head * const pl) struct i915_request *w = container_of(p->waiter, typeof(*w), sched); + if (p->flags & I915_DEPENDENCY_WEAK) + continue; + /* Leave semaphores spinning on the other engines */ if (w->engine != rq->engine) continue; /* No waiter should start before its signaler */ - GEM_BUG_ON(i915_request_started(w) && + GEM_BUG_ON(i915_request_has_initial_breadcrumb(w) && + i915_request_started(w) && !i915_request_completed(rq)); GEM_BUG_ON(i915_request_is_active(w)); @@ -1728,22 +1894,47 @@ static void defer_active(struct intel_engine_cs *engine) } static bool -need_timeslice(struct intel_engine_cs *engine, const struct i915_request *rq) +need_timeslice(const struct intel_engine_cs *engine, + const struct i915_request *rq) { int hint; if (!intel_engine_has_timeslices(engine)) return false; - if (list_is_last(&rq->sched.link, &engine->active.requests)) - return false; - - hint = max(rq_prio(list_next_entry(rq, sched.link)), - engine->execlists.queue_priority_hint); + hint = engine->execlists.queue_priority_hint; + if (!list_is_last(&rq->sched.link, &engine->active.requests)) + hint = max(hint, rq_prio(list_next_entry(rq, sched.link))); return hint >= effective_prio(rq); } +static bool +timeslice_yield(const struct intel_engine_execlists *el, + const struct i915_request *rq) +{ + /* + * Once bitten, forever smitten! + * + * If the active context ever busy-waited on a semaphore, + * it will be treated as a hog until the end of its timeslice (i.e. + * until it is scheduled out and replaced by a new submission, + * possibly even its own lite-restore). The HW only sends an interrupt + * on the first miss, and we do know if that semaphore has been + * signaled, or even if it is now stuck on another semaphore. Play + * safe, yield if it might be stuck -- it will be given a fresh + * timeslice in the near future. + */ + return rq->context->lrc.ccid == READ_ONCE(el->yield); +} + +static bool +timeslice_expired(const struct intel_engine_execlists *el, + const struct i915_request *rq) +{ + return timer_expired(&el->timer) || timeslice_yield(el, rq); +} + static int switch_prio(struct intel_engine_cs *engine, const struct i915_request *rq) { @@ -1759,15 +1950,15 @@ timeslice(const struct intel_engine_cs *engine) return READ_ONCE(engine->props.timeslice_duration_ms); } -static unsigned long -active_timeslice(const struct intel_engine_cs *engine) +static unsigned long active_timeslice(const struct intel_engine_cs *engine) { - const struct i915_request *rq = *engine->execlists.active; + const struct intel_engine_execlists *execlists = &engine->execlists; + const struct i915_request *rq = *execlists->active; if (!rq || i915_request_completed(rq)) return 0; - if (engine->execlists.switch_priority_hint < effective_prio(rq)) + if (READ_ONCE(execlists->switch_priority_hint) < effective_prio(rq)) return 0; return timeslice(engine); @@ -1775,10 +1966,39 @@ active_timeslice(const struct intel_engine_cs *engine) static void set_timeslice(struct intel_engine_cs *engine) { + unsigned long duration; + if (!intel_engine_has_timeslices(engine)) return; - set_timer_ms(&engine->execlists.timer, active_timeslice(engine)); + duration = active_timeslice(engine); + ENGINE_TRACE(engine, "bump timeslicing, interval:%lu", duration); + + set_timer_ms(&engine->execlists.timer, duration); +} + +static void start_timeslice(struct intel_engine_cs *engine) +{ + struct intel_engine_execlists *execlists = &engine->execlists; + const int prio = queue_prio(execlists); + unsigned long duration; + + if (!intel_engine_has_timeslices(engine)) + return; + + WRITE_ONCE(execlists->switch_priority_hint, prio); + if (prio == INT_MIN) + return; + + if (timer_pending(&execlists->timer)) + return; + + duration = timeslice(engine); + ENGINE_TRACE(engine, + "start timeslicing, prio:%d, interval:%lu", + prio, duration); + + set_timer_ms(&execlists->timer, duration); } static void record_preemption(struct intel_engine_execlists *execlists) @@ -1786,11 +2006,9 @@ static void record_preemption(struct intel_engine_execlists *execlists) (void)I915_SELFTEST_ONLY(execlists->preempt_hang.count++); } -static unsigned long active_preempt_timeout(struct intel_engine_cs *engine) +static unsigned long active_preempt_timeout(struct intel_engine_cs *engine, + const struct i915_request *rq) { - struct i915_request *rq; - - rq = last_active(&engine->execlists); if (!rq) return 0; @@ -1801,13 +2019,14 @@ static unsigned long active_preempt_timeout(struct intel_engine_cs *engine) return READ_ONCE(engine->props.preempt_timeout_ms); } -static void set_preempt_timeout(struct intel_engine_cs *engine) +static void set_preempt_timeout(struct intel_engine_cs *engine, + const struct i915_request *rq) { if (!intel_engine_has_preempt_reset(engine)) return; set_timer_ms(&engine->execlists.preempt, - active_preempt_timeout(engine)); + active_preempt_timeout(engine, rq)); } static inline void clear_ports(struct i915_request **ports, int count) @@ -1820,6 +2039,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) struct intel_engine_execlists * const execlists = &engine->execlists; struct i915_request **port = execlists->pending; struct i915_request ** const last_port = port + execlists->port_mask; + struct i915_request * const *active; struct i915_request *last; struct rb_node *rb; bool submit = false; @@ -1874,9 +2094,27 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * i.e. we will retrigger preemption following the ack in case * of trouble. */ - last = last_active(execlists); - if (last) { + active = READ_ONCE(execlists->active); + + /* + * In theory we can skip over completed contexts that have not + * yet been processed by events (as those events are in flight): + * + * while ((last = *active) && i915_request_completed(last)) + * active++; + * + * However, the GPU cannot handle this as it will ultimately + * find itself trying to jump back into a context it has just + * completed and barf. + */ + + if ((last = *active)) { if (need_preempt(engine, last, rb)) { + if (i915_request_completed(last)) { + tasklet_hi_schedule(&execlists->tasklet); + return; + } + ENGINE_TRACE(engine, "preempting last=%llx:%lld, prio=%d, hint=%d\n", last->fence.context, @@ -1903,13 +2141,19 @@ static void execlists_dequeue(struct intel_engine_cs *engine) last = NULL; } else if (need_timeslice(engine, last) && - timer_expired(&engine->execlists.timer)) { + timeslice_expired(execlists, last)) { + if (i915_request_completed(last)) { + tasklet_hi_schedule(&execlists->tasklet); + return; + } + ENGINE_TRACE(engine, - "expired last=%llx:%lld, prio=%d, hint=%d\n", + "expired last=%llx:%lld, prio=%d, hint=%d, yield?=%s\n", last->fence.context, last->fence.seqno, last->sched.attr.priority, - execlists->queue_priority_hint); + execlists->queue_priority_hint, + yesno(timeslice_yield(execlists, last))); ring_set_paused(engine, 1); defer_active(engine); @@ -1944,11 +2188,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * Even if ELSP[1] is occupied and not worthy * of timeslices, our queue might be. */ - if (!execlists->timer.expires && - need_timeslice(engine, last)) - set_timer_ms(&execlists->timer, - timeslice(engine)); - + start_timeslice(engine); return; } } @@ -1983,7 +2223,8 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (last && !can_merge_rq(last, rq)) { spin_unlock(&ve->base.active.lock); - return; /* leave this for another */ + start_timeslice(engine); + return; /* leave this for another sibling */ } ENGINE_TRACE(engine, @@ -1995,13 +2236,14 @@ static void execlists_dequeue(struct intel_engine_cs *engine) "", yesno(engine != ve->siblings[0])); - ve->request = NULL; - ve->base.execlists.queue_priority_hint = INT_MIN; + WRITE_ONCE(ve->request, NULL); + WRITE_ONCE(ve->base.execlists.queue_priority_hint, + INT_MIN); rb_erase_cached(rb, &execlists->virtual); RB_CLEAR_NODE(rb); GEM_BUG_ON(!(rq->execution_mask & engine->mask)); - rq->engine = engine; + WRITE_ONCE(rq->engine, engine); if (engine != ve->siblings[0]) { u32 *regs = ve->context.lrc_reg_state; @@ -2014,7 +2256,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) engine); if (!list_empty(&ve->context.signals)) - virtual_xfer_breadcrumbs(ve, engine); + virtual_xfer_breadcrumbs(ve); /* * Move the bound engine to the top of the list @@ -2121,6 +2363,9 @@ static void execlists_dequeue(struct intel_engine_cs *engine) GEM_BUG_ON(last && !can_merge_ctx(last->context, rq->context)); + GEM_BUG_ON(last && + i915_seqno_passed(last->fence.seqno, + rq->fence.seqno)); submit = true; last = rq; @@ -2159,7 +2404,7 @@ done: * Skip if we ended up with exactly the same set of requests, * e.g. trying to timeslice a pair of ordered contexts */ - if (!memcmp(execlists->active, execlists->pending, + if (!memcmp(active, execlists->pending, (port - execlists->pending + 1) * sizeof(*port))) { do execlists_schedule_out(fetch_and_zero(port)); @@ -2169,8 +2414,9 @@ done: } clear_ports(port + 1, last_port - port); + WRITE_ONCE(execlists->yield, -1); + set_preempt_timeout(engine, *active); execlists_submit_ports(engine); - set_preempt_timeout(engine); } else { skip_submit: ring_set_paused(engine, 0); @@ -2191,6 +2437,7 @@ cancel_port_requests(struct intel_engine_execlists * const execlists) execlists_schedule_out(*port); clear_ports(execlists->inflight, ARRAY_SIZE(execlists->inflight)); + smp_wmb(); /* complete the seqlock for execlists_active() */ WRITE_ONCE(execlists->active, execlists->inflight); } @@ -2339,12 +2586,11 @@ static void process_csb(struct intel_engine_cs *engine) if (promote) { struct i915_request * const *old = execlists->active; - GEM_BUG_ON(!assert_pending_valid(execlists, "promote")); - ring_set_paused(engine, 0); /* Point active to the new ELSP; prevent overwriting */ WRITE_ONCE(execlists->active, execlists->pending); + smp_wmb(); /* notify execlists_active() */ /* cancel old inflight, prepare for switch */ trace_ports(execlists, "preempted", old); @@ -2352,11 +2598,13 @@ static void process_csb(struct intel_engine_cs *engine) execlists_schedule_out(*old++); /* switch pending to inflight */ - WRITE_ONCE(execlists->active, - memcpy(execlists->inflight, - execlists->pending, - execlists_num_ports(execlists) * - sizeof(*execlists->pending))); + GEM_BUG_ON(!assert_pending_valid(execlists, "promote")); + memcpy(execlists->inflight, + execlists->pending, + execlists_num_ports(execlists) * + sizeof(*execlists->pending)); + smp_wmb(); /* complete the seqlock */ + WRITE_ONCE(execlists->active, execlists->inflight); WRITE_ONCE(execlists->pending[0], NULL); } else { @@ -2369,17 +2617,21 @@ static void process_csb(struct intel_engine_cs *engine) * We rely on the hardware being strongly * ordered, that the breadcrumb write is * coherent (visible from the CPU) before the - * user interrupt and CSB is processed. + * user interrupt is processed. One might assume + * that the breadcrumb write being before the + * user interrupt and the CS event for the context + * switch would therefore be before the CS event + * itself... */ if (GEM_SHOW_DEBUG() && - !i915_request_completed(*execlists->active) && - !reset_in_progress(execlists)) { - struct i915_request *rq __maybe_unused = - *execlists->active; + !i915_request_completed(*execlists->active)) { + struct i915_request *rq = *execlists->active; const u32 *regs __maybe_unused = rq->context->lrc_reg_state; ENGINE_TRACE(engine, + "context completed before request!\n"); + ENGINE_TRACE(engine, "ring:{start:0x%08x, head:%04x, tail:%04x, ctl:%08x, mode:%08x}\n", ENGINE_READ(engine, RING_START), ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR, @@ -2398,8 +2650,6 @@ static void process_csb(struct intel_engine_cs *engine) regs[CTX_RING_START], regs[CTX_RING_HEAD], regs[CTX_RING_TAIL]); - - GEM_BUG_ON("context completed before request"); } execlists_schedule_out(*execlists->active++); @@ -2533,11 +2783,13 @@ unlock: static bool hold_request(const struct i915_request *rq) { struct i915_dependency *p; + bool result = false; /* * If one of our ancestors is on hold, we must also be on hold, * otherwise we will bypass it and execute before it. */ + rcu_read_lock(); for_each_signaler(p, rq) { const struct i915_request *s = container_of(p->signaler, typeof(*s), sched); @@ -2545,11 +2797,13 @@ static bool hold_request(const struct i915_request *rq) if (s->engine != rq->engine) continue; - if (i915_request_on_hold(s)) - return true; + result = i915_request_on_hold(s); + if (result) + break; } + rcu_read_unlock(); - return false; + return result; } static void __execlists_unhold(struct i915_request *rq) @@ -2575,6 +2829,10 @@ static void __execlists_unhold(struct i915_request *rq) struct i915_request *w = container_of(p->waiter, typeof(*w), sched); + /* Propagate any change in error status */ + if (rq->fence.error) + i915_request_set_error_once(w, rq->fence.error); + if (w->engine != rq->engine) continue; @@ -2681,6 +2939,45 @@ err_cap: return NULL; } +static struct i915_request * +active_context(struct intel_engine_cs *engine, u32 ccid) +{ + const struct intel_engine_execlists * const el = &engine->execlists; + struct i915_request * const *port, *rq; + + /* + * Use the most recent result from process_csb(), but just in case + * we trigger an error (via interrupt) before the first CS event has + * been written, peek at the next submission. + */ + + for (port = el->active; (rq = *port); port++) { + if (rq->context->lrc.ccid == ccid) { + ENGINE_TRACE(engine, + "ccid found at active:%zd\n", + port - el->active); + return rq; + } + } + + for (port = el->pending; (rq = *port); port++) { + if (rq->context->lrc.ccid == ccid) { + ENGINE_TRACE(engine, + "ccid found at pending:%zd\n", + port - el->pending); + return rq; + } + } + + ENGINE_TRACE(engine, "ccid:%x not found\n", ccid); + return NULL; +} + +static u32 active_ccid(struct intel_engine_cs *engine) +{ + return ENGINE_READ_FW(engine, RING_EXECLIST_STATUS_HI); +} + static bool execlists_capture(struct intel_engine_cs *engine) { struct execlists_capture *cap; @@ -2698,7 +2995,7 @@ static bool execlists_capture(struct intel_engine_cs *engine) return true; spin_lock_irq(&engine->active.lock); - cap->rq = execlists_active(&engine->execlists); + cap->rq = active_context(engine, active_ccid(engine)); if (cap->rq) { cap->rq = active_request(cap->rq->context->timeline, cap->rq); cap->rq = i915_request_get_rcu(cap->rq); @@ -2846,10 +3143,14 @@ static void __submit_queue_imm(struct intel_engine_cs *engine) if (reset_in_progress(execlists)) return; /* defer until we restart the engine following reset */ - if (execlists->tasklet.func == execlists_submission_tasklet) - __execlists_submission_tasklet(engine); - else - tasklet_hi_schedule(&execlists->tasklet); + /* Hopefully we clear execlists->pending[] to let us through */ + if (READ_ONCE(execlists->pending[0]) && + tasklet_trylock(&execlists->tasklet)) { + process_csb(engine); + tasklet_unlock(&execlists->tasklet); + } + + __execlists_submission_tasklet(engine); } static void submit_queue(struct intel_engine_cs *engine, @@ -2935,19 +3236,139 @@ check_redzone(const void *vaddr, const struct intel_engine_cs *engine) vaddr += engine->context_size; if (memchr_inv(vaddr, CONTEXT_REDZONE, I915_GTT_PAGE_SIZE)) - dev_err_once(engine->i915->drm.dev, + drm_err_once(&engine->i915->drm, "%s context redzone overwritten!\n", engine->name); } static void execlists_context_unpin(struct intel_context *ce) { - check_redzone((void *)ce->lrc_reg_state - LRC_STATE_PN * PAGE_SIZE, + check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, ce->engine); i915_gem_object_unpin_map(ce->state->obj); } +static u32 * +gen12_emit_timestamp_wa(const struct intel_context *ce, u32 *cs) +{ + *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); + *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + + CTX_TIMESTAMP * sizeof(u32); + *cs++ = 0; + + *cs++ = MI_LOAD_REGISTER_REG | + MI_LRR_SOURCE_CS_MMIO | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); + *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); + + *cs++ = MI_LOAD_REGISTER_REG | + MI_LRR_SOURCE_CS_MMIO | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); + *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(0)); + + return cs; +} + +static u32 * +gen12_emit_restore_scratch(const struct intel_context *ce, u32 *cs) +{ + GEM_BUG_ON(lrc_ring_gpr0(ce->engine) == -1); + + *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); + *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + + (lrc_ring_gpr0(ce->engine) + 1) * sizeof(u32); + *cs++ = 0; + + return cs; +} + +static u32 * +gen12_emit_cmd_buf_wa(const struct intel_context *ce, u32 *cs) +{ + GEM_BUG_ON(lrc_ring_cmd_buf_cctl(ce->engine) == -1); + + *cs++ = MI_LOAD_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); + *cs++ = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET + + (lrc_ring_cmd_buf_cctl(ce->engine) + 1) * sizeof(u32); + *cs++ = 0; + + *cs++ = MI_LOAD_REGISTER_REG | + MI_LRR_SOURCE_CS_MMIO | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(GEN8_RING_CS_GPR(0, 0)); + *cs++ = i915_mmio_reg_offset(RING_CMD_BUF_CCTL(0)); + + return cs; +} + +static u32 * +gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs) +{ + cs = gen12_emit_timestamp_wa(ce, cs); + cs = gen12_emit_cmd_buf_wa(ce, cs); + cs = gen12_emit_restore_scratch(ce, cs); + + return cs; +} + +static u32 * +gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs) +{ + cs = gen12_emit_timestamp_wa(ce, cs); + cs = gen12_emit_restore_scratch(ce, cs); + + return cs; +} + +static inline u32 context_wa_bb_offset(const struct intel_context *ce) +{ + return PAGE_SIZE * ce->wa_bb_page; +} + +static u32 *context_indirect_bb(const struct intel_context *ce) +{ + void *ptr; + + GEM_BUG_ON(!ce->wa_bb_page); + + ptr = ce->lrc_reg_state; + ptr -= LRC_STATE_OFFSET; /* back to start of context image */ + ptr += context_wa_bb_offset(ce); + + return ptr; +} + +static void +setup_indirect_ctx_bb(const struct intel_context *ce, + const struct intel_engine_cs *engine, + u32 *(*emit)(const struct intel_context *, u32 *)) +{ + u32 * const start = context_indirect_bb(ce); + u32 *cs; + + cs = emit(ce, start); + GEM_BUG_ON(cs - start > I915_GTT_PAGE_SIZE / sizeof(*cs)); + while ((unsigned long)cs % CACHELINE_BYTES) + *cs++ = MI_NOOP; + + lrc_ring_setup_indirect_ctx(ce->lrc_reg_state, engine, + i915_ggtt_offset(ce->state) + + context_wa_bb_offset(ce), + (cs - start) * sizeof(*cs)); +} + static void __execlists_update_reg_state(const struct intel_context *ce, const struct intel_engine_cs *engine, @@ -2962,6 +3383,7 @@ __execlists_update_reg_state(const struct intel_context *ce, regs[CTX_RING_START] = i915_ggtt_offset(ring->vma); regs[CTX_RING_HEAD] = head; regs[CTX_RING_TAIL] = ring->tail; + regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; /* RPCS */ if (engine->class == RENDER_CLASS) { @@ -2970,6 +3392,18 @@ __execlists_update_reg_state(const struct intel_context *ce, i915_oa_init_reg_state(ce, engine); } + + if (ce->wa_bb_page) { + u32 *(*fn)(const struct intel_context *ce, u32 *cs); + + fn = gen12_emit_indirect_ctx_xcs; + if (ce->engine->class == RENDER_CLASS) + fn = gen12_emit_indirect_ctx_rcs; + + /* Mutually exclusive wrt to global indirect bb */ + GEM_BUG_ON(engine->wa_ctx.indirect_ctx.size); + setup_indirect_ctx_bb(ce, engine, fn); + } } static int @@ -2987,8 +3421,8 @@ __execlists_context_pin(struct intel_context *ce, if (IS_ERR(vaddr)) return PTR_ERR(vaddr); - ce->lrc_desc = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE; - ce->lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE; + ce->lrc.lrca = lrc_descriptor(ce, engine) | CTX_DESC_FORCE_RESTORE; + ce->lrc_reg_state = vaddr + LRC_STATE_OFFSET; __execlists_update_reg_state(ce, engine, ce->ring->tail); return 0; @@ -3016,7 +3450,7 @@ static void execlists_context_reset(struct intel_context *ce) ce, ce->engine, ce->ring, true); __execlists_update_reg_state(ce, ce->engine, ce->ring->tail); - ce->lrc_desc |= CTX_DESC_FORCE_RESTORE; + ce->lrc.desc |= CTX_DESC_FORCE_RESTORE; } static const struct intel_context_ops execlists_context_ops = { @@ -3036,6 +3470,7 @@ static int gen8_emit_init_breadcrumb(struct i915_request *rq) { u32 *cs; + GEM_BUG_ON(i915_request_has_initial_breadcrumb(rq)); if (!i915_request_timeline(rq)->has_initial_breadcrumb) return 0; @@ -3062,6 +3497,56 @@ static int gen8_emit_init_breadcrumb(struct i915_request *rq) /* Record the updated position of the request's payload */ rq->infix = intel_ring_offset(rq, cs); + __set_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags); + + return 0; +} + +static int emit_pdps(struct i915_request *rq) +{ + const struct intel_engine_cs * const engine = rq->engine; + struct i915_ppgtt * const ppgtt = i915_vm_to_ppgtt(rq->context->vm); + int err, i; + u32 *cs; + + GEM_BUG_ON(intel_vgpu_active(rq->i915)); + + /* + * Beware ye of the dragons, this sequence is magic! + * + * Small changes to this sequence can cause anything from + * GPU hangs to forcewake errors and machine lockups! + */ + + /* Flush any residual operations from the context load */ + err = engine->emit_flush(rq, EMIT_FLUSH); + if (err) + return err; + + /* Magic required to prevent forcewake errors! */ + err = engine->emit_flush(rq, EMIT_INVALIDATE); + if (err) + return err; + + cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); + if (IS_ERR(cs)) + return PTR_ERR(cs); + + /* Ensure the LRI have landed before we invalidate & continue */ + *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED; + for (i = GEN8_3LVL_PDPES; i--; ) { + const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); + u32 base = engine->mmio_base; + + *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); + *cs++ = upper_32_bits(pd_daddr); + *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); + *cs++ = lower_32_bits(pd_daddr); + } + *cs++ = MI_NOOP; + + intel_ring_advance(rq, cs); + return 0; } @@ -3086,6 +3571,12 @@ static int execlists_request_alloc(struct i915_request *request) * to cancel/unwind this request now. */ + if (!i915_vm_is_4lvl(request->context->vm)) { + ret = emit_pdps(request); + if (ret) + return ret; + } + /* Unconditionally invalidate GPU caches and TLBs. */ ret = request->engine->emit_flush(request, EMIT_INVALIDATE); if (ret) @@ -3386,7 +3877,8 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) ret = lrc_setup_wa_ctx(engine); if (ret) { - DRM_DEBUG_DRIVER("Failed to setup context WA page: %d\n", ret); + drm_dbg(&engine->i915->drm, + "Failed to setup context WA page: %d\n", ret); return ret; } @@ -3419,6 +3911,72 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine) return ret; } +static void reset_csb_pointers(struct intel_engine_cs *engine) +{ + struct intel_engine_execlists * const execlists = &engine->execlists; + const unsigned int reset_value = execlists->csb_size - 1; + + ring_set_paused(engine, 0); + + /* + * Sometimes Icelake forgets to reset its pointers on a GPU reset. + * Bludgeon them with a mmio update to be sure. + */ + ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR, + 0xffff << 16 | reset_value << 8 | reset_value); + ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); + + /* + * After a reset, the HW starts writing into CSB entry [0]. We + * therefore have to set our HEAD pointer back one entry so that + * the *first* entry we check is entry 0. To complicate this further, + * as we don't wait for the first interrupt after reset, we have to + * fake the HW write to point back to the last entry so that our + * inline comparison of our cached head position against the last HW + * write works even before the first interrupt. + */ + execlists->csb_head = reset_value; + WRITE_ONCE(*execlists->csb_write, reset_value); + wmb(); /* Make sure this is visible to HW (paranoia?) */ + + invalidate_csb_entries(&execlists->csb_status[0], + &execlists->csb_status[reset_value]); + + /* Once more for luck and our trusty paranoia */ + ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR, + 0xffff << 16 | reset_value << 8 | reset_value); + ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); + + GEM_BUG_ON(READ_ONCE(*execlists->csb_write) != reset_value); +} + +static void execlists_sanitize(struct intel_engine_cs *engine) +{ + /* + * Poison residual state on resume, in case the suspend didn't! + * + * We have to assume that across suspend/resume (or other loss + * of control) that the contents of our pinned buffers has been + * lost, replaced by garbage. Since this doesn't always happen, + * let's poison such state so that we more quickly spot when + * we falsely assume it has been preserved. + */ + if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) + memset(engine->status_page.addr, POISON_INUSE, PAGE_SIZE); + + reset_csb_pointers(engine); + + /* + * The kernel_context HWSP is stored in the status_page. As above, + * that may be lost on resume/initialisation, and so we need to + * reset the value in the HWSP. + */ + intel_timeline_reset_seqno(engine->kernel_context->timeline); + + /* And scrub the dirty cachelines for the HWSP */ + clflush_cache_range(engine->status_page.addr, PAGE_SIZE); +} + static void enable_error_interrupt(struct intel_engine_cs *engine) { u32 status; @@ -3429,7 +3987,7 @@ static void enable_error_interrupt(struct intel_engine_cs *engine) status = ENGINE_READ(engine, RING_ESR); if (unlikely(status)) { - dev_err(engine->i915->drm.dev, + drm_err(&engine->i915->drm, "engine '%s' resumed still in error: %08x\n", engine->name, status); __intel_gt_reset(engine->gt, engine->mask); @@ -3485,7 +4043,7 @@ static void enable_execlists(struct intel_engine_cs *engine) enable_error_interrupt(engine); - engine->context_tag = 0; + engine->context_tag = GENMASK(BITS_PER_LONG - 2, 0); } static bool unexpected_starting_state(struct intel_engine_cs *engine) @@ -3493,7 +4051,8 @@ static bool unexpected_starting_state(struct intel_engine_cs *engine) bool unexpected = false; if (ENGINE_READ_FW(engine, RING_MI_MODE) & STOP_RING) { - DRM_DEBUG_DRIVER("STOP_RING still set in RING_MI_MODE\n"); + drm_dbg(&engine->i915->drm, + "STOP_RING still set in RING_MI_MODE\n"); unexpected = true; } @@ -3553,39 +4112,10 @@ static void execlists_reset_prepare(struct intel_engine_cs *engine) * * FIXME: Wa for more modern gens needs to be validated */ + ring_set_paused(engine, 1); intel_engine_stop_cs(engine); -} - -static void reset_csb_pointers(struct intel_engine_cs *engine) -{ - struct intel_engine_execlists * const execlists = &engine->execlists; - const unsigned int reset_value = execlists->csb_size - 1; - - ring_set_paused(engine, 0); - - /* - * After a reset, the HW starts writing into CSB entry [0]. We - * therefore have to set our HEAD pointer back one entry so that - * the *first* entry we check is entry 0. To complicate this further, - * as we don't wait for the first interrupt after reset, we have to - * fake the HW write to point back to the last entry so that our - * inline comparison of our cached head position against the last HW - * write works even before the first interrupt. - */ - execlists->csb_head = reset_value; - WRITE_ONCE(*execlists->csb_write, reset_value); - wmb(); /* Make sure this is visible to HW (paranoia?) */ - /* - * Sometimes Icelake forgets to reset its pointers on a GPU reset. - * Bludgeon them with a mmio update to be sure. - */ - ENGINE_WRITE(engine, RING_CONTEXT_STATUS_PTR, - reset_value << 8 | reset_value); - ENGINE_POSTING_READ(engine, RING_CONTEXT_STATUS_PTR); - - invalidate_csb_entries(&execlists->csb_status[0], - &execlists->csb_status[reset_value]); + engine->execlists.reset_ccid = active_ccid(engine); } static void __reset_stop_ring(u32 *regs, const struct intel_engine_cs *engine) @@ -3628,13 +4158,10 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) * its request, it was still running at the time of the * reset and will have been clobbered. */ - rq = execlists_active(execlists); + rq = active_context(engine, engine->execlists.reset_ccid); if (!rq) goto unwind; - /* We still have requests in-flight; the engine should be active */ - GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); - ce = rq->context; GEM_BUG_ON(!i915_vma_is_pinned(ce->state)); @@ -3644,8 +4171,12 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) goto out_replay; } + /* We still have requests in-flight; the engine should be active */ + GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); + /* Context has requests still in-flight; it should not be idle! */ GEM_BUG_ON(i915_active_is_idle(&ce->active)); + rq = active_request(ce->timeline, rq); head = intel_ring_wrap(ce->ring, rq->head); GEM_BUG_ON(head == ce->ring->tail); @@ -3677,8 +4208,6 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) * image back to the expected values to skip over the guilty request. */ __i915_request_reset(rq, stalled); - if (!stalled) - goto out_replay; /* * We want a simple context + ring to execute the breadcrumb update. @@ -3688,15 +4217,12 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) * future request will be after userspace has had the opportunity * to recreate its own state. */ - GEM_BUG_ON(!intel_context_is_pinned(ce)); - restore_default_state(ce, engine); - out_replay: ENGINE_TRACE(engine, "replay {head:%04x, tail:%04x}\n", head, ce->ring->tail); __execlists_reset_reg_state(ce, engine); __execlists_update_reg_state(ce, engine, head); - ce->lrc_desc |= CTX_DESC_FORCE_RESTORE; /* paranoid: GPU was reset! */ + ce->lrc.desc |= CTX_DESC_FORCE_RESTORE; /* paranoid: GPU was reset! */ unwind: /* Push back any incomplete requests for replay after the reset. */ @@ -3719,7 +4245,10 @@ static void execlists_reset_rewind(struct intel_engine_cs *engine, bool stalled) static void nop_submission_tasklet(unsigned long data) { + struct intel_engine_cs * const engine = (struct intel_engine_cs *)data; + /* The driver is wedged; don't process any more events. */ + WRITE_ONCE(engine->execlists.queue_priority_hint, INT_MIN); } static void execlists_reset_cancel(struct intel_engine_cs *engine) @@ -4053,6 +4582,42 @@ static u32 preparser_disable(bool state) return MI_ARB_CHECK | 1 << 8 | state; } +static i915_reg_t aux_inv_reg(const struct intel_engine_cs *engine) +{ + static const i915_reg_t vd[] = { + GEN12_VD0_AUX_NV, + GEN12_VD1_AUX_NV, + GEN12_VD2_AUX_NV, + GEN12_VD3_AUX_NV, + }; + + static const i915_reg_t ve[] = { + GEN12_VE0_AUX_NV, + GEN12_VE1_AUX_NV, + }; + + if (engine->class == VIDEO_DECODE_CLASS) + return vd[engine->instance]; + + if (engine->class == VIDEO_ENHANCEMENT_CLASS) + return ve[engine->instance]; + + GEM_BUG_ON("unknown aux_inv_reg\n"); + + return INVALID_MMIO_REG; +} + +static u32 * +gen12_emit_aux_table_inv(const i915_reg_t inv_reg, u32 *cs) +{ + *cs++ = MI_LOAD_REGISTER_IMM(1); + *cs++ = i915_mmio_reg_offset(inv_reg); + *cs++ = AUX_INV; + *cs++ = MI_NOOP; + + return cs; +} + static int gen12_emit_flush_render(struct i915_request *request, u32 mode) { @@ -4061,13 +4626,13 @@ static int gen12_emit_flush_render(struct i915_request *request, u32 *cs; flags |= PIPE_CONTROL_TILE_CACHE_FLUSH; + flags |= PIPE_CONTROL_FLUSH_L3; flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; /* Wa_1409600907:tgl */ flags |= PIPE_CONTROL_DEPTH_STALL; flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; flags |= PIPE_CONTROL_FLUSH_ENABLE; - flags |= PIPE_CONTROL_HDC_PIPELINE_FLUSH; flags |= PIPE_CONTROL_STORE_DATA_INDEX; flags |= PIPE_CONTROL_QW_WRITE; @@ -4078,7 +4643,9 @@ static int gen12_emit_flush_render(struct i915_request *request, if (IS_ERR(cs)) return PTR_ERR(cs); - cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); + cs = gen12_emit_pipe_control(cs, + PIPE_CONTROL0_HDC_PIPELINE_FLUSH, + flags, LRC_PPHWSP_SCRATCH_ADDR); intel_ring_advance(request, cs); } @@ -4093,14 +4660,13 @@ static int gen12_emit_flush_render(struct i915_request *request, flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; - flags |= PIPE_CONTROL_L3_RO_CACHE_INVALIDATE; flags |= PIPE_CONTROL_STORE_DATA_INDEX; flags |= PIPE_CONTROL_QW_WRITE; flags |= PIPE_CONTROL_CS_STALL; - cs = intel_ring_begin(request, 8); + cs = intel_ring_begin(request, 8 + 4); if (IS_ERR(cs)) return PTR_ERR(cs); @@ -4113,29 +4679,62 @@ static int gen12_emit_flush_render(struct i915_request *request, cs = gen8_emit_pipe_control(cs, flags, LRC_PPHWSP_SCRATCH_ADDR); + /* hsdes: 1809175790 */ + cs = gen12_emit_aux_table_inv(GEN12_GFX_CCS_AUX_NV, cs); + *cs++ = preparser_disable(false); intel_ring_advance(request, cs); + } - /* - * Wa_1604544889:tgl - */ - if (IS_TGL_REVID(request->i915, TGL_REVID_A0, TGL_REVID_A0)) { - flags = 0; - flags |= PIPE_CONTROL_CS_STALL; - flags |= PIPE_CONTROL_HDC_PIPELINE_FLUSH; + return 0; +} - flags |= PIPE_CONTROL_STORE_DATA_INDEX; - flags |= PIPE_CONTROL_QW_WRITE; +static int gen12_emit_flush(struct i915_request *request, u32 mode) +{ + intel_engine_mask_t aux_inv = 0; + u32 cmd, *cs; + + if (mode & EMIT_INVALIDATE) + aux_inv = request->engine->mask & ~BIT(BCS0); + + cs = intel_ring_begin(request, + 4 + (aux_inv ? 2 * hweight8(aux_inv) + 2 : 0)); + if (IS_ERR(cs)) + return PTR_ERR(cs); - cs = intel_ring_begin(request, 6); - if (IS_ERR(cs)) - return PTR_ERR(cs); + cmd = MI_FLUSH_DW + 1; + + /* We always require a command barrier so that subsequent + * commands, such as breadcrumb interrupts, are strictly ordered + * wrt the contents of the write cache being flushed to memory + * (and thus being coherent from the CPU). + */ + cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; - cs = gen8_emit_pipe_control(cs, flags, - LRC_PPHWSP_SCRATCH_ADDR); - intel_ring_advance(request, cs); + if (mode & EMIT_INVALIDATE) { + cmd |= MI_INVALIDATE_TLB; + if (request->engine->class == VIDEO_DECODE_CLASS) + cmd |= MI_INVALIDATE_BSD; + } + + *cs++ = cmd; + *cs++ = LRC_PPHWSP_SCRATCH_ADDR; + *cs++ = 0; /* upper addr */ + *cs++ = 0; /* value */ + + if (aux_inv) { /* hsdes: 1809175790 */ + struct intel_engine_cs *engine; + unsigned int tmp; + + *cs++ = MI_LOAD_REGISTER_IMM(hweight8(aux_inv)); + for_each_engine_masked(engine, request->engine->gt, + aux_inv, tmp) { + *cs++ = i915_mmio_reg_offset(aux_inv_reg(engine)); + *cs++ = AUX_INV; } + *cs++ = MI_NOOP; } + intel_ring_advance(request, cs); return 0; } @@ -4169,8 +4768,7 @@ static u32 *emit_preempt_busywait(struct i915_request *request, u32 *cs) } static __always_inline u32* -gen8_emit_fini_breadcrumb_footer(struct i915_request *request, - u32 *cs) +gen8_emit_fini_breadcrumb_tail(struct i915_request *request, u32 *cs) { *cs++ = MI_USER_INTERRUPT; @@ -4184,14 +4782,16 @@ gen8_emit_fini_breadcrumb_footer(struct i915_request *request, return gen8_emit_wa_tail(request, cs); } -static u32 *gen8_emit_fini_breadcrumb(struct i915_request *request, u32 *cs) +static u32 *emit_xcs_breadcrumb(struct i915_request *request, u32 *cs) { - cs = gen8_emit_ggtt_write(cs, - request->fence.seqno, - i915_request_active_timeline(request)->hwsp_offset, - 0); + u32 addr = i915_request_active_timeline(request)->hwsp_offset; - return gen8_emit_fini_breadcrumb_footer(request, cs); + return gen8_emit_ggtt_write(cs, request->fence.seqno, addr, 0); +} + +static u32 *gen8_emit_fini_breadcrumb(struct i915_request *rq, u32 *cs) +{ + return gen8_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); } static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs) @@ -4209,7 +4809,7 @@ static u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs) PIPE_CONTROL_FLUSH_ENABLE | PIPE_CONTROL_CS_STALL); - return gen8_emit_fini_breadcrumb_footer(request, cs); + return gen8_emit_fini_breadcrumb_tail(request, cs); } static u32 * @@ -4225,7 +4825,7 @@ gen11_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs) PIPE_CONTROL_DC_FLUSH_ENABLE | PIPE_CONTROL_FLUSH_ENABLE); - return gen8_emit_fini_breadcrumb_footer(request, cs); + return gen8_emit_fini_breadcrumb_tail(request, cs); } /* @@ -4263,7 +4863,7 @@ static u32 *gen12_emit_preempt_busywait(struct i915_request *request, u32 *cs) } static __always_inline u32* -gen12_emit_fini_breadcrumb_footer(struct i915_request *request, u32 *cs) +gen12_emit_fini_breadcrumb_tail(struct i915_request *request, u32 *cs) { *cs++ = MI_USER_INTERRUPT; @@ -4277,33 +4877,29 @@ gen12_emit_fini_breadcrumb_footer(struct i915_request *request, u32 *cs) return gen8_emit_wa_tail(request, cs); } -static u32 *gen12_emit_fini_breadcrumb(struct i915_request *request, u32 *cs) +static u32 *gen12_emit_fini_breadcrumb(struct i915_request *rq, u32 *cs) { - cs = gen8_emit_ggtt_write(cs, - request->fence.seqno, - i915_request_active_timeline(request)->hwsp_offset, - 0); - - return gen12_emit_fini_breadcrumb_footer(request, cs); + return gen12_emit_fini_breadcrumb_tail(rq, emit_xcs_breadcrumb(rq, cs)); } static u32 * gen12_emit_fini_breadcrumb_rcs(struct i915_request *request, u32 *cs) { - cs = gen8_emit_ggtt_write_rcs(cs, - request->fence.seqno, - i915_request_active_timeline(request)->hwsp_offset, - PIPE_CONTROL_CS_STALL | - PIPE_CONTROL_TILE_CACHE_FLUSH | - PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | - PIPE_CONTROL_DEPTH_CACHE_FLUSH | - /* Wa_1409600907:tgl */ - PIPE_CONTROL_DEPTH_STALL | - PIPE_CONTROL_DC_FLUSH_ENABLE | - PIPE_CONTROL_FLUSH_ENABLE | - PIPE_CONTROL_HDC_PIPELINE_FLUSH); + cs = gen12_emit_ggtt_write_rcs(cs, + request->fence.seqno, + i915_request_active_timeline(request)->hwsp_offset, + PIPE_CONTROL0_HDC_PIPELINE_FLUSH, + PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_TILE_CACHE_FLUSH | + PIPE_CONTROL_FLUSH_L3 | + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | + PIPE_CONTROL_DEPTH_CACHE_FLUSH | + /* Wa_1409600907:tgl */ + PIPE_CONTROL_DEPTH_STALL | + PIPE_CONTROL_DC_FLUSH_ENABLE | + PIPE_CONTROL_FLUSH_ENABLE); - return gen12_emit_fini_breadcrumb_footer(request, cs); + return gen12_emit_fini_breadcrumb_tail(request, cs); } static void execlists_park(struct intel_engine_cs *engine) @@ -4329,8 +4925,11 @@ void intel_execlists_set_default_submission(struct intel_engine_cs *engine) engine->flags |= I915_ENGINE_SUPPORTS_STATS; if (!intel_vgpu_active(engine->i915)) { engine->flags |= I915_ENGINE_HAS_SEMAPHORES; - if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) + if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) { engine->flags |= I915_ENGINE_HAS_PREEMPTION; + if (IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION)) + engine->flags |= I915_ENGINE_HAS_TIMESLICES; + } } if (INTEL_GEN(engine->i915) >= 12) @@ -4352,6 +4951,8 @@ static void execlists_shutdown(struct intel_engine_cs *engine) static void execlists_release(struct intel_engine_cs *engine) { + engine->sanitize = NULL; /* no longer in control, nothing to sanitize */ + execlists_shutdown(engine); intel_engine_cleanup_common(engine); @@ -4371,9 +4972,10 @@ logical_ring_default_vfuncs(struct intel_engine_cs *engine) engine->emit_flush = gen8_emit_flush; engine->emit_init_breadcrumb = gen8_emit_init_breadcrumb; engine->emit_fini_breadcrumb = gen8_emit_fini_breadcrumb; - if (INTEL_GEN(engine->i915) >= 12) + if (INTEL_GEN(engine->i915) >= 12) { engine->emit_fini_breadcrumb = gen12_emit_fini_breadcrumb; - + engine->emit_flush = gen12_emit_flush; + } engine->set_default_submission = intel_execlists_set_default_submission; if (INTEL_GEN(engine->i915) < 11) { @@ -4409,6 +5011,7 @@ logical_ring_default_irqs(struct intel_engine_cs *engine) engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << shift; engine->irq_keep_mask = GT_CONTEXT_SWITCH_INTERRUPT << shift; engine->irq_keep_mask |= GT_CS_MASTER_ERROR_INTERRUPT << shift; + engine->irq_keep_mask |= GT_WAIT_SEMAPHORE_INTERRUPT << shift; } static void rcs_submission_override(struct intel_engine_cs *engine) @@ -4453,7 +5056,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) * because we only expect rare glitches but nothing * critical to prevent us from using GPU */ - DRM_ERROR("WA batch buffer initialization failed\n"); + drm_err(&i915->drm, "WA batch buffer initialization failed\n"); if (HAS_LOGICAL_RING_ELSQ(i915)) { execlists->submit_reg = uncore->regs + @@ -4476,48 +5079,18 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine) else execlists->csb_size = GEN11_CSB_ENTRIES; - reset_csb_pointers(engine); + if (INTEL_GEN(engine->i915) >= 11) { + execlists->ccid |= engine->instance << (GEN11_ENGINE_INSTANCE_SHIFT - 32); + execlists->ccid |= engine->class << (GEN11_ENGINE_CLASS_SHIFT - 32); + } /* Finally, take ownership and responsibility for cleanup! */ + engine->sanitize = execlists_sanitize; engine->release = execlists_release; return 0; } -static u32 intel_lr_indirect_ctx_offset(const struct intel_engine_cs *engine) -{ - u32 indirect_ctx_offset; - - switch (INTEL_GEN(engine->i915)) { - default: - MISSING_CASE(INTEL_GEN(engine->i915)); - /* fall through */ - case 12: - indirect_ctx_offset = - GEN12_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; - break; - case 11: - indirect_ctx_offset = - GEN11_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; - break; - case 10: - indirect_ctx_offset = - GEN10_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; - break; - case 9: - indirect_ctx_offset = - GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; - break; - case 8: - indirect_ctx_offset = - GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT; - break; - } - - return indirect_ctx_offset; -} - - static void init_common_reg_state(u32 * const regs, const struct intel_engine_cs *engine, const struct intel_ring *ring, @@ -4535,30 +5108,27 @@ static void init_common_reg_state(u32 * const regs, regs[CTX_CONTEXT_CONTROL] = ctl; regs[CTX_RING_CTL] = RING_CTL_SIZE(ring->size) | RING_VALID; + regs[CTX_TIMESTAMP] = 0; } static void init_wa_bb_reg_state(u32 * const regs, - const struct intel_engine_cs *engine, - u32 pos_bb_per_ctx) + const struct intel_engine_cs *engine) { const struct i915_ctx_workarounds * const wa_ctx = &engine->wa_ctx; if (wa_ctx->per_ctx.size) { const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); - regs[pos_bb_per_ctx] = + GEM_BUG_ON(lrc_ring_wa_bb_per_ctx(engine) == -1); + regs[lrc_ring_wa_bb_per_ctx(engine) + 1] = (ggtt_offset + wa_ctx->per_ctx.offset) | 0x01; } if (wa_ctx->indirect_ctx.size) { - const u32 ggtt_offset = i915_ggtt_offset(wa_ctx->vma); - - regs[pos_bb_per_ctx + 2] = - (ggtt_offset + wa_ctx->indirect_ctx.offset) | - (wa_ctx->indirect_ctx.size / CACHELINE_BYTES); - - regs[pos_bb_per_ctx + 4] = - intel_lr_indirect_ctx_offset(engine) << 6; + lrc_ring_setup_indirect_ctx(regs, engine, + i915_ggtt_offset(wa_ctx->vma) + + wa_ctx->indirect_ctx.offset, + wa_ctx->indirect_ctx.size); } } @@ -4607,10 +5177,7 @@ static void execlists_init_reg_state(u32 *regs, init_common_reg_state(regs, engine, ring, inhibit); init_ppgtt_reg_state(regs, vm_alias(ce->vm)); - init_wa_bb_reg_state(regs, engine, - INTEL_GEN(engine->i915) >= 12 ? - GEN12_CTX_BB_PER_CTX_PTR : - CTX_BB_PER_CTX_PTR); + init_wa_bb_reg_state(regs, engine); __reset_stop_ring(regs, engine); } @@ -4623,29 +5190,18 @@ populate_lr_context(struct intel_context *ce, { bool inhibit = true; void *vaddr; - int ret; vaddr = i915_gem_object_pin_map(ctx_obj, I915_MAP_WB); if (IS_ERR(vaddr)) { - ret = PTR_ERR(vaddr); - DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret); - return ret; + drm_dbg(&engine->i915->drm, "Could not map object pages!\n"); + return PTR_ERR(vaddr); } set_redzone(vaddr, engine); if (engine->default_state) { - void *defaults; - - defaults = i915_gem_object_pin_map(engine->default_state, - I915_MAP_WB); - if (IS_ERR(defaults)) { - ret = PTR_ERR(defaults); - goto err_unpin_ctx; - } - - memcpy(vaddr, defaults, engine->context_size); - i915_gem_object_unpin_map(engine->default_state); + shmem_read(engine->default_state, 0, + vaddr, engine->context_size); __set_bit(CONTEXT_VALID_BIT, &ce->flags); inhibit = false; } @@ -4657,14 +5213,12 @@ populate_lr_context(struct intel_context *ce, * The second page of the context object contains some registers which * must be set up prior to the first execution. */ - execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE, + execlists_init_reg_state(vaddr + LRC_STATE_OFFSET, ce, engine, ring, inhibit); - ret = 0; -err_unpin_ctx: __i915_gem_object_flush_map(ctx_obj, 0, engine->context_size); i915_gem_object_unpin_map(ctx_obj); - return ret; + return 0; } static int __execlists_context_alloc(struct intel_context *ce, @@ -4682,6 +5236,11 @@ static int __execlists_context_alloc(struct intel_context *ce, if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) context_size += I915_GTT_PAGE_SIZE; /* for redzone */ + if (INTEL_GEN(engine->i915) == 12) { + ce->wa_bb_page = context_size / PAGE_SIZE; + context_size += PAGE_SIZE; + } + ctx_obj = i915_gem_object_create_shmem(engine->i915, context_size); if (IS_ERR(ctx_obj)) return PTR_ERR(ctx_obj); @@ -4721,7 +5280,8 @@ static int __execlists_context_alloc(struct intel_context *ce, ret = populate_lr_context(ce, ctx_obj, engine, ring); if (ret) { - DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret); + drm_dbg(&engine->i915->drm, + "Failed to populate LRC: %d\n", ret); goto error_ring_free; } @@ -4774,6 +5334,8 @@ static void virtual_context_destroy(struct kref *kref) __execlists_context_fini(&ve->context); intel_context_fini(&ve->context); + intel_engine_free_request_pool(&ve->base); + kfree(ve->bonds); kfree(ve); } @@ -4873,7 +5435,7 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve) mask = rq->execution_mask; if (unlikely(!mask)) { /* Invalid selection, submit to a random engine in error */ - i915_request_skip(rq, -ENODEV); + i915_request_set_error_once(rq, -ENODEV); mask = ve->siblings[0]->mask; } @@ -4887,7 +5449,7 @@ static intel_engine_mask_t virtual_submission_mask(struct virtual_engine *ve) static void virtual_submission_tasklet(unsigned long data) { struct virtual_engine * const ve = (struct virtual_engine *)data; - const int prio = ve->base.execlists.queue_priority_hint; + const int prio = READ_ONCE(ve->base.execlists.queue_priority_hint); intel_engine_mask_t mask; unsigned int n; @@ -4898,12 +5460,15 @@ static void virtual_submission_tasklet(unsigned long data) return; local_irq_disable(); - for (n = 0; READ_ONCE(ve->request) && n < ve->num_siblings; n++) { - struct intel_engine_cs *sibling = ve->siblings[n]; + for (n = 0; n < ve->num_siblings; n++) { + struct intel_engine_cs *sibling = READ_ONCE(ve->siblings[n]); struct ve_node * const node = &ve->nodes[sibling->id]; struct rb_node **parent, *rb; bool first; + if (!READ_ONCE(ve->request)) + break; /* already handled by a sibling's tasklet */ + if (unlikely(!(mask & sibling->mask))) { if (!RB_EMPTY_NODE(&node->rb)) { spin_lock(&sibling->active.lock); @@ -4954,10 +5519,8 @@ static void virtual_submission_tasklet(unsigned long data) submit_engine: GEM_BUG_ON(RB_EMPTY_NODE(&node->rb)); node->prio = prio; - if (first && prio > sibling->execlists.queue_priority_hint) { - sibling->execlists.queue_priority_hint = prio; + if (first && prio > sibling->execlists.queue_priority_hint) tasklet_hi_schedule(&sibling->execlists.tasklet); - } spin_unlock(&sibling->active.lock); } @@ -5283,11 +5846,15 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, show_request(m, last, "\t\tE "); } - last = NULL; - count = 0; + if (execlists->switch_priority_hint != INT_MIN) + drm_printf(m, "\t\tSwitch priority hint: %d\n", + READ_ONCE(execlists->switch_priority_hint)); if (execlists->queue_priority_hint != INT_MIN) drm_printf(m, "\t\tQueue priority hint: %d\n", - execlists->queue_priority_hint); + READ_ONCE(execlists->queue_priority_hint)); + + last = NULL; + count = 0; for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { struct i915_priolist *p = rb_entry(rb, typeof(*p), node); int i; diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.h b/drivers/gpu/drm/i915/gt/intel_lrc.h index dfbc214e14f5..91fd8e452d9b 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc.h @@ -90,6 +90,7 @@ int intel_execlists_submission_setup(struct intel_engine_cs *engine); #define LRC_PPHWSP_SZ (1) /* After the PPHWSP we have the logical state for the context */ #define LRC_STATE_PN (LRC_PPHWSP_PN + LRC_PPHWSP_SZ) +#define LRC_STATE_OFFSET (LRC_STATE_PN * PAGE_SIZE) /* Space within PPHWSP reserved to be used as scratch */ #define LRC_PPHWSP_SCRATCH 0x34 diff --git a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h index d39b72590e40..93cb6c460508 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc_reg.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc_reg.h @@ -9,14 +9,13 @@ #include <linux/types.h> -/* GEN8 to GEN11 Reg State Context */ +/* GEN8 to GEN12 Reg State Context */ #define CTX_CONTEXT_CONTROL (0x02 + 1) #define CTX_RING_HEAD (0x04 + 1) #define CTX_RING_TAIL (0x06 + 1) #define CTX_RING_START (0x08 + 1) #define CTX_RING_CTL (0x0a + 1) #define CTX_BB_STATE (0x10 + 1) -#define CTX_BB_PER_CTX_PTR (0x18 + 1) #define CTX_TIMESTAMP (0x22 + 1) #define CTX_PDP3_UDW (0x24 + 1) #define CTX_PDP3_LDW (0x26 + 1) @@ -30,9 +29,6 @@ #define GEN9_CTX_RING_MI_MODE 0x54 -/* GEN12+ Reg State Context */ -#define GEN12_CTX_BB_PER_CTX_PTR (0x12 + 1) - #define ASSIGN_CTX_PDP(ppgtt, reg_state, n) do { \ u32 *reg_state__ = (reg_state); \ const u64 addr__ = i915_page_dir_dma_addr((ppgtt), (n)); \ diff --git a/drivers/gpu/drm/i915/gt/intel_rc6.c b/drivers/gpu/drm/i915/gt/intel_rc6.c index bef132709854..ab675d35030d 100644 --- a/drivers/gpu/drm/i915/gt/intel_rc6.c +++ b/drivers/gpu/drm/i915/gt/intel_rc6.c @@ -7,6 +7,7 @@ #include <linux/pm_runtime.h> #include "i915_drv.h" +#include "i915_vgpu.h" #include "intel_gt.h" #include "intel_gt_pm.h" #include "intel_rc6.h" @@ -112,7 +113,6 @@ static void gen9_rc6_enable(struct intel_rc6 *rc6) struct intel_uncore *uncore = rc6_to_uncore(rc6); struct intel_engine_cs *engine; enum intel_engine_id id; - u32 rc6_mode; /* 2b: Program RC6 thresholds.*/ if (INTEL_GEN(rc6_to_i915(rc6)) >= 10) { @@ -164,16 +164,11 @@ static void gen9_rc6_enable(struct intel_rc6 *rc6) /* 3a: Enable RC6 */ set(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ - /* WaRsUseTimeoutMode:cnl (pre-prod) */ - if (IS_CNL_REVID(rc6_to_i915(rc6), CNL_REVID_A0, CNL_REVID_C0)) - rc6_mode = GEN7_RC_CTL_TO_MODE; - else - rc6_mode = GEN6_RC_CTL_EI_MODE(1); rc6->ctl_enable = GEN6_RC_CTL_HW_ENABLE | GEN6_RC_CTL_RC6_ENABLE | - rc6_mode; + GEN6_RC_CTL_EI_MODE(1); /* * WaRsDisableCoarsePowerGating:skl,cnl @@ -245,16 +240,18 @@ static void gen6_rc6_enable(struct intel_rc6 *rc6) ret = sandybridge_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, &rc6vids, NULL); if (IS_GEN(i915, 6) && ret) { - DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n"); + drm_dbg(&i915->drm, "Couldn't check for BIOS workaround\n"); } else if (IS_GEN(i915, 6) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { - DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", - GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); + drm_dbg(&i915->drm, + "You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", + GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); rc6vids &= 0xffff00; rc6vids |= GEN6_ENCODE_RC6_VID(450); ret = sandybridge_pcode_write(i915, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); if (ret) - DRM_ERROR("Couldn't fix incorrect rc6 voltage\n"); + drm_err(&i915->drm, + "Couldn't fix incorrect rc6 voltage\n"); } } @@ -262,14 +259,15 @@ static void gen6_rc6_enable(struct intel_rc6 *rc6) static int chv_rc6_init(struct intel_rc6 *rc6) { struct intel_uncore *uncore = rc6_to_uncore(rc6); + struct drm_i915_private *i915 = rc6_to_i915(rc6); resource_size_t pctx_paddr, paddr; resource_size_t pctx_size = 32 * SZ_1K; u32 pcbr; pcbr = intel_uncore_read(uncore, VLV_PCBR); if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) { - DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n"); - paddr = rc6_to_i915(rc6)->dsm.end + 1 - pctx_size; + drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n"); + paddr = i915->dsm.end + 1 - pctx_size; GEM_BUG_ON(paddr > U32_MAX); pctx_paddr = (paddr & ~4095); @@ -303,7 +301,7 @@ static int vlv_rc6_init(struct intel_rc6 *rc6) goto out; } - DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n"); + drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n"); /* * From the Gunit register HAS: @@ -315,14 +313,15 @@ static int vlv_rc6_init(struct intel_rc6 *rc6) */ pctx = i915_gem_object_create_stolen(i915, pctx_size); if (IS_ERR(pctx)) { - DRM_DEBUG("not enough stolen space for PCTX, disabling\n"); + drm_dbg(&i915->drm, + "not enough stolen space for PCTX, disabling\n"); return PTR_ERR(pctx); } - GEM_BUG_ON(range_overflows_t(u64, - i915->dsm.start, - pctx->stolen->start, - U32_MAX)); + GEM_BUG_ON(range_overflows_end_t(u64, + i915->dsm.start, + pctx->stolen->start, + U32_MAX)); pctx_paddr = i915->dsm.start + pctx->stolen->start; intel_uncore_write(uncore, VLV_PCBR, pctx_paddr); @@ -397,14 +396,14 @@ static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6) rc_sw_target = intel_uncore_read(uncore, GEN6_RC_STATE); rc_sw_target &= RC_SW_TARGET_STATE_MASK; rc_sw_target >>= RC_SW_TARGET_STATE_SHIFT; - DRM_DEBUG_DRIVER("BIOS enabled RC states: " + drm_dbg(&i915->drm, "BIOS enabled RC states: " "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n", onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE), onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE), rc_sw_target); if (!(intel_uncore_read(uncore, RC6_LOCATION) & RC6_CTX_IN_DRAM)) { - DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n"); + drm_dbg(&i915->drm, "RC6 Base location not set properly.\n"); enable_rc6 = false; } @@ -416,7 +415,7 @@ static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6) intel_uncore_read(uncore, RC6_CTX_BASE) & RC6_CTX_BASE_MASK; if (!(rc6_ctx_base >= i915->dsm_reserved.start && rc6_ctx_base + PAGE_SIZE < i915->dsm_reserved.end)) { - DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n"); + drm_dbg(&i915->drm, "RC6 Base address not as expected.\n"); enable_rc6 = false; } @@ -424,24 +423,25 @@ static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6) (intel_uncore_read(uncore, PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1 && (intel_uncore_read(uncore, PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1 && (intel_uncore_read(uncore, PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1)) { - DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n"); + drm_dbg(&i915->drm, + "Engine Idle wait time not set properly.\n"); enable_rc6 = false; } if (!intel_uncore_read(uncore, GEN8_PUSHBUS_CONTROL) || !intel_uncore_read(uncore, GEN8_PUSHBUS_ENABLE) || !intel_uncore_read(uncore, GEN8_PUSHBUS_SHIFT)) { - DRM_DEBUG_DRIVER("Pushbus not setup properly.\n"); + drm_dbg(&i915->drm, "Pushbus not setup properly.\n"); enable_rc6 = false; } if (!intel_uncore_read(uncore, GEN6_GFXPAUSE)) { - DRM_DEBUG_DRIVER("GFX pause not setup properly.\n"); + drm_dbg(&i915->drm, "GFX pause not setup properly.\n"); enable_rc6 = false; } if (!intel_uncore_read(uncore, GEN8_MISC_CTRL0)) { - DRM_DEBUG_DRIVER("GPM control not setup properly.\n"); + drm_dbg(&i915->drm, "GPM control not setup properly.\n"); enable_rc6 = false; } @@ -462,7 +462,7 @@ static bool rc6_supported(struct intel_rc6 *rc6) return false; if (IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(rc6)) { - dev_notice(i915->drm.dev, + drm_notice(&i915->drm, "RC6 and powersaving disabled by BIOS\n"); return false; } @@ -494,7 +494,7 @@ static bool pctx_corrupted(struct intel_rc6 *rc6) if (intel_uncore_read(rc6_to_uncore(rc6), GEN8_RC6_CTX_INFO)) return false; - dev_notice(i915->drm.dev, + drm_notice(&i915->drm, "RC6 context corruption, disabling runtime power management\n"); return true; } @@ -602,6 +602,7 @@ void intel_rc6_unpark(struct intel_rc6 *rc6) void intel_rc6_park(struct intel_rc6 *rc6) { struct intel_uncore *uncore = rc6_to_uncore(rc6); + unsigned int target; if (!rc6->enabled) return; @@ -616,7 +617,14 @@ void intel_rc6_park(struct intel_rc6 *rc6) /* Turn off the HW timers and go directly to rc6 */ set(uncore, GEN6_RC_CONTROL, GEN6_RC_CTL_RC6_ENABLE); - set(uncore, GEN6_RC_STATE, 0x4 << RC_SW_TARGET_STATE_SHIFT); + + if (HAS_RC6pp(rc6_to_i915(rc6))) + target = 0x6; /* deepest rc6 */ + else if (HAS_RC6p(rc6_to_i915(rc6))) + target = 0x5; /* deep rc6 */ + else + target = 0x4; /* normal rc6 */ + set(uncore, GEN6_RC_STATE, target << RC_SW_TARGET_STATE_SHIFT); } void intel_rc6_disable(struct intel_rc6 *rc6) diff --git a/drivers/gpu/drm/i915/gt/intel_renderstate.c b/drivers/gpu/drm/i915/gt/intel_renderstate.c index 5954ecc3207f..f59e7875cc5e 100644 --- a/drivers/gpu/drm/i915/gt/intel_renderstate.c +++ b/drivers/gpu/drm/i915/gt/intel_renderstate.c @@ -102,7 +102,7 @@ static int render_state_setup(struct intel_renderstate *so, } if (rodata->reloc[reloc_index] != -1) { - DRM_ERROR("only %d relocs resolved\n", reloc_index); + drm_err(&i915->drm, "only %d relocs resolved\n", reloc_index); goto err; } @@ -194,7 +194,7 @@ int intel_renderstate_init(struct intel_renderstate *so, err = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH); if (err) - goto err_vma; + goto err_obj; err = render_state_setup(so, engine->i915); if (err) @@ -204,8 +204,6 @@ int intel_renderstate_init(struct intel_renderstate *so, err_unpin: i915_vma_unpin(so->vma); -err_vma: - i915_vma_close(so->vma); err_obj: i915_gem_object_put(obj); so->vma = NULL; @@ -221,6 +219,14 @@ int intel_renderstate_emit(struct intel_renderstate *so, if (!so->vma) return 0; + i915_vma_lock(so->vma); + err = i915_request_await_object(rq, so->vma->obj, false); + if (err == 0) + err = i915_vma_move_to_active(so->vma, rq, 0); + i915_vma_unlock(so->vma); + if (err) + return err; + err = engine->emit_bb_start(rq, so->batch_offset, so->batch_size, I915_DISPATCH_SECURE); @@ -235,13 +241,7 @@ int intel_renderstate_emit(struct intel_renderstate *so, return err; } - i915_vma_lock(so->vma); - err = i915_request_await_object(rq, so->vma->obj, false); - if (err == 0) - err = i915_vma_move_to_active(so->vma, rq, 0); - i915_vma_unlock(so->vma); - - return err; + return 0; } void intel_renderstate_fini(struct intel_renderstate *so) diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index aef6ab58d7d9..39070b514e65 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -48,8 +48,10 @@ static void engine_skip_context(struct i915_request *rq) lockdep_assert_held(&engine->active.lock); list_for_each_entry_continue(rq, &engine->active.requests, sched.link) - if (rq->context == hung_ctx) - i915_request_skip(rq, -EIO); + if (rq->context == hung_ctx) { + i915_request_set_error_once(rq, -EIO); + __i915_request_skip(rq); + } } static void client_mark_guilty(struct i915_gem_context *ctx, bool banned) @@ -86,19 +88,18 @@ static bool mark_guilty(struct i915_request *rq) bool banned; int i; + if (intel_context_is_closed(rq->context)) { + intel_context_set_banned(rq->context); + return true; + } + rcu_read_lock(); ctx = rcu_dereference(rq->context->gem_context); if (ctx && !kref_get_unless_zero(&ctx->ref)) ctx = NULL; rcu_read_unlock(); if (!ctx) - return false; - - if (i915_gem_context_is_closed(ctx)) { - intel_context_set_banned(rq->context); - banned = true; - goto out; - } + return intel_context_is_banned(rq->context); atomic_inc(&ctx->guilty_count); @@ -108,7 +109,7 @@ static bool mark_guilty(struct i915_request *rq) goto out; } - dev_notice(ctx->i915->drm.dev, + drm_notice(&ctx->i915->drm, "%s context reset due to GPU hang\n", ctx->name); @@ -154,11 +155,12 @@ void __i915_request_reset(struct i915_request *rq, bool guilty) rcu_read_lock(); /* protect the GEM context */ if (guilty) { - i915_request_skip(rq, -EIO); + i915_request_set_error_once(rq, -EIO); + __i915_request_skip(rq); if (mark_guilty(rq)) engine_skip_context(rq); } else { - dma_fence_set_error(&rq->fence, -EAGAIN); + i915_request_set_error_once(rq, -EAGAIN); mark_innocent(rq); } rcu_read_unlock(); @@ -753,7 +755,7 @@ static int gt_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask) for_each_engine(engine, gt, id) __intel_engine_reset(engine, stalled_mask & engine->mask); - i915_gem_restore_fences(gt->ggtt); + intel_ggtt_restore_fences(gt->ggtt); return err; } @@ -785,7 +787,7 @@ static void nop_submit_request(struct i915_request *request) unsigned long flags; RQ_TRACE(request, "-EIO\n"); - dma_fence_set_error(&request->fence, -EIO); + i915_request_set_error_once(request, -EIO); spin_lock_irqsave(&engine->active.lock, flags); __i915_request_submit(request); @@ -1029,7 +1031,7 @@ void intel_gt_reset(struct intel_gt *gt, goto unlock; if (reason) - dev_notice(gt->i915->drm.dev, + drm_notice(>->i915->drm, "Resetting chip for %s\n", reason); atomic_inc(>->i915->gpu_error.reset_count); @@ -1037,7 +1039,7 @@ void intel_gt_reset(struct intel_gt *gt, if (!intel_has_gpu_reset(gt)) { if (i915_modparams.reset) - dev_err(gt->i915->drm.dev, "GPU reset not supported\n"); + drm_err(>->i915->drm, "GPU reset not supported\n"); else drm_dbg(>->i915->drm, "GPU reset disabled\n"); goto error; @@ -1047,7 +1049,7 @@ void intel_gt_reset(struct intel_gt *gt, intel_runtime_pm_disable_interrupts(gt->i915); if (do_reset(gt, stalled_mask)) { - dev_err(gt->i915->drm.dev, "Failed to reset chip\n"); + drm_err(>->i915->drm, "Failed to reset chip\n"); goto taint; } @@ -1109,7 +1111,7 @@ static inline int intel_gt_reset_engine(struct intel_engine_cs *engine) /** * intel_engine_reset - reset GPU engine to recover from a hang * @engine: engine to reset - * @msg: reason for GPU reset; or NULL for no dev_notice() + * @msg: reason for GPU reset; or NULL for no drm_notice() * * Reset a specific GPU engine. Useful if a hang is detected. * Returns zero on successful reset or otherwise an error code. @@ -1134,7 +1136,7 @@ int intel_engine_reset(struct intel_engine_cs *engine, const char *msg) reset_prepare_engine(engine); if (msg) - dev_notice(engine->i915->drm.dev, + drm_notice(&engine->i915->drm, "Resetting %s for %s\n", engine->name, msg); atomic_inc(&engine->i915->gpu_error.reset_engine_count[engine->uabi_class]); @@ -1379,7 +1381,7 @@ static void intel_wedge_me(struct work_struct *work) { struct intel_wedge_me *w = container_of(work, typeof(*w), work.work); - dev_err(w->gt->i915->drm.dev, + drm_err(&w->gt->i915->drm, "%s timed out, cancelling all in-flight rendering.\n", w->name); intel_gt_set_wedged(w->gt); diff --git a/drivers/gpu/drm/i915/gt/intel_ring.h b/drivers/gpu/drm/i915/gt/intel_ring.h index 5bdce24994aa..cc0ebca65167 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring.h +++ b/drivers/gpu/drm/i915/gt/intel_ring.h @@ -88,6 +88,8 @@ static inline u32 intel_ring_offset(const struct i915_request *rq, void *addr) static inline void assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail) { + unsigned int head = READ_ONCE(ring->head); + GEM_BUG_ON(!intel_ring_offset_valid(ring, tail)); /* @@ -105,8 +107,7 @@ assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail) * into the same cacheline as ring->head. */ #define cacheline(a) round_down(a, CACHELINE_BYTES) - GEM_BUG_ON(cacheline(tail) == cacheline(ring->head) && - tail < ring->head); + GEM_BUG_ON(cacheline(tail) == cacheline(head) && tail < head); #undef cacheline } diff --git a/drivers/gpu/drm/i915/gt/intel_ring_submission.c b/drivers/gpu/drm/i915/gt/intel_ring_submission.c index f70b903a98bc..ca7286e58409 100644 --- a/drivers/gpu/drm/i915/gt/intel_ring_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_ring_submission.c @@ -29,11 +29,10 @@ #include <linux/log2.h> -#include <drm/i915_drm.h> - #include "gem/i915_gem_context.h" #include "gen6_ppgtt.h" +#include "gen7_renderclear.h" #include "i915_drv.h" #include "i915_trace.h" #include "intel_context.h" @@ -43,6 +42,7 @@ #include "intel_reset.h" #include "intel_ring.h" #include "intel_workarounds.h" +#include "shmem_utils.h" /* Rough estimate of the typical request size, performing a flush, * set-context and then emitting the batch. @@ -578,8 +578,9 @@ static void flush_cs_tlb(struct intel_engine_cs *engine) RING_INSTPM(engine->mmio_base), INSTPM_SYNC_FLUSH, 0, 1000)) - DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", - engine->name); + drm_err(&dev_priv->drm, + "%s: wait for SyncFlush to complete for TLB invalidation timed out\n", + engine->name); } static void ring_setup_status_page(struct intel_engine_cs *engine) @@ -602,8 +603,9 @@ static bool stop_ring(struct intel_engine_cs *engine) MODE_IDLE, MODE_IDLE, 1000)) { - DRM_ERROR("%s : timed out trying to stop ring\n", - engine->name); + drm_err(&dev_priv->drm, + "%s : timed out trying to stop ring\n", + engine->name); /* * Sometimes we observe that the idle flag is not @@ -662,22 +664,23 @@ static int xcs_resume(struct intel_engine_cs *engine) /* WaClearRingBufHeadRegAtInit:ctg,elk */ if (!stop_ring(engine)) { /* G45 ring initialization often fails to reset head to zero */ - DRM_DEBUG_DRIVER("%s head not reset to zero " + drm_dbg(&dev_priv->drm, "%s head not reset to zero " + "ctl %08x head %08x tail %08x start %08x\n", + engine->name, + ENGINE_READ(engine, RING_CTL), + ENGINE_READ(engine, RING_HEAD), + ENGINE_READ(engine, RING_TAIL), + ENGINE_READ(engine, RING_START)); + + if (!stop_ring(engine)) { + drm_err(&dev_priv->drm, + "failed to set %s head to zero " "ctl %08x head %08x tail %08x start %08x\n", engine->name, ENGINE_READ(engine, RING_CTL), ENGINE_READ(engine, RING_HEAD), ENGINE_READ(engine, RING_TAIL), ENGINE_READ(engine, RING_START)); - - if (!stop_ring(engine)) { - DRM_ERROR("failed to set %s head to zero " - "ctl %08x head %08x tail %08x start %08x\n", - engine->name, - ENGINE_READ(engine, RING_CTL), - ENGINE_READ(engine, RING_HEAD), - ENGINE_READ(engine, RING_TAIL), - ENGINE_READ(engine, RING_START)); ret = -EIO; goto out; } @@ -720,7 +723,7 @@ static int xcs_resume(struct intel_engine_cs *engine) RING_CTL(engine->mmio_base), RING_VALID, RING_VALID, 50)) { - DRM_ERROR("%s initialization failed " + drm_err(&dev_priv->drm, "%s initialization failed " "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", engine->name, ENGINE_READ(engine, RING_CTL), @@ -897,9 +900,7 @@ static void reset_cancel(struct intel_engine_cs *engine) /* Mark all submitted requests as skipped. */ list_for_each_entry(request, &engine->active.requests, sched.link) { - if (!i915_request_signaled(request)) - dma_fence_set_error(&request->fence, -EIO); - + i915_request_set_error_once(request, -EIO); i915_request_mark_complete(request); } @@ -1241,7 +1242,7 @@ alloc_context_vma(struct intel_engine_cs *engine) i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); if (engine->default_state) { - void *defaults, *vaddr; + void *vaddr; vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); if (IS_ERR(vaddr)) { @@ -1249,15 +1250,8 @@ alloc_context_vma(struct intel_engine_cs *engine) goto err_obj; } - defaults = i915_gem_object_pin_map(engine->default_state, - I915_MAP_WB); - if (IS_ERR(defaults)) { - err = PTR_ERR(defaults); - goto err_map; - } - - memcpy(vaddr, defaults, engine->context_size); - i915_gem_object_unpin_map(engine->default_state); + shmem_read(engine->default_state, 0, + vaddr, engine->context_size); i915_gem_object_flush_map(obj); i915_gem_object_unpin_map(obj); @@ -1271,8 +1265,6 @@ alloc_context_vma(struct intel_engine_cs *engine) return vma; -err_map: - i915_gem_object_unpin_map(obj); err_obj: i915_gem_object_put(obj); return ERR_PTR(err); @@ -1360,7 +1352,9 @@ static int load_pd_dir(struct i915_request *rq, return rq->engine->emit_flush(rq, EMIT_FLUSH); } -static inline int mi_set_context(struct i915_request *rq, u32 flags) +static inline int mi_set_context(struct i915_request *rq, + struct intel_context *ce, + u32 flags) { struct drm_i915_private *i915 = rq->i915; struct intel_engine_cs *engine = rq->engine; @@ -1435,7 +1429,7 @@ static inline int mi_set_context(struct i915_request *rq, u32 flags) *cs++ = MI_NOOP; *cs++ = MI_SET_CONTEXT; - *cs++ = i915_ggtt_offset(rq->context->state) | flags; + *cs++ = i915_ggtt_offset(ce->state) | flags; /* * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP * WaMiSetContext_Hang:snb,ivb,vlv @@ -1550,13 +1544,56 @@ static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) return rq->engine->emit_flush(rq, EMIT_INVALIDATE); } +static int clear_residuals(struct i915_request *rq) +{ + struct intel_engine_cs *engine = rq->engine; + int ret; + + ret = switch_mm(rq, vm_alias(engine->kernel_context->vm)); + if (ret) + return ret; + + if (engine->kernel_context->state) { + ret = mi_set_context(rq, + engine->kernel_context, + MI_MM_SPACE_GTT | MI_RESTORE_INHIBIT); + if (ret) + return ret; + } + + ret = engine->emit_bb_start(rq, + engine->wa_ctx.vma->node.start, 0, + 0); + if (ret) + return ret; + + ret = engine->emit_flush(rq, EMIT_FLUSH); + if (ret) + return ret; + + /* Always invalidate before the next switch_mm() */ + return engine->emit_flush(rq, EMIT_INVALIDATE); +} + static int switch_context(struct i915_request *rq) { + struct intel_engine_cs *engine = rq->engine; struct intel_context *ce = rq->context; + void **residuals = NULL; int ret; GEM_BUG_ON(HAS_EXECLISTS(rq->i915)); + if (engine->wa_ctx.vma && ce != engine->kernel_context) { + if (engine->wa_ctx.vma->private != ce) { + ret = clear_residuals(rq); + if (ret) + return ret; + + residuals = &engine->wa_ctx.vma->private; + } + } + ret = switch_mm(rq, vm_alias(ce->vm)); if (ret) return ret; @@ -1564,7 +1601,7 @@ static int switch_context(struct i915_request *rq) if (ce->state) { u32 flags; - GEM_BUG_ON(rq->engine->id != RCS0); + GEM_BUG_ON(engine->id != RCS0); /* For resource streamer on HSW+ and power context elsewhere */ BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN); @@ -1576,7 +1613,7 @@ static int switch_context(struct i915_request *rq) else flags |= MI_RESTORE_INHIBIT; - ret = mi_set_context(rq, flags); + ret = mi_set_context(rq, ce, flags); if (ret) return ret; } @@ -1585,6 +1622,20 @@ static int switch_context(struct i915_request *rq) if (ret) return ret; + /* + * Now past the point of no return, this request _will_ be emitted. + * + * Or at least this preamble will be emitted, the request may be + * interrupted prior to submitting the user payload. If so, we + * still submit the "empty" request in order to preserve global + * state tracking such as this, our tracking of the current + * dirty context. + */ + if (residuals) { + intel_context_put(*residuals); + *residuals = intel_context_get(ce); + } + return 0; } @@ -1769,6 +1820,11 @@ static void ring_release(struct intel_engine_cs *engine) intel_engine_cleanup_common(engine); + if (engine->wa_ctx.vma) { + intel_context_put(engine->wa_ctx.vma->private); + i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0); + } + intel_ring_unpin(engine->legacy.ring); intel_ring_put(engine->legacy.ring); @@ -1916,6 +1972,64 @@ static void setup_vecs(struct intel_engine_cs *engine) engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; } +static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, + struct i915_vma * const vma) +{ + return gen7_setup_clear_gpr_bb(engine, vma); +} + +static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + int size; + int err; + + size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */); + if (size <= 0) + return size; + + size = ALIGN(size, PAGE_SIZE); + obj = i915_gem_object_create_internal(engine->i915, size); + if (IS_ERR(obj)) + return PTR_ERR(obj); + + vma = i915_vma_instance(obj, engine->gt->vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto err_obj; + } + + vma->private = intel_context_create(engine); /* dummy residuals */ + if (IS_ERR(vma->private)) { + err = PTR_ERR(vma->private); + goto err_obj; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH); + if (err) + goto err_private; + + err = i915_vma_sync(vma); + if (err) + goto err_unpin; + + err = gen7_ctx_switch_bb_setup(engine, vma); + if (err) + goto err_unpin; + + engine->wa_ctx.vma = vma; + return 0; + +err_unpin: + i915_vma_unpin(vma); +err_private: + intel_context_put(vma->private); +err_obj: + i915_gem_object_put(obj); + return err; +} + int intel_ring_submission_setup(struct intel_engine_cs *engine) { struct intel_timeline *timeline; @@ -1969,11 +2083,19 @@ int intel_ring_submission_setup(struct intel_engine_cs *engine) GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); + if (IS_HASWELL(engine->i915) && engine->class == RENDER_CLASS) { + err = gen7_ctx_switch_bb_init(engine); + if (err) + goto err_ring_unpin; + } + /* Finally, take ownership and responsibility for cleanup! */ engine->release = ring_release; return 0; +err_ring_unpin: + intel_ring_unpin(ring); err_ring: intel_ring_put(ring); err_timeline_unpin: @@ -1984,3 +2106,7 @@ err: intel_engine_cleanup_common(engine); return err; } + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftest_ring_submission.c" +#endif diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 30ae29b30f11..2f59fc6df3c2 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -4,14 +4,19 @@ * Copyright © 2019 Intel Corporation */ +#include <drm/i915_drm.h> + #include "i915_drv.h" #include "intel_gt.h" +#include "intel_gt_clock_utils.h" #include "intel_gt_irq.h" #include "intel_gt_pm_irq.h" #include "intel_rps.h" #include "intel_sideband.h" #include "../../../platform/x86/intel_ips.h" +#define BUSY_MAX_EI 20u /* ms */ + /* * Lock protecting IPS related data structures */ @@ -42,6 +47,100 @@ static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) intel_uncore_write_fw(uncore, reg, val); } +static void rps_timer(struct timer_list *t) +{ + struct intel_rps *rps = from_timer(rps, t, timer); + struct intel_engine_cs *engine; + enum intel_engine_id id; + s64 max_busy[3] = {}; + ktime_t dt, last; + + for_each_engine(engine, rps_to_gt(rps), id) { + s64 busy; + int i; + + dt = intel_engine_get_busy_time(engine); + last = engine->stats.rps; + engine->stats.rps = dt; + + busy = ktime_to_ns(ktime_sub(dt, last)); + for (i = 0; i < ARRAY_SIZE(max_busy); i++) { + if (busy > max_busy[i]) + swap(busy, max_busy[i]); + } + } + + dt = ktime_get(); + last = rps->pm_timestamp; + rps->pm_timestamp = dt; + + if (intel_rps_is_active(rps)) { + s64 busy; + int i; + + dt = ktime_sub(dt, last); + + /* + * Our goal is to evaluate each engine independently, so we run + * at the lowest clocks required to sustain the heaviest + * workload. However, a task may be split into sequential + * dependent operations across a set of engines, such that + * the independent contributions do not account for high load, + * but overall the task is GPU bound. For example, consider + * video decode on vcs followed by colour post-processing + * on vecs, followed by general post-processing on rcs. + * Since multi-engines being active does imply a single + * continuous workload across all engines, we hedge our + * bets by only contributing a factor of the distributed + * load into our busyness calculation. + */ + busy = max_busy[0]; + for (i = 1; i < ARRAY_SIZE(max_busy); i++) { + if (!max_busy[i]) + break; + + busy += div_u64(max_busy[i], 1 << i); + } + GT_TRACE(rps_to_gt(rps), + "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n", + busy, (int)div64_u64(100 * busy, dt), + max_busy[0], max_busy[1], max_busy[2], + rps->pm_interval); + + if (100 * busy > rps->power.up_threshold * dt && + rps->cur_freq < rps->max_freq_softlimit) { + rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD; + rps->pm_interval = 1; + schedule_work(&rps->work); + } else if (100 * busy < rps->power.down_threshold * dt && + rps->cur_freq > rps->min_freq_softlimit) { + rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD; + rps->pm_interval = 1; + schedule_work(&rps->work); + } else { + rps->last_adj = 0; + } + + mod_timer(&rps->timer, + jiffies + msecs_to_jiffies(rps->pm_interval)); + rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI); + } +} + +static void rps_start_timer(struct intel_rps *rps) +{ + rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); + rps->pm_interval = 1; + mod_timer(&rps->timer, jiffies + 1); +} + +static void rps_stop_timer(struct intel_rps *rps) +{ + del_timer_sync(&rps->timer); + rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp); + cancel_work_sync(&rps->work); +} + static u32 rps_pm_mask(struct intel_rps *rps, u8 val) { u32 mask = 0; @@ -69,21 +168,17 @@ static void rps_enable_interrupts(struct intel_rps *rps) { struct intel_gt *gt = rps_to_gt(rps); - rps_reset_ei(rps); + GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n", + rps->pm_events, rps_pm_mask(rps, rps->last_freq)); - if (IS_VALLEYVIEW(gt->i915)) - /* WaGsvRC0ResidencyMethod:vlv */ - rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; - else - rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | - GEN6_PM_RP_DOWN_THRESHOLD | - GEN6_PM_RP_DOWN_TIMEOUT); + rps_reset_ei(rps); spin_lock_irq(>->irq_lock); gen6_gt_pm_enable_irq(gt, rps->pm_events); spin_unlock_irq(>->irq_lock); - set(gt->uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, rps->cur_freq)); + intel_uncore_write(gt->uncore, + GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); } static void gen6_rps_reset_interrupts(struct intel_rps *rps) @@ -115,9 +210,8 @@ static void rps_disable_interrupts(struct intel_rps *rps) { struct intel_gt *gt = rps_to_gt(rps); - rps->pm_events = 0; - - set(gt->uncore, GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); + intel_uncore_write(gt->uncore, + GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); spin_lock_irq(>->irq_lock); gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); @@ -134,6 +228,7 @@ static void rps_disable_interrupts(struct intel_rps *rps) cancel_work_sync(&rps->work); rps_reset_interrupts(rps); + GT_TRACE(gt, "interrupts:off\n"); } static const struct cparams { @@ -180,14 +275,12 @@ static void gen5_rps_init(struct intel_rps *rps) fmin = (rgvmodectl & MEMMODE_FMIN_MASK); fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT; - DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", - fmax, fmin, fstart); + drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n", + fmax, fmin, fstart); rps->min_freq = fmax; + rps->efficient_freq = fstart; rps->max_freq = fmin; - - rps->idle_freq = rps->min_freq; - rps->cur_freq = rps->idle_freq; } static unsigned long @@ -450,7 +543,8 @@ static bool gen5_rps_enable(struct intel_rps *rps) if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10)) - DRM_ERROR("stuck trying to change perf mode\n"); + drm_err(&uncore->i915->drm, + "stuck trying to change perf mode\n"); mdelay(1); gen5_rps_set(rps, rps->cur_freq); @@ -527,8 +621,8 @@ static u32 rps_limits(struct intel_rps *rps, u8 val) static void rps_set_power(struct intel_rps *rps, int new_power) { - struct intel_uncore *uncore = rps_to_uncore(rps); - struct drm_i915_private *i915 = rps_to_i915(rps); + struct intel_gt *gt = rps_to_gt(rps); + struct intel_uncore *uncore = gt->uncore; u32 threshold_up = 0, threshold_down = 0; /* in % */ u32 ei_up = 0, ei_down = 0; @@ -537,55 +631,49 @@ static void rps_set_power(struct intel_rps *rps, int new_power) if (new_power == rps->power.mode) return; + threshold_up = 95; + threshold_down = 85; + /* Note the units here are not exactly 1us, but 1280ns. */ switch (new_power) { case LOW_POWER: - /* Upclock if more than 95% busy over 16ms */ ei_up = 16000; - threshold_up = 95; - - /* Downclock if less than 85% busy over 32ms */ ei_down = 32000; - threshold_down = 85; break; case BETWEEN: - /* Upclock if more than 90% busy over 13ms */ ei_up = 13000; - threshold_up = 90; - - /* Downclock if less than 75% busy over 32ms */ ei_down = 32000; - threshold_down = 75; break; case HIGH_POWER: - /* Upclock if more than 85% busy over 10ms */ ei_up = 10000; - threshold_up = 85; - - /* Downclock if less than 60% busy over 32ms */ ei_down = 32000; - threshold_down = 60; break; } /* When byt can survive without system hang with dynamic * sw freq adjustments, this restriction can be lifted. */ - if (IS_VALLEYVIEW(i915)) + if (IS_VALLEYVIEW(gt->i915)) goto skip_hw_write; - set(uncore, GEN6_RP_UP_EI, GT_INTERVAL_FROM_US(i915, ei_up)); + GT_TRACE(gt, + "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n", + new_power, threshold_up, ei_up, threshold_down, ei_down); + + set(uncore, GEN6_RP_UP_EI, + intel_gt_ns_to_pm_interval(gt, ei_up * 1000)); set(uncore, GEN6_RP_UP_THRESHOLD, - GT_INTERVAL_FROM_US(i915, ei_up * threshold_up / 100)); + intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10)); - set(uncore, GEN6_RP_DOWN_EI, GT_INTERVAL_FROM_US(i915, ei_down)); + set(uncore, GEN6_RP_DOWN_EI, + intel_gt_ns_to_pm_interval(gt, ei_down * 1000)); set(uncore, GEN6_RP_DOWN_THRESHOLD, - GT_INTERVAL_FROM_US(i915, ei_down * threshold_down / 100)); + intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10)); set(uncore, GEN6_RP_CONTROL, - (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | + (INTEL_GEN(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | GEN6_RP_MEDIA_HW_NORMAL_MODE | GEN6_RP_MEDIA_IS_GFX | GEN6_RP_ENABLE | @@ -640,9 +728,11 @@ static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) { + GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n", yesno(interactive)); + mutex_lock(&rps->power.mutex); if (interactive) { - if (!rps->power.interactive++ && rps->active) + if (!rps->power.interactive++ && intel_rps_is_active(rps)) rps_set_power(rps, HIGH_POWER); } else { GEM_BUG_ON(!rps->power.interactive); @@ -667,6 +757,9 @@ static int gen6_rps_set(struct intel_rps *rps, u8 val) GEN6_AGGRESSIVE_TURBO); set(uncore, GEN6_RPNSWREQ, swreq); + GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n", + val, intel_gpu_freq(rps, val), swreq); + return 0; } @@ -679,6 +772,9 @@ static int vlv_rps_set(struct intel_rps *rps, u8 val) err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); vlv_punit_put(i915); + GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n", + val, intel_gpu_freq(rps, val)); + return err; } @@ -709,25 +805,30 @@ static int rps_set(struct intel_rps *rps, u8 val, bool update) void intel_rps_unpark(struct intel_rps *rps) { - u8 freq; - - if (!rps->enabled) + if (!intel_rps_is_enabled(rps)) return; + GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq); + /* * Use the user's desired frequency as a guide, but for better * performance, jump directly to RPe as our starting frequency. */ mutex_lock(&rps->lock); - rps->active = true; - freq = max(rps->cur_freq, rps->efficient_freq), - freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit); - intel_rps_set(rps, freq); - rps->last_adj = 0; + + intel_rps_set_active(rps); + intel_rps_set(rps, + clamp(rps->cur_freq, + rps->min_freq_softlimit, + rps->max_freq_softlimit)); + mutex_unlock(&rps->lock); - if (INTEL_GEN(rps_to_i915(rps)) >= 6) + rps->pm_iir = 0; + if (intel_rps_has_interrupts(rps)) rps_enable_interrupts(rps); + if (intel_rps_uses_timer(rps)) + rps_start_timer(rps); if (IS_GEN(rps_to_i915(rps), 5)) gen5_rps_update(rps); @@ -735,15 +836,16 @@ void intel_rps_unpark(struct intel_rps *rps) void intel_rps_park(struct intel_rps *rps) { - struct drm_i915_private *i915 = rps_to_i915(rps); + int adj; - if (!rps->enabled) + if (!intel_rps_clear_active(rps)) return; - if (INTEL_GEN(i915) >= 6) + if (intel_rps_uses_timer(rps)) + rps_stop_timer(rps); + if (intel_rps_has_interrupts(rps)) rps_disable_interrupts(rps); - rps->active = false; if (rps->last_freq <= rps->idle_freq) return; @@ -763,14 +865,34 @@ void intel_rps_park(struct intel_rps *rps) intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); rps_set(rps, rps->idle_freq, false); intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); + + /* + * Since we will try and restart from the previously requested + * frequency on unparking, treat this idle point as a downclock + * interrupt and reduce the frequency for resume. If we park/unpark + * more frequently than the rps worker can run, we will not respond + * to any EI and never see a change in frequency. + * + * (Note we accommodate Cherryview's limitation of only using an + * even bin by applying it to all.) + */ + adj = rps->last_adj; + if (adj < 0) + adj *= 2; + else /* CHV needs even encode values */ + adj = -2; + rps->last_adj = adj; + rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq); + + GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq); } void intel_rps_boost(struct i915_request *rq) { - struct intel_rps *rps = &rq->engine->gt->rps; + struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; unsigned long flags; - if (i915_request_signaled(rq) || !rps->active) + if (i915_request_signaled(rq) || !intel_rps_is_active(rps)) return; /* Serializes with i915_request_retire() */ @@ -779,6 +901,9 @@ void intel_rps_boost(struct i915_request *rq) !dma_fence_is_signaled_locked(&rq->fence)) { set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags); + GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n", + rq->fence.context, rq->fence.seqno); + if (!atomic_fetch_inc(&rps->num_waiters) && READ_ONCE(rps->cur_freq) < rps->boost_freq) schedule_work(&rps->work); @@ -796,7 +921,7 @@ int intel_rps_set(struct intel_rps *rps, u8 val) GEM_BUG_ON(val > rps->max_freq); GEM_BUG_ON(val < rps->min_freq); - if (rps->active) { + if (intel_rps_is_active(rps)) { err = rps_set(rps, val, true); if (err) return err; @@ -805,7 +930,7 @@ int intel_rps_set(struct intel_rps *rps, u8 val) * Make sure we continue to get interrupts * until we hit the minimum or maximum frequencies. */ - if (INTEL_GEN(rps_to_i915(rps)) >= 6) { + if (intel_rps_has_interrupts(rps)) { struct intel_uncore *uncore = rps_to_uncore(rps); set(uncore, @@ -873,12 +998,14 @@ static void gen6_rps_init(struct intel_rps *rps) static bool rps_reset(struct intel_rps *rps) { + struct drm_i915_private *i915 = rps_to_i915(rps); + /* force a reset */ rps->power.mode = -1; rps->last_freq = -1; if (rps_set(rps, rps->min_freq, true)) { - DRM_ERROR("Failed to reset RPS to initial values\n"); + drm_err(&i915->drm, "Failed to reset RPS to initial values\n"); return false; } @@ -889,20 +1016,18 @@ static bool rps_reset(struct intel_rps *rps) /* See the Gen9_GT_PM_Programming_Guide doc for the below */ static bool gen9_rps_enable(struct intel_rps *rps) { - struct drm_i915_private *i915 = rps_to_i915(rps); - struct intel_uncore *uncore = rps_to_uncore(rps); + struct intel_gt *gt = rps_to_gt(rps); + struct intel_uncore *uncore = gt->uncore; /* Program defaults and thresholds for RPS */ - if (IS_GEN(i915, 9)) + if (IS_GEN(gt->i915, 9)) intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, GEN9_FREQUENCY(rps->rp1_freq)); - /* 1 second timeout */ - intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, - GT_INTERVAL_FROM_US(i915, 1000000)); - intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); + rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; + return rps_reset(rps); } @@ -913,12 +1038,10 @@ static bool gen8_rps_enable(struct intel_rps *rps) intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(rps->rp1_freq)); - /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ - intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, - 100000000 / 128); /* 1 second timeout */ - intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); + rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD; + return rps_reset(rps); } @@ -930,6 +1053,10 @@ static bool gen6_rps_enable(struct intel_rps *rps) intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); + rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | + GEN6_PM_RP_DOWN_THRESHOLD | + GEN6_PM_RP_DOWN_TIMEOUT); + return rps_reset(rps); } @@ -1015,6 +1142,10 @@ static bool chv_rps_enable(struct intel_rps *rps) GEN6_RP_UP_BUSY_AVG | GEN6_RP_DOWN_IDLE_AVG); + rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD | + GEN6_PM_RP_DOWN_THRESHOLD | + GEN6_PM_RP_DOWN_TIMEOUT); + /* Setting Fixed Bias */ vlv_punit_get(i915); @@ -1029,8 +1160,8 @@ static bool chv_rps_enable(struct intel_rps *rps) drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, "GPLL not enabled\n"); - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); + drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); + drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); return rps_reset(rps); } @@ -1113,6 +1244,9 @@ static bool vlv_rps_enable(struct intel_rps *rps) GEN6_RP_UP_BUSY_AVG | GEN6_RP_DOWN_IDLE_CONT); + /* WaGsvRC0ResidencyMethod:vlv */ + rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED; + vlv_punit_get(i915); /* Setting Fixed Bias */ @@ -1127,8 +1261,8 @@ static bool vlv_rps_enable(struct intel_rps *rps) drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, "GPLL not enabled\n"); - DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); - DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); + drm_dbg(&i915->drm, "GPLL enabled? %s\n", yesno(val & GPLLENABLE)); + drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val); return rps_reset(rps); } @@ -1171,33 +1305,71 @@ static unsigned long __ips_gfx_val(struct intel_ips *ips) return ips->gfx_power + state2; } +static bool has_busy_stats(struct intel_rps *rps) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, rps_to_gt(rps), id) { + if (!intel_engine_supports_stats(engine)) + return false; + } + + return true; +} + void intel_rps_enable(struct intel_rps *rps) { struct drm_i915_private *i915 = rps_to_i915(rps); struct intel_uncore *uncore = rps_to_uncore(rps); + bool enabled = false; + + if (!HAS_RPS(i915)) + return; + + intel_gt_check_clock_frequency(rps_to_gt(rps)); intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); - if (IS_CHERRYVIEW(i915)) - rps->enabled = chv_rps_enable(rps); + if (rps->max_freq <= rps->min_freq) + /* leave disabled, no room for dynamic reclocking */; + else if (IS_CHERRYVIEW(i915)) + enabled = chv_rps_enable(rps); else if (IS_VALLEYVIEW(i915)) - rps->enabled = vlv_rps_enable(rps); + enabled = vlv_rps_enable(rps); else if (INTEL_GEN(i915) >= 9) - rps->enabled = gen9_rps_enable(rps); + enabled = gen9_rps_enable(rps); else if (INTEL_GEN(i915) >= 8) - rps->enabled = gen8_rps_enable(rps); + enabled = gen8_rps_enable(rps); else if (INTEL_GEN(i915) >= 6) - rps->enabled = gen6_rps_enable(rps); + enabled = gen6_rps_enable(rps); else if (IS_IRONLAKE_M(i915)) - rps->enabled = gen5_rps_enable(rps); + enabled = gen5_rps_enable(rps); + else + MISSING_CASE(INTEL_GEN(i915)); intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); - if (!rps->enabled) + if (!enabled) return; - drm_WARN_ON(&i915->drm, rps->max_freq < rps->min_freq); - drm_WARN_ON(&i915->drm, rps->idle_freq > rps->max_freq); + GT_TRACE(rps_to_gt(rps), + "min:%x, max:%x, freq:[%d, %d]\n", + rps->min_freq, rps->max_freq, + intel_gpu_freq(rps, rps->min_freq), + intel_gpu_freq(rps, rps->max_freq)); + + GEM_BUG_ON(rps->max_freq < rps->min_freq); + GEM_BUG_ON(rps->idle_freq > rps->max_freq); + + GEM_BUG_ON(rps->efficient_freq < rps->min_freq); + GEM_BUG_ON(rps->efficient_freq > rps->max_freq); - drm_WARN_ON(&i915->drm, rps->efficient_freq < rps->min_freq); - drm_WARN_ON(&i915->drm, rps->efficient_freq > rps->max_freq); + if (has_busy_stats(rps)) + intel_rps_set_timer(rps); + else if (INTEL_GEN(i915) >= 6) + intel_rps_set_interrupts(rps); + else + /* Ironlake currently uses intel_ips.ko */ {} + + intel_rps_set_enabled(rps); } static void gen6_rps_disable(struct intel_rps *rps) @@ -1209,7 +1381,9 @@ void intel_rps_disable(struct intel_rps *rps) { struct drm_i915_private *i915 = rps_to_i915(rps); - rps->enabled = false; + intel_rps_clear_enabled(rps); + intel_rps_clear_interrupts(rps); + intel_rps_clear_timer(rps); if (INTEL_GEN(i915) >= 6) gen6_rps_disable(rps); @@ -1285,7 +1459,8 @@ static void vlv_init_gpll_ref_freq(struct intel_rps *rps) CCK_GPLL_CLOCK_CONTROL, i915->czclk_freq); - DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq); + drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n", + rps->gpll_ref_freq); } static void vlv_rps_init(struct intel_rps *rps) @@ -1313,28 +1488,24 @@ static void vlv_rps_init(struct intel_rps *rps) i915->mem_freq = 1333; break; } - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); + drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); rps->max_freq = vlv_rps_max_freq(rps); rps->rp0_freq = rps->max_freq; - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->max_freq), - rps->max_freq); + drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->max_freq), rps->max_freq); rps->efficient_freq = vlv_rps_rpe_freq(rps); - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->efficient_freq), - rps->efficient_freq); + drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); rps->rp1_freq = vlv_rps_guar_freq(rps); - DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->rp1_freq), - rps->rp1_freq); + drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); rps->min_freq = vlv_rps_min_freq(rps); - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->min_freq), - rps->min_freq); + drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->min_freq), rps->min_freq); vlv_iosf_sb_put(i915, BIT(VLV_IOSF_SB_PUNIT) | @@ -1364,28 +1535,24 @@ static void chv_rps_init(struct intel_rps *rps) i915->mem_freq = 1600; break; } - DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); + drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq); rps->max_freq = chv_rps_max_freq(rps); rps->rp0_freq = rps->max_freq; - DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->max_freq), - rps->max_freq); + drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->max_freq), rps->max_freq); rps->efficient_freq = chv_rps_rpe_freq(rps); - DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->efficient_freq), - rps->efficient_freq); + drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq); rps->rp1_freq = chv_rps_guar_freq(rps); - DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->rp1_freq), - rps->rp1_freq); + drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq); rps->min_freq = chv_rps_min_freq(rps); - DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", - intel_gpu_freq(rps, rps->min_freq), - rps->min_freq); + drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n", + intel_gpu_freq(rps, rps->min_freq), rps->min_freq); vlv_iosf_sb_put(i915, BIT(VLV_IOSF_SB_PUNIT) | @@ -1448,20 +1615,25 @@ static void rps_work(struct work_struct *work) { struct intel_rps *rps = container_of(work, typeof(*rps), work); struct intel_gt *gt = rps_to_gt(rps); + struct drm_i915_private *i915 = rps_to_i915(rps); bool client_boost = false; int new_freq, adj, min, max; u32 pm_iir = 0; spin_lock_irq(>->irq_lock); - pm_iir = fetch_and_zero(&rps->pm_iir); + pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events; client_boost = atomic_read(&rps->num_waiters); spin_unlock_irq(>->irq_lock); /* Make sure we didn't queue anything we're not going to process. */ - if ((pm_iir & rps->pm_events) == 0 && !client_boost) + if (!pm_iir && !client_boost) goto out; mutex_lock(&rps->lock); + if (!intel_rps_is_active(rps)) { + mutex_unlock(&rps->lock); + return; + } pm_iir |= vlv_wa_c0_ei(rps, pm_iir); @@ -1471,6 +1643,12 @@ static void rps_work(struct work_struct *work) max = rps->max_freq_softlimit; if (client_boost) max = rps->max_freq; + + GT_TRACE(gt, + "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n", + pm_iir, yesno(client_boost), + adj, new_freq, min, max); + if (client_boost && new_freq < rps->boost_freq) { new_freq = rps->boost_freq; adj = 0; @@ -1502,30 +1680,18 @@ static void rps_work(struct work_struct *work) adj = 0; } - rps->last_adj = adj; - /* - * Limit deboosting and boosting to keep ourselves at the extremes - * when in the respective power modes (i.e. slowly decrease frequencies - * while in the HIGH_POWER zone and slowly increase frequencies while - * in the LOW_POWER zone). On idle, we will hit the timeout and drop - * to the next level quickly, and conversely if busy we expect to - * hit a waitboost and rapidly switch into max power. - */ - if ((adj < 0 && rps->power.mode == HIGH_POWER) || - (adj > 0 && rps->power.mode == LOW_POWER)) - rps->last_adj = 0; - - /* sysfs frequency interfaces may have snuck in while servicing the - * interrupt + * sysfs frequency limits may have snuck in while + * servicing the interrupt */ new_freq += adj; new_freq = clamp_t(int, new_freq, min, max); if (intel_rps_set(rps, new_freq)) { - DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); - rps->last_adj = 0; + drm_dbg(&i915->drm, "Failed to set new GPU frequency\n"); + adj = 0; } + rps->last_adj = adj; mutex_unlock(&rps->lock); @@ -1545,6 +1711,8 @@ void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) if (unlikely(!events)) return; + GT_TRACE(gt, "irq events:%x\n", events); + gen6_gt_pm_mask_irq(gt, events); rps->pm_iir |= events; @@ -1554,11 +1722,17 @@ void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) { struct intel_gt *gt = rps_to_gt(rps); + u32 events; - if (pm_iir & rps->pm_events) { + events = pm_iir & rps->pm_events; + if (events) { spin_lock(>->irq_lock); - gen6_gt_pm_mask_irq(gt, pm_iir & rps->pm_events); - rps->pm_iir |= pm_iir & rps->pm_events; + + GT_TRACE(gt, "irq events:%x\n", events); + + gen6_gt_pm_mask_irq(gt, events); + rps->pm_iir |= events; + schedule_work(&rps->work); spin_unlock(>->irq_lock); } @@ -1613,6 +1787,7 @@ void intel_rps_init_early(struct intel_rps *rps) mutex_init(&rps->power.mutex); INIT_WORK(&rps->work, rps_work); + timer_setup(&rps->timer, rps_timer, 0); atomic_set(&rps->num_waiters, 0); } @@ -1641,9 +1816,10 @@ void intel_rps_init(struct intel_rps *rps) sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, ¶ms, NULL); if (params & BIT(31)) { /* OC supported */ - DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n", - (rps->max_freq & 0xff) * 50, - (params & 0xff) * 50); + drm_dbg(&i915->drm, + "Overclocking supported, max: %dMHz, overclock: %dMHz\n", + (rps->max_freq & 0xff) * 50, + (params & 0xff) * 50); rps->max_freq = params & 0xff; } } @@ -1651,7 +1827,9 @@ void intel_rps_init(struct intel_rps *rps) /* Finally allow us to boost to max by default */ rps->boost_freq = rps->max_freq; rps->idle_freq = rps->min_freq; - rps->cur_freq = rps->idle_freq; + + /* Start in the middle, from here we will autotune based on workload */ + rps->cur_freq = rps->efficient_freq; rps->pm_intrmsk_mbz = 0; @@ -1668,6 +1846,12 @@ void intel_rps_init(struct intel_rps *rps) rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; } +void intel_rps_sanitize(struct intel_rps *rps) +{ + if (INTEL_GEN(rps_to_i915(rps)) >= 6) + rps_disable_interrupts(rps); +} + u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) { struct drm_i915_private *i915 = rps_to_i915(rps); @@ -1695,7 +1879,7 @@ static u32 read_cagf(struct intel_rps *rps) freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); vlv_punit_put(i915); } else { - freq = intel_uncore_read(rps_to_gt(rps)->uncore, GEN6_RPSTAT1); + freq = intel_uncore_read(rps_to_uncore(rps), GEN6_RPSTAT1); } return intel_rps_get_cagf(rps, freq); @@ -1703,7 +1887,7 @@ static u32 read_cagf(struct intel_rps *rps) u32 intel_rps_read_actual_frequency(struct intel_rps *rps) { - struct intel_runtime_pm *rpm = rps_to_gt(rps)->uncore->rpm; + struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm; intel_wakeref_t wakeref; u32 freq = 0; @@ -1903,3 +2087,7 @@ bool i915_gpu_turbo_disable(void) return ret; } EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "selftest_rps.c" +#endif diff --git a/drivers/gpu/drm/i915/gt/intel_rps.h b/drivers/gpu/drm/i915/gt/intel_rps.h index dfa98194f3b2..8d3c9d663662 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.h +++ b/drivers/gpu/drm/i915/gt/intel_rps.h @@ -13,6 +13,7 @@ struct i915_request; void intel_rps_init_early(struct intel_rps *rps); void intel_rps_init(struct intel_rps *rps); +void intel_rps_sanitize(struct intel_rps *rps); void intel_rps_driver_register(struct intel_rps *rps); void intel_rps_driver_unregister(struct intel_rps *rps); @@ -36,4 +37,64 @@ void gen5_rps_irq_handler(struct intel_rps *rps); void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir); +static inline bool intel_rps_is_enabled(const struct intel_rps *rps) +{ + return test_bit(INTEL_RPS_ENABLED, &rps->flags); +} + +static inline void intel_rps_set_enabled(struct intel_rps *rps) +{ + set_bit(INTEL_RPS_ENABLED, &rps->flags); +} + +static inline void intel_rps_clear_enabled(struct intel_rps *rps) +{ + clear_bit(INTEL_RPS_ENABLED, &rps->flags); +} + +static inline bool intel_rps_is_active(const struct intel_rps *rps) +{ + return test_bit(INTEL_RPS_ACTIVE, &rps->flags); +} + +static inline void intel_rps_set_active(struct intel_rps *rps) +{ + set_bit(INTEL_RPS_ACTIVE, &rps->flags); +} + +static inline bool intel_rps_clear_active(struct intel_rps *rps) +{ + return test_and_clear_bit(INTEL_RPS_ACTIVE, &rps->flags); +} + +static inline bool intel_rps_has_interrupts(const struct intel_rps *rps) +{ + return test_bit(INTEL_RPS_INTERRUPTS, &rps->flags); +} + +static inline void intel_rps_set_interrupts(struct intel_rps *rps) +{ + set_bit(INTEL_RPS_INTERRUPTS, &rps->flags); +} + +static inline void intel_rps_clear_interrupts(struct intel_rps *rps) +{ + clear_bit(INTEL_RPS_INTERRUPTS, &rps->flags); +} + +static inline bool intel_rps_uses_timer(const struct intel_rps *rps) +{ + return test_bit(INTEL_RPS_TIMER, &rps->flags); +} + +static inline void intel_rps_set_timer(struct intel_rps *rps) +{ + set_bit(INTEL_RPS_TIMER, &rps->flags); +} + +static inline void intel_rps_clear_timer(struct intel_rps *rps) +{ + clear_bit(INTEL_RPS_TIMER, &rps->flags); +} + #endif /* INTEL_RPS_H */ diff --git a/drivers/gpu/drm/i915/gt/intel_rps_types.h b/drivers/gpu/drm/i915/gt/intel_rps_types.h index c2e279154bd5..38083f0402d9 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps_types.h +++ b/drivers/gpu/drm/i915/gt/intel_rps_types.h @@ -31,6 +31,13 @@ struct intel_rps_ei { u32 media_c0; }; +enum { + INTEL_RPS_ENABLED = 0, + INTEL_RPS_ACTIVE, + INTEL_RPS_INTERRUPTS, + INTEL_RPS_TIMER, +}; + struct intel_rps { struct mutex lock; /* protects enabling and the worker */ @@ -38,9 +45,12 @@ struct intel_rps { * work, interrupts_enabled and pm_iir are protected by * dev_priv->irq_lock */ + struct timer_list timer; struct work_struct work; - bool enabled; - bool active; + unsigned long flags; + + ktime_t pm_timestamp; + u32 pm_interval; u32 pm_iir; /* PM interrupt bits that should never be masked */ diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c index 74f793423231..d173271c7397 100644 --- a/drivers/gpu/drm/i915/gt/intel_sseu.c +++ b/drivers/gpu/drm/i915/gt/intel_sseu.c @@ -65,7 +65,6 @@ u32 intel_sseu_make_rpcs(struct drm_i915_private *i915, { const struct sseu_dev_info *sseu = &RUNTIME_INFO(i915)->sseu; bool subslice_pg = sseu->has_subslice_pg; - struct intel_sseu ctx_sseu; u8 slices, subslices; u32 rpcs = 0; @@ -78,31 +77,13 @@ u32 intel_sseu_make_rpcs(struct drm_i915_private *i915, /* * If i915/perf is active, we want a stable powergating configuration - * on the system. - * - * We could choose full enablement, but on ICL we know there are use - * cases which disable slices for functional, apart for performance - * reasons. So in this case we select a known stable subset. + * on the system. Use the configuration pinned by i915/perf. */ - if (!i915->perf.exclusive_stream) { - ctx_sseu = *req_sseu; - } else { - ctx_sseu = intel_sseu_from_device_info(sseu); - - if (IS_GEN(i915, 11)) { - /* - * We only need subslice count so it doesn't matter - * which ones we select - just turn off low bits in the - * amount of half of all available subslices per slice. - */ - ctx_sseu.subslice_mask = - ~(~0 << (hweight8(ctx_sseu.subslice_mask) / 2)); - ctx_sseu.slice_mask = 0x1; - } - } + if (i915->perf.exclusive_stream) + req_sseu = &i915->perf.sseu; - slices = hweight8(ctx_sseu.slice_mask); - subslices = hweight8(ctx_sseu.subslice_mask); + slices = hweight8(req_sseu->slice_mask); + subslices = hweight8(req_sseu->subslice_mask); /* * Since the SScount bitfield in GEN8_R_PWR_CLK_STATE is only three bits @@ -175,13 +156,13 @@ u32 intel_sseu_make_rpcs(struct drm_i915_private *i915, if (sseu->has_eu_pg) { u32 val; - val = ctx_sseu.min_eus_per_subslice << GEN8_RPCS_EU_MIN_SHIFT; + val = req_sseu->min_eus_per_subslice << GEN8_RPCS_EU_MIN_SHIFT; GEM_BUG_ON(val & ~GEN8_RPCS_EU_MIN_MASK); val &= GEN8_RPCS_EU_MIN_MASK; rpcs |= val; - val = ctx_sseu.max_eus_per_subslice << GEN8_RPCS_EU_MAX_SHIFT; + val = req_sseu->max_eus_per_subslice << GEN8_RPCS_EU_MAX_SHIFT; GEM_BUG_ON(val & ~GEN8_RPCS_EU_MAX_MASK); val &= GEN8_RPCS_EU_MAX_MASK; diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.c b/drivers/gpu/drm/i915/gt/intel_timeline.c index 54e1e55f3c81..4546284fede1 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.c +++ b/drivers/gpu/drm/i915/gt/intel_timeline.c @@ -119,6 +119,15 @@ static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline) spin_unlock_irqrestore(>->hwsp_lock, flags); } +static void __rcu_cacheline_free(struct rcu_head *rcu) +{ + struct intel_timeline_cacheline *cl = + container_of(rcu, typeof(*cl), rcu); + + i915_active_fini(&cl->active); + kfree(cl); +} + static void __idle_cacheline_free(struct intel_timeline_cacheline *cl) { GEM_BUG_ON(!i915_active_is_idle(&cl->active)); @@ -127,8 +136,7 @@ static void __idle_cacheline_free(struct intel_timeline_cacheline *cl) i915_vma_put(cl->hwsp->vma); __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS)); - i915_active_fini(&cl->active); - kfree_rcu(cl, rcu); + call_rcu(&cl->rcu, __rcu_cacheline_free); } __i915_active_call @@ -192,16 +200,20 @@ static void cacheline_release(struct intel_timeline_cacheline *cl) static void cacheline_free(struct intel_timeline_cacheline *cl) { + if (!i915_active_acquire_if_busy(&cl->active)) { + __idle_cacheline_free(cl); + return; + } + GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE)); cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE); - if (i915_active_is_idle(&cl->active)) - __idle_cacheline_free(cl); + i915_active_release(&cl->active); } -int intel_timeline_init(struct intel_timeline *timeline, - struct intel_gt *gt, - struct i915_vma *hwsp) +static int intel_timeline_init(struct intel_timeline *timeline, + struct intel_gt *gt, + struct i915_vma *hwsp) { void *vaddr; @@ -268,7 +280,7 @@ void intel_gt_init_timelines(struct intel_gt *gt) INIT_LIST_HEAD(&timelines->hwsp_free_list); } -void intel_timeline_fini(struct intel_timeline *timeline) +static void intel_timeline_fini(struct intel_timeline *timeline) { GEM_BUG_ON(atomic_read(&timeline->pin_count)); GEM_BUG_ON(!list_empty(&timeline->requests)); @@ -325,6 +337,13 @@ int intel_timeline_pin(struct intel_timeline *tl) return 0; } +void intel_timeline_reset_seqno(const struct intel_timeline *tl) +{ + /* Must be pinned to be writable, and no requests in flight. */ + GEM_BUG_ON(!atomic_read(&tl->pin_count)); + WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno); +} + void intel_timeline_enter(struct intel_timeline *tl) { struct intel_gt_timelines *timelines = &tl->gt->timelines; @@ -353,8 +372,16 @@ void intel_timeline_enter(struct intel_timeline *tl) return; spin_lock(&timelines->lock); - if (!atomic_fetch_inc(&tl->active_count)) + if (!atomic_fetch_inc(&tl->active_count)) { + /* + * The HWSP is volatile, and may have been lost while inactive, + * e.g. across suspend/resume. Be paranoid, and ensure that + * the HWSP value matches our seqno so we don't proclaim + * the next request as already complete. + */ + intel_timeline_reset_seqno(tl); list_add_tail(&tl->link, &timelines->active_list); + } spin_unlock(&timelines->lock); } @@ -517,6 +544,8 @@ int intel_timeline_read_hwsp(struct i915_request *from, rcu_read_lock(); cl = rcu_dereference(from->hwsp_cacheline); + if (i915_request_completed(from)) /* confirm cacheline is valid */ + goto unlock; if (unlikely(!i915_active_acquire_if_busy(&cl->active))) goto unlock; /* seqno wrapped and completed! */ if (unlikely(i915_request_completed(from))) diff --git a/drivers/gpu/drm/i915/gt/intel_timeline.h b/drivers/gpu/drm/i915/gt/intel_timeline.h index f5b7eade3809..4298b9ac7327 100644 --- a/drivers/gpu/drm/i915/gt/intel_timeline.h +++ b/drivers/gpu/drm/i915/gt/intel_timeline.h @@ -31,11 +31,6 @@ #include "i915_syncmap.h" #include "gt/intel_timeline_types.h" -int intel_timeline_init(struct intel_timeline *tl, - struct intel_gt *gt, - struct i915_vma *hwsp); -void intel_timeline_fini(struct intel_timeline *tl); - struct intel_timeline * intel_timeline_create(struct intel_gt *gt, struct i915_vma *global_hwsp); @@ -84,6 +79,8 @@ int intel_timeline_get_seqno(struct intel_timeline *tl, void intel_timeline_exit(struct intel_timeline *tl); void intel_timeline_unpin(struct intel_timeline *tl); +void intel_timeline_reset_seqno(const struct intel_timeline *tl); + int intel_timeline_read_hwsp(struct i915_request *from, struct i915_request *until, u32 *hwsp_offset); diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 887e0dc701f7..90a2b9e399b0 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -485,25 +485,14 @@ static void cfl_ctx_workarounds_init(struct intel_engine_cs *engine, static void cnl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { - struct drm_i915_private *i915 = engine->i915; - /* WaForceContextSaveRestoreNonCoherent:cnl */ WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0, HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT); - /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */ - if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0)) - WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5); - /* WaDisableReplayBufferBankArbitrationOptimization:cnl */ WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION); - /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */ - if (IS_CNL_REVID(i915, 0, CNL_REVID_B0)) - WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, - GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE); - /* WaPushConstantDereferenceHoldDisable:cnl */ WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE); @@ -575,29 +564,46 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine, /* allow headerless messages for preemptible GPGPU context */ WA_SET_BIT_MASKED(GEN10_SAMPLER_MODE, GEN11_SAMPLER_ENABLE_HEADLESS_MSG); + + /* Wa_1604278689:icl,ehl */ + wa_write(wal, IVB_FBC_RT_BASE, 0xFFFFFFFF & ~ILK_FBC_RT_VALID); + wa_write_masked_or(wal, IVB_FBC_RT_BASE_UPPER, + 0, /* write-only register; skip validation */ + 0xFFFFFFFF); + + /* Wa_1406306137:icl,ehl */ + wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN11_DIS_PICK_2ND_EU); } static void tgl_ctx_workarounds_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) { - u32 val; - - /* Wa_1409142259:tgl */ + /* + * Wa_1409142259:tgl + * Wa_1409347922:tgl + * Wa_1409252684:tgl + * Wa_1409217633:tgl + * Wa_1409207793:tgl + * Wa_1409178076:tgl + * Wa_1408979724:tgl + */ WA_SET_BIT_MASKED(GEN11_COMMON_SLICE_CHICKEN3, GEN12_DISABLE_CPS_AWARE_COLOR_PIPE); - /* Wa_1604555607:tgl */ - val = intel_uncore_read(engine->uncore, FF_MODE2); - val &= ~FF_MODE2_TDS_TIMER_MASK; - val |= FF_MODE2_TDS_TIMER_128; /* - * FIXME: FF_MODE2 register is not readable till TGL B0. We can - * enable verification of WA from the later steppings, which enables - * the read of FF_MODE2. + * Wa_1604555607:gen12 and Wa_1608008084:gen12 + * FF_MODE2 register will return the wrong value when read. The default + * value for this register is zero for all fields and there are no bit + * masks. So instead of doing a RMW we should just write the TDS timer + * value for Wa_1604555607. */ - wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK, val, - IS_TGL_REVID(engine->i915, TGL_REVID_A0, TGL_REVID_A0) ? 0 : - FF_MODE2_TDS_TIMER_MASK); + wa_add(wal, FF_MODE2, FF_MODE2_TDS_TIMER_MASK, + FF_MODE2_TDS_TIMER_128, 0); + + /* WaDisableGPGPUMidThreadPreemption:tgl */ + WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, + GEN9_PREEMPT_GPGPU_LEVEL_MASK, + GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL); } static void @@ -820,7 +826,7 @@ wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) intel_uncore_read(&i915->uncore, GEN10_MIRROR_FUSE3) & GEN10_L3BANK_MASK; - DRM_DEBUG_DRIVER("L3 fuse = %x\n", l3_fuse); + drm_dbg(&i915->drm, "L3 fuse = %x\n", l3_fuse); l3_en = ~(l3_fuse << GEN10_L3BANK_PAIR_COUNT | l3_fuse); } else { l3_en = ~0; @@ -829,7 +835,8 @@ wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) slice = fls(sseu->slice_mask) - 1; subslice = fls(l3_en & intel_sseu_get_subslices(sseu, slice)); if (!subslice) { - DRM_WARN("No common index found between subslice mask %x and L3 bank mask %x!\n", + drm_warn(&i915->drm, + "No common index found between subslice mask %x and L3 bank mask %x!\n", intel_sseu_get_subslices(sseu, slice), l3_en); subslice = fls(l3_en); drm_WARN_ON(&i915->drm, !subslice); @@ -844,7 +851,7 @@ wa_init_mcr(struct drm_i915_private *i915, struct i915_wa_list *wal) mcr_mask = GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK; } - DRM_DEBUG_DRIVER("MCR slice/subslice = %x\n", mcr); + drm_dbg(&i915->drm, "MCR slice/subslice = %x\n", mcr); wa_write_masked_or(wal, GEN8_MCR_SELECTOR, mcr_mask, mcr); } @@ -854,12 +861,6 @@ cnl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { wa_init_mcr(i915, wal); - /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */ - if (IS_CNL_REVID(i915, CNL_REVID_B0, CNL_REVID_B0)) - wa_write_or(wal, - GAMT_CHKN_BIT_REG, - GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT); - /* WaInPlaceDecompressionHang:cnl */ wa_write_or(wal, GEN9_GAMT_ECO_REG_RW_IA, @@ -903,11 +904,6 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) SLICE_UNIT_LEVEL_CLKGATE, MSCUNIT_CLKGATE_DIS); - /* Wa_1406680159:icl */ - wa_write_or(wal, - SUBSLICE_UNIT_LEVEL_CLKGATE, - GWUNIT_CLKGATE_DIS); - /* Wa_1406838659:icl (pre-prod) */ if (IS_ICL_REVID(i915, ICL_REVID_A0, ICL_REVID_B0)) wa_write_or(wal, @@ -921,22 +917,27 @@ icl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) GAMT_CHKN_BIT_REG, GAMT_CHKN_DISABLE_L3_COH_PIPE); - /* Wa_1607087056:icl */ - wa_write_or(wal, - SLICE_UNIT_LEVEL_CLKGATE, - L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); + /* Wa_1607087056:icl,ehl,jsl */ + if (IS_ICELAKE(i915) || + IS_EHL_REVID(i915, EHL_REVID_A0, EHL_REVID_A0)) { + wa_write_or(wal, + SLICE_UNIT_LEVEL_CLKGATE, + L3_CLKGATE_DIS | L3_CR2X_CLKGATE_DIS); + } } static void tgl_gt_workarounds_init(struct drm_i915_private *i915, struct i915_wa_list *wal) { + wa_init_mcr(i915, wal); + /* Wa_1409420604:tgl */ if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) wa_write_or(wal, SUBSLICE_UNIT_LEVEL_CLKGATE2, CPSSUNIT_CLKGATE_DIS); - /* Wa_1409180338:tgl */ + /* Wa_1607087056:tgl also know as BUG:1409180338 */ if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) wa_write_or(wal, SLICE_UNIT_LEVEL_CLKGATE, @@ -1251,6 +1252,7 @@ static void tgl_whitelist_build(struct intel_engine_cs *engine) case RENDER_CLASS: /* * WaAllowPMDepthAndInvocationCountAccessFromUMD:tgl + * Wa_1408556865:tgl * * This covers 4 registers which are next to one another : * - PS_INVOCATION_COUNT @@ -1264,6 +1266,9 @@ static void tgl_whitelist_build(struct intel_engine_cs *engine) /* Wa_1808121037:tgl */ whitelist_reg(w, GEN7_COMMON_SLICE_CHICKEN1); + + /* Wa_1806527549:tgl */ + whitelist_reg(w, HIZ_CHICKEN); break; default: break; @@ -1330,19 +1335,21 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) struct drm_i915_private *i915 = engine->i915; if (IS_TGL_REVID(i915, TGL_REVID_A0, TGL_REVID_A0)) { - /* Wa_1606700617:tgl */ - wa_masked_en(wal, - GEN9_CS_DEBUG_MODE1, - FF_DOP_CLOCK_GATE_DISABLE); - - /* Wa_1607138336:tgl */ + /* + * Wa_1607138336:tgl + * Wa_1607063988:tgl + */ wa_write_or(wal, GEN9_CTX_PREEMPT_REG, GEN12_DISABLE_POSH_BUSY_FF_DOP_CG); - /* Wa_1607030317:tgl */ - /* Wa_1607186500:tgl */ - /* Wa_1607297627:tgl */ + /* + * Wa_1607030317:tgl + * Wa_1607186500:tgl + * Wa_1607297627:tgl there is 3 entries for this WA on BSpec, 2 + * of then says it is fixed on B0 the other one says it is + * permanent + */ wa_masked_en(wal, GEN6_RC_SLEEP_PSMI_CONTROL, GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | @@ -1361,10 +1368,29 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN7_FF_THREAD_MODE, GEN12_FF_TESSELATION_DOP_GATE_DISABLE); + /* Wa_1408615072:tgl */ + wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, + VSUNIT_CLKGATE_DIS_TGL); + } + + if (IS_TIGERLAKE(i915)) { /* Wa_1606931601:tgl */ + wa_masked_en(wal, GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ); + + /* Wa_1409804808:tgl */ + wa_masked_en(wal, GEN7_ROW_CHICKEN2, + GEN12_PUSH_CONST_DEREF_HOLD_DIS); + + /* Wa_1606700617:tgl */ wa_masked_en(wal, - GEN7_ROW_CHICKEN2, - GEN12_DISABLE_EARLY_READ); + GEN9_CS_DEBUG_MODE1, + FF_DOP_CLOCK_GATE_DISABLE); + + /* + * Wa_1409085225:tgl + * Wa_14010229206:tgl + */ + wa_masked_en(wal, GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH); } if (IS_GEN(i915, 11)) { @@ -1430,10 +1456,38 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) GEN11_SCRATCH2, GEN11_COHERENT_PARTIAL_WRITE_MERGE_ENABLE, 0); + + /* WaEnable32PlaneMode:icl */ + wa_masked_en(wal, GEN9_CSFE_CHICKEN1_RCS, + GEN11_ENABLE_32_PLANE_MODE); + + /* + * Wa_1408615072:icl,ehl (vsunit) + * Wa_1407596294:icl,ehl (hsunit) + */ + wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE, + VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); + + /* Wa_1407352427:icl,ehl */ + wa_write_or(wal, UNSLICE_UNIT_LEVEL_CLKGATE2, + PSDUNIT_CLKGATE_DIS); + + /* Wa_1406680159:icl,ehl */ + wa_write_or(wal, + SUBSLICE_UNIT_LEVEL_CLKGATE, + GWUNIT_CLKGATE_DIS); + + /* + * Wa_1408767742:icl[a2..forever],ehl[all] + * Wa_1605460711:icl[a0..c0] + */ + wa_write_or(wal, + GEN7_FF_THREAD_MODE, + GEN12_FF_TESSELATION_DOP_GATE_DISABLE); } - if (IS_GEN_RANGE(i915, 9, 11)) { - /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl */ + if (IS_GEN_RANGE(i915, 9, 12)) { + /* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */ wa_masked_en(wal, GEN7_FF_SLICE_CS_CHICKEN1, GEN9_FFSC_PERCTX_PREEMPT_CTRL); @@ -1600,15 +1654,34 @@ err_obj: return ERR_PTR(err); } +static const struct { + u32 start; + u32 end; +} mcr_ranges_gen8[] = { + { .start = 0x5500, .end = 0x55ff }, + { .start = 0x7000, .end = 0x7fff }, + { .start = 0x9400, .end = 0x97ff }, + { .start = 0xb000, .end = 0xb3ff }, + { .start = 0xe000, .end = 0xe7ff }, + {}, +}; + static bool mcr_range(struct drm_i915_private *i915, u32 offset) { + int i; + + if (INTEL_GEN(i915) < 8) + return false; + /* - * Registers in this range are affected by the MCR selector + * Registers in these ranges are affected by the MCR selector * which only controls CPU initiated MMIO. Routing does not * work for CS access so we cannot verify them on this path. */ - if (INTEL_GEN(i915) >= 8 && (offset >= 0xb000 && offset <= 0xb4ff)) - return true; + for (i = 0; mcr_ranges_gen8[i].start; i++) + if (offset >= mcr_ranges_gen8[i].start && + offset <= mcr_ranges_gen8[i].end) + return true; return false; } diff --git a/drivers/gpu/drm/i915/gt/ivb_clear_kernel.c b/drivers/gpu/drm/i915/gt/ivb_clear_kernel.c new file mode 100644 index 000000000000..610ca7687735 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/ivb_clear_kernel.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + * + * Generated by: IGT Gpu Tools on Fri 21 Feb 2020 05:29:32 AM UTC + */ + +static const u32 ivb_clear_kernel[] = { + 0x00000001, 0x26020128, 0x00000024, 0x00000000, + 0x00000040, 0x20280c21, 0x00000028, 0x00000001, + 0x01000010, 0x20000c20, 0x0000002c, 0x00000000, + 0x00010220, 0x34001c00, 0x00001400, 0x0000002c, + 0x00600001, 0x20600061, 0x00000000, 0x00000000, + 0x00000008, 0x20601c85, 0x00000e00, 0x0000000c, + 0x00000005, 0x20601ca5, 0x00000060, 0x00000001, + 0x00000008, 0x20641c85, 0x00000e00, 0x0000000d, + 0x00000005, 0x20641ca5, 0x00000064, 0x00000003, + 0x00000041, 0x207424a5, 0x00000064, 0x00000034, + 0x00000040, 0x206014a5, 0x00000060, 0x00000074, + 0x00000008, 0x20681c85, 0x00000e00, 0x00000008, + 0x00000005, 0x20681ca5, 0x00000068, 0x0000000f, + 0x00000041, 0x20701ca5, 0x00000060, 0x00000010, + 0x00000040, 0x206814a5, 0x00000068, 0x00000070, + 0x00600001, 0x20a00061, 0x00000000, 0x00000000, + 0x00000005, 0x206c1c85, 0x00000e00, 0x00000007, + 0x00000041, 0x206c1ca5, 0x0000006c, 0x00000004, + 0x00600001, 0x20800021, 0x008d0000, 0x00000000, + 0x00000001, 0x20800021, 0x0000006c, 0x00000000, + 0x00000001, 0x20840021, 0x00000068, 0x00000000, + 0x00000001, 0x20880061, 0x00000000, 0x00000003, + 0x00000005, 0x208c0d21, 0x00000086, 0xffffffff, + 0x05600032, 0x20a00fa1, 0x008d0080, 0x02190001, + 0x00000040, 0x20a01ca5, 0x000000a0, 0x00000001, + 0x05600032, 0x20a00fa1, 0x008d0080, 0x040a8001, + 0x02000040, 0x20281c21, 0x00000028, 0xffffffff, + 0x00010220, 0x34001c00, 0x00001400, 0xfffffffc, + 0x00000001, 0x26020128, 0x00000024, 0x00000000, + 0x00000001, 0x220010e4, 0x00000000, 0x00000000, + 0x00000001, 0x220831ec, 0x00000000, 0x007f007f, + 0x00600001, 0x20400021, 0x008d0000, 0x00000000, + 0x00600001, 0x2fe00021, 0x008d0000, 0x00000000, + 0x00200001, 0x20400121, 0x00450020, 0x00000000, + 0x00000001, 0x20480061, 0x00000000, 0x000f000f, + 0x00000005, 0x204c0d21, 0x00000046, 0xffffffef, + 0x00800001, 0x20600061, 0x00000000, 0x00000000, + 0x00800001, 0x20800061, 0x00000000, 0x00000000, + 0x00800001, 0x20a00061, 0x00000000, 0x00000000, + 0x00800001, 0x20c00061, 0x00000000, 0x00000000, + 0x00800001, 0x20e00061, 0x00000000, 0x00000000, + 0x00800001, 0x21000061, 0x00000000, 0x00000000, + 0x00800001, 0x21200061, 0x00000000, 0x00000000, + 0x00800001, 0x21400061, 0x00000000, 0x00000000, + 0x05600032, 0x20000fa0, 0x008d0040, 0x120a8000, + 0x00000040, 0x20402d21, 0x00000020, 0x00100010, + 0x05600032, 0x20000fa0, 0x008d0040, 0x120a8000, + 0x02000040, 0x22083d8c, 0x00000208, 0xffffffff, + 0x00800001, 0xa0000109, 0x00000602, 0x00000000, + 0x00000040, 0x22001c84, 0x00000200, 0x00000020, + 0x00010220, 0x34001c00, 0x00001400, 0xfffffff8, + 0x07600032, 0x20000fa0, 0x008d0fe0, 0x82000010, +}; diff --git a/drivers/gpu/drm/i915/gt/mock_engine.c b/drivers/gpu/drm/i915/gt/mock_engine.c index 5633515c12e9..b8dd3cbc8696 100644 --- a/drivers/gpu/drm/i915/gt/mock_engine.c +++ b/drivers/gpu/drm/i915/gt/mock_engine.c @@ -28,7 +28,6 @@ #include "i915_drv.h" #include "intel_context.h" #include "intel_engine_pm.h" -#include "intel_engine_pool.h" #include "mock_engine.h" #include "selftests/mock_request.h" @@ -244,9 +243,7 @@ static void mock_reset_cancel(struct intel_engine_cs *engine) /* Mark all submitted requests as skipped. */ list_for_each_entry(request, &engine->active.requests, sched.link) { - if (!i915_request_signaled(request)) - dma_fence_set_error(&request->fence, -EIO); - + i915_request_set_error_once(request, -EIO); i915_request_mark_complete(request); } @@ -330,7 +327,6 @@ int mock_engine_init(struct intel_engine_cs *engine) intel_engine_init_execlists(engine); intel_engine_init__pm(engine); intel_engine_init_retire(engine); - intel_engine_pool_init(&engine->pool); ce = create_kernel_context(engine); if (IS_ERR(ce)) diff --git a/drivers/gpu/drm/i915/gt/selftest_context.c b/drivers/gpu/drm/i915/gt/selftest_context.c index e874dfaa5316..52af1cee9a94 100644 --- a/drivers/gpu/drm/i915/gt/selftest_context.c +++ b/drivers/gpu/drm/i915/gt/selftest_context.c @@ -24,6 +24,7 @@ static int request_sync(struct i915_request *rq) /* Opencode i915_request_add() so we can keep the timeline locked. */ __i915_request_commit(rq); + rq->sched.attr.priority = I915_PRIORITY_BARRIER; __i915_request_queue(rq, NULL); timeout = i915_request_wait(rq, 0, HZ / 10); @@ -154,10 +155,7 @@ static int live_context_size(void *arg) */ for_each_engine(engine, gt, id) { - struct { - struct drm_i915_gem_object *state; - void *pinned; - } saved; + struct file *saved; if (!engine->context_size) continue; @@ -171,8 +169,7 @@ static int live_context_size(void *arg) * active state is sufficient, we are only checking that we * don't use more than we planned. */ - saved.state = fetch_and_zero(&engine->default_state); - saved.pinned = fetch_and_zero(&engine->pinned_default_state); + saved = fetch_and_zero(&engine->default_state); /* Overlaps with the execlists redzone */ engine->context_size += I915_GTT_PAGE_SIZE; @@ -181,8 +178,7 @@ static int live_context_size(void *arg) engine->context_size -= I915_GTT_PAGE_SIZE; - engine->pinned_default_state = saved.pinned; - engine->default_state = saved.state; + engine->default_state = saved; intel_engine_pm_put(engine); diff --git a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c index 43d4d589749f..697114dd1f47 100644 --- a/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c +++ b/drivers/gpu/drm/i915/gt/selftest_engine_heartbeat.c @@ -142,6 +142,24 @@ out: return err; } +static void engine_heartbeat_disable(struct intel_engine_cs *engine, + unsigned long *saved) +{ + *saved = engine->props.heartbeat_interval_ms; + engine->props.heartbeat_interval_ms = 0; + + intel_engine_pm_get(engine); + intel_engine_park_heartbeat(engine); +} + +static void engine_heartbeat_enable(struct intel_engine_cs *engine, + unsigned long saved) +{ + intel_engine_pm_put(engine); + + engine->props.heartbeat_interval_ms = saved; +} + static int live_idle_flush(void *arg) { struct intel_gt *gt = arg; @@ -152,9 +170,11 @@ static int live_idle_flush(void *arg) /* Check that we can flush the idle barriers */ for_each_engine(engine, gt, id) { - intel_engine_pm_get(engine); + unsigned long heartbeat; + + engine_heartbeat_disable(engine, &heartbeat); err = __live_idle_pulse(engine, intel_engine_flush_barriers); - intel_engine_pm_put(engine); + engine_heartbeat_enable(engine, heartbeat); if (err) break; } @@ -172,9 +192,11 @@ static int live_idle_pulse(void *arg) /* Check that heartbeat pulses flush the idle barriers */ for_each_engine(engine, gt, id) { - intel_engine_pm_get(engine); + unsigned long heartbeat; + + engine_heartbeat_disable(engine, &heartbeat); err = __live_idle_pulse(engine, intel_engine_pulse); - intel_engine_pm_put(engine); + engine_heartbeat_enable(engine, heartbeat); if (err && err != -ENODEV) break; diff --git a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c index 09ff8e4f88af..242181a5214c 100644 --- a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c @@ -7,6 +7,7 @@ #include "selftest_llc.h" #include "selftest_rc6.h" +#include "selftest_rps.h" static int live_gt_resume(void *arg) { @@ -52,6 +53,13 @@ int intel_gt_pm_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(live_rc6_manual), + SUBTEST(live_rps_clock_interval), + SUBTEST(live_rps_control), + SUBTEST(live_rps_frequency_cs), + SUBTEST(live_rps_frequency_srm), + SUBTEST(live_rps_power), + SUBTEST(live_rps_interrupt), + SUBTEST(live_rps_dynamic), SUBTEST(live_gt_resume), }; diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c index 3e5e6c86e843..2b2efff6e19d 100644 --- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c @@ -268,7 +268,7 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine) cancel_rq: if (err) { - i915_request_skip(rq, err); + i915_request_set_error_once(rq, err); i915_request_add(rq); } unpin_hws: @@ -1640,7 +1640,7 @@ static int igt_reset_engines_atomic(void *arg) if (!intel_has_reset_engine(gt)) return 0; - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; igt_global_reset_lock(gt); diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 49b93cda04ca..824f99c4cc7c 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -21,7 +21,8 @@ #include "gem/selftests/mock_context.h" #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) -#define NUM_GPR_DW (16 * 2) /* each GPR is 2 dwords */ +#define NUM_GPR 16 +#define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ static struct i915_vma *create_scratch(struct intel_gt *gt) { @@ -68,26 +69,84 @@ static void engine_heartbeat_enable(struct intel_engine_cs *engine, engine->props.heartbeat_interval_ms = saved; } +static bool is_active(struct i915_request *rq) +{ + if (i915_request_is_active(rq)) + return true; + + if (i915_request_on_hold(rq)) + return true; + + if (i915_request_started(rq)) + return true; + + return false; +} + static int wait_for_submit(struct intel_engine_cs *engine, struct i915_request *rq, unsigned long timeout) { timeout += jiffies; do { + bool done = time_after(jiffies, timeout); + + if (i915_request_completed(rq)) /* that was quick! */ + return 0; + + /* Wait until the HW has acknowleged the submission (or err) */ + intel_engine_flush_submission(engine); + if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) + return 0; + + if (done) + return -ETIME; + + cond_resched(); + } while (1); +} + +static int wait_for_reset(struct intel_engine_cs *engine, + struct i915_request *rq, + unsigned long timeout) +{ + timeout += jiffies; + + do { cond_resched(); intel_engine_flush_submission(engine); if (READ_ONCE(engine->execlists.pending[0])) continue; - if (i915_request_is_active(rq)) - return 0; + if (i915_request_completed(rq)) + break; - if (i915_request_started(rq)) /* that was quick! */ - return 0; + if (READ_ONCE(rq->fence.error)) + break; } while (time_before(jiffies, timeout)); - return -ETIME; + flush_scheduled_work(); + + if (rq->fence.error != -EIO) { + pr_err("%s: hanging request %llx:%lld not reset\n", + engine->name, + rq->fence.context, + rq->fence.seqno); + return -EINVAL; + } + + /* Give the request a jiffie to complete after flushing the worker */ + if (i915_request_wait(rq, 0, + max(0l, (long)(timeout - jiffies)) + 1) < 0) { + pr_err("%s: hanging request %llx:%lld did not complete\n", + engine->name, + rq->fence.context, + rq->fence.seqno); + return -ETIME; + } + + return 0; } static int live_sanitycheck(void *arg) @@ -591,9 +650,9 @@ static int live_error_interrupt(void *arg) error_repr(p->error[i])); if (!i915_request_started(client[i])) { - pr_debug("%s: %s request not stated!\n", - engine->name, - error_repr(p->error[i])); + pr_err("%s: %s request not started!\n", + engine->name, + error_repr(p->error[i])); err = -ETIME; goto out; } @@ -601,9 +660,10 @@ static int live_error_interrupt(void *arg) /* Kick the tasklet to process the error */ intel_engine_flush_submission(engine); if (client[i]->fence.error != p->error[i]) { - pr_err("%s: %s request completed with wrong error code: %d\n", + pr_err("%s: %s request (%s) with wrong error code: %d\n", engine->name, error_repr(p->error[i]), + i915_request_completed(client[i]) ? "completed" : "running", client[i]->fence.error); err = -EINVAL; goto out; @@ -886,7 +946,7 @@ create_rewinder(struct intel_context *ce, goto err; } - cs = intel_ring_begin(rq, 10); + cs = intel_ring_begin(rq, 14); if (IS_ERR(cs)) { err = PTR_ERR(cs); goto err; @@ -898,8 +958,8 @@ create_rewinder(struct intel_context *ce, *cs++ = MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT | MI_SEMAPHORE_POLL | - MI_SEMAPHORE_SAD_NEQ_SDD; - *cs++ = 0; + MI_SEMAPHORE_SAD_GTE_SDD; + *cs++ = idx; *cs++ = offset; *cs++ = 0; @@ -908,6 +968,11 @@ create_rewinder(struct intel_context *ce, *cs++ = offset + idx * sizeof(u32); *cs++ = 0; + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = offset; + *cs++ = 0; + *cs++ = idx + 1; + intel_ring_advance(rq, cs); rq->sched.attr.priority = I915_PRIORITY_MASK; @@ -941,7 +1006,7 @@ static int live_timeslice_rewind(void *arg) for_each_engine(engine, gt, id) { enum { A1, A2, B1 }; - enum { X = 1, Y, Z }; + enum { X = 1, Z, Y }; struct i915_request *rq[3] = {}; struct intel_context *ce; unsigned long heartbeat; @@ -974,13 +1039,13 @@ static int live_timeslice_rewind(void *arg) goto err; } - rq[0] = create_rewinder(ce, NULL, slot, 1); + rq[0] = create_rewinder(ce, NULL, slot, X); if (IS_ERR(rq[0])) { intel_context_put(ce); goto err; } - rq[1] = create_rewinder(ce, NULL, slot, 2); + rq[1] = create_rewinder(ce, NULL, slot, Y); intel_context_put(ce); if (IS_ERR(rq[1])) goto err; @@ -998,7 +1063,7 @@ static int live_timeslice_rewind(void *arg) goto err; } - rq[2] = create_rewinder(ce, rq[0], slot, 3); + rq[2] = create_rewinder(ce, rq[0], slot, Z); intel_context_put(ce); if (IS_ERR(rq[2])) goto err; @@ -1009,18 +1074,14 @@ static int live_timeslice_rewind(void *arg) engine->name); goto err; } - GEM_BUG_ON(!timer_pending(&engine->execlists.timer)); /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */ - GEM_BUG_ON(!i915_request_is_active(rq[A1])); - GEM_BUG_ON(!i915_request_is_active(rq[A2])); - GEM_BUG_ON(!i915_request_is_active(rq[B1])); - - /* Wait for the timeslice to kick in */ - del_timer(&engine->execlists.timer); - tasklet_hi_schedule(&engine->execlists.tasklet); - intel_engine_flush_submission(engine); - + if (i915_request_is_active(rq[A2])) { /* semaphore yielded! */ + /* Wait for the timeslice to kick in */ + del_timer(&engine->execlists.timer); + tasklet_hi_schedule(&engine->execlists.tasklet); + intel_engine_flush_submission(engine); + } /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */ GEM_BUG_ON(!i915_request_is_active(rq[A1])); GEM_BUG_ON(!i915_request_is_active(rq[B1])); @@ -1185,8 +1246,14 @@ static int live_timeslice_queue(void *arg) if (err) goto err_rq; - intel_engine_flush_submission(engine); + /* Wait until we ack the release_queue and start timeslicing */ + do { + cond_resched(); + intel_engine_flush_submission(engine); + } while (READ_ONCE(engine->execlists.pending[0])); + if (!READ_ONCE(engine->execlists.timer.expires) && + execlists_active(&engine->execlists) == rq && !i915_request_completed(rq)) { struct drm_printer p = drm_info_printer(gt->i915->drm.dev); @@ -1805,14 +1872,9 @@ static int __cancel_active0(struct live_preempt_cancel *arg) if (err) goto out; - if (i915_request_wait(rq, 0, HZ / 5) < 0) { - err = -EIO; - goto out; - } - - if (rq->fence.error != -EIO) { - pr_err("Cancelled inflight0 request did not report -EIO\n"); - err = -EINVAL; + err = wait_for_reset(arg->engine, rq, HZ / 2); + if (err) { + pr_err("Cancelled inflight0 request did not reset\n"); goto out; } @@ -1870,10 +1932,9 @@ static int __cancel_active1(struct live_preempt_cancel *arg) goto out; igt_spinner_end(&arg->a.spin); - if (i915_request_wait(rq[1], 0, HZ / 5) < 0) { - err = -EIO; + err = wait_for_reset(arg->engine, rq[1], HZ / 2); + if (err) goto out; - } if (rq[0]->fence.error != 0) { pr_err("Normal inflight0 request did not complete\n"); @@ -1953,10 +2014,9 @@ static int __cancel_queued(struct live_preempt_cancel *arg) if (err) goto out; - if (i915_request_wait(rq[2], 0, HZ / 5) < 0) { - err = -EIO; + err = wait_for_reset(arg->engine, rq[2], HZ / 2); + if (err) goto out; - } if (rq[0]->fence.error != -EIO) { pr_err("Cancelled inflight0 request did not report -EIO\n"); @@ -1994,6 +2054,9 @@ static int __cancel_hostile(struct live_preempt_cancel *arg) if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT)) return 0; + if (!intel_has_reset_engine(arg->engine->gt)) + return 0; + GEM_TRACE("%s(%s)\n", __func__, arg->engine->name); rq = spinner_create_request(&arg->a.spin, arg->a.ctx, arg->engine, @@ -2014,14 +2077,9 @@ static int __cancel_hostile(struct live_preempt_cancel *arg) if (err) goto out; - if (i915_request_wait(rq, 0, HZ / 5) < 0) { - err = -EIO; - goto out; - } - - if (rq->fence.error != -EIO) { - pr_err("Cancelled inflight0 request did not report -EIO\n"); - err = -EINVAL; + err = wait_for_reset(arg->engine, rq, HZ / 2); + if (err) { + pr_err("Cancelled inflight0 request did not reset\n"); goto out; } @@ -2109,7 +2167,7 @@ static int live_suppress_self_preempt(void *arg) if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915)) return 0; - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; /* presume black blox */ if (intel_vgpu_active(gt->i915)) @@ -2599,7 +2657,7 @@ static int create_gang(struct intel_engine_cs *engine, if (IS_ERR(rq)) goto err_obj; - rq->batch = vma; + rq->batch = i915_vma_get(vma); i915_request_get(rq); i915_vma_lock(vma); @@ -2623,6 +2681,7 @@ static int create_gang(struct intel_engine_cs *engine, return 0; err_rq: + i915_vma_put(rq->batch); i915_request_put(rq); err_obj: i915_gem_object_put(obj); @@ -2719,6 +2778,7 @@ static int live_preempt_gang(void *arg) err = -ETIME; } + i915_vma_put(rq->batch); i915_request_put(rq); rq = n; } @@ -2732,6 +2792,331 @@ static int live_preempt_gang(void *arg) return 0; } +static struct i915_vma * +create_gpr_user(struct intel_engine_cs *engine, + struct i915_vma *result, + unsigned int offset) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + u32 *cs; + int err; + int i; + + obj = i915_gem_object_create_internal(engine->i915, 4096); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + vma = i915_vma_instance(obj, result->vm, NULL); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return vma; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) { + i915_vma_put(vma); + return ERR_PTR(err); + } + + cs = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(cs)) { + i915_vma_put(vma); + return ERR_CAST(cs); + } + + /* All GPR are clear for new contexts. We use GPR(0) as a constant */ + *cs++ = MI_LOAD_REGISTER_IMM(1); + *cs++ = CS_GPR(engine, 0); + *cs++ = 1; + + for (i = 1; i < NUM_GPR; i++) { + u64 addr; + + /* + * Perform: GPR[i]++ + * + * As we read and write into the context saved GPR[i], if + * we restart this batch buffer from an earlier point, we + * will repeat the increment and store a value > 1. + */ + *cs++ = MI_MATH(4); + *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(i)); + *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(0)); + *cs++ = MI_MATH_ADD; + *cs++ = MI_MATH_STORE(MI_MATH_REG(i), MI_MATH_REG_ACCU); + + addr = result->node.start + offset + i * sizeof(*cs); + *cs++ = MI_STORE_REGISTER_MEM_GEN8; + *cs++ = CS_GPR(engine, 2 * i); + *cs++ = lower_32_bits(addr); + *cs++ = upper_32_bits(addr); + + *cs++ = MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_GTE_SDD; + *cs++ = i; + *cs++ = lower_32_bits(result->node.start); + *cs++ = upper_32_bits(result->node.start); + } + + *cs++ = MI_BATCH_BUFFER_END; + i915_gem_object_flush_map(obj); + i915_gem_object_unpin_map(obj); + + return vma; +} + +static struct i915_vma *create_global(struct intel_gt *gt, size_t sz) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + int err; + + obj = i915_gem_object_create_internal(gt->i915, sz); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + vma = i915_vma_instance(obj, >->ggtt->vm, NULL); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return vma; + } + + err = i915_ggtt_pin(vma, 0, 0); + if (err) { + i915_vma_put(vma); + return ERR_PTR(err); + } + + return vma; +} + +static struct i915_request * +create_gpr_client(struct intel_engine_cs *engine, + struct i915_vma *global, + unsigned int offset) +{ + struct i915_vma *batch, *vma; + struct intel_context *ce; + struct i915_request *rq; + int err; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return ERR_CAST(ce); + + vma = i915_vma_instance(global->obj, ce->vm, NULL); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + goto out_ce; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) + goto out_ce; + + batch = create_gpr_user(engine, vma, offset); + if (IS_ERR(batch)) { + err = PTR_ERR(batch); + goto out_vma; + } + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto out_batch; + } + + i915_vma_lock(vma); + err = i915_request_await_object(rq, vma->obj, false); + if (!err) + err = i915_vma_move_to_active(vma, rq, 0); + i915_vma_unlock(vma); + + i915_vma_lock(batch); + if (!err) + err = i915_request_await_object(rq, batch->obj, false); + if (!err) + err = i915_vma_move_to_active(batch, rq, 0); + if (!err) + err = rq->engine->emit_bb_start(rq, + batch->node.start, + PAGE_SIZE, 0); + i915_vma_unlock(batch); + i915_vma_unpin(batch); + + if (!err) + i915_request_get(rq); + i915_request_add(rq); + +out_batch: + i915_vma_put(batch); +out_vma: + i915_vma_unpin(vma); +out_ce: + intel_context_put(ce); + return err ? ERR_PTR(err) : rq; +} + +static int preempt_user(struct intel_engine_cs *engine, + struct i915_vma *global, + int id) +{ + struct i915_sched_attr attr = { + .priority = I915_PRIORITY_MAX + }; + struct i915_request *rq; + int err = 0; + u32 *cs; + + rq = intel_engine_create_kernel_request(engine); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + cs = intel_ring_begin(rq, 4); + if (IS_ERR(cs)) { + i915_request_add(rq); + return PTR_ERR(cs); + } + + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = i915_ggtt_offset(global); + *cs++ = 0; + *cs++ = id; + + intel_ring_advance(rq, cs); + + i915_request_get(rq); + i915_request_add(rq); + + engine->schedule(rq, &attr); + + if (i915_request_wait(rq, 0, HZ / 2) < 0) + err = -ETIME; + i915_request_put(rq); + + return err; +} + +static int live_preempt_user(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + struct i915_vma *global; + enum intel_engine_id id; + u32 *result; + int err = 0; + + if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915)) + return 0; + + /* + * In our other tests, we look at preemption in carefully + * controlled conditions in the ringbuffer. Since most of the + * time is spent in user batches, most of our preemptions naturally + * occur there. We want to verify that when we preempt inside a batch + * we continue on from the current instruction and do not roll back + * to the start, or another earlier arbitration point. + * + * To verify this, we create a batch which is a mixture of + * MI_MATH (gpr++) MI_SRM (gpr) and preemption points. Then with + * a few preempting contexts thrown into the mix, we look for any + * repeated instructions (which show up as incorrect values). + */ + + global = create_global(gt, 4096); + if (IS_ERR(global)) + return PTR_ERR(global); + + result = i915_gem_object_pin_map(global->obj, I915_MAP_WC); + if (IS_ERR(result)) { + i915_vma_unpin_and_release(&global, 0); + return PTR_ERR(result); + } + + for_each_engine(engine, gt, id) { + struct i915_request *client[3] = {}; + struct igt_live_test t; + int i; + + if (!intel_engine_has_preemption(engine)) + continue; + + if (IS_GEN(gt->i915, 8) && engine->class != RENDER_CLASS) + continue; /* we need per-context GPR */ + + if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) { + err = -EIO; + break; + } + + memset(result, 0, 4096); + + for (i = 0; i < ARRAY_SIZE(client); i++) { + struct i915_request *rq; + + rq = create_gpr_client(engine, global, + NUM_GPR * i * sizeof(u32)); + if (IS_ERR(rq)) + goto end_test; + + client[i] = rq; + } + + /* Continuously preempt the set of 3 running contexts */ + for (i = 1; i <= NUM_GPR; i++) { + err = preempt_user(engine, global, i); + if (err) + goto end_test; + } + + if (READ_ONCE(result[0]) != NUM_GPR) { + pr_err("%s: Failed to release semaphore\n", + engine->name); + err = -EIO; + goto end_test; + } + + for (i = 0; i < ARRAY_SIZE(client); i++) { + int gpr; + + if (i915_request_wait(client[i], 0, HZ / 2) < 0) { + err = -ETIME; + goto end_test; + } + + for (gpr = 1; gpr < NUM_GPR; gpr++) { + if (result[NUM_GPR * i + gpr] != 1) { + pr_err("%s: Invalid result, client %d, gpr %d, result: %d\n", + engine->name, + i, gpr, result[NUM_GPR * i + gpr]); + err = -EINVAL; + goto end_test; + } + } + } + +end_test: + for (i = 0; i < ARRAY_SIZE(client); i++) { + if (!client[i]) + break; + + i915_request_put(client[i]); + } + + /* Flush the semaphores on error */ + smp_store_mb(result[0], -1); + if (igt_live_test_end(&t)) + err = -EIO; + if (err) + break; + } + + i915_vma_unpin_and_release(&global, I915_VMA_RELEASE_MAP); + return err; +} + static int live_preempt_timeout(void *arg) { struct intel_gt *gt = arg; @@ -3224,7 +3609,7 @@ static int live_virtual_engine(void *arg) unsigned int class, inst; int err; - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; for_each_engine(engine, gt, id) { @@ -3357,7 +3742,7 @@ static int live_virtual_mask(void *arg) unsigned int class, inst; int err; - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; for (class = 0; class <= MAX_ENGINE_CLASS; class++) { @@ -3499,7 +3884,7 @@ static int live_virtual_preserved(void *arg) * are preserved. */ - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; /* As we use CS_GPR we cannot run before they existed on all engines. */ @@ -3729,7 +4114,7 @@ static int live_virtual_bond(void *arg) unsigned int class, inst; int err; - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; for (class = 0; class <= MAX_ENGINE_CLASS; class++) { @@ -3890,7 +4275,7 @@ static int live_virtual_reset(void *arg) * forgotten. */ - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; if (!intel_has_reset_engine(gt)) @@ -3939,6 +4324,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_chain_preempt), SUBTEST(live_preempt_gang), SUBTEST(live_preempt_timeout), + SUBTEST(live_preempt_user), SUBTEST(live_preempt_smoke), SUBTEST(live_virtual_engine), SUBTEST(live_virtual_mask), @@ -3956,35 +4342,6 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) return intel_gt_live_subtests(tests, &i915->gt); } -static void hexdump(const void *buf, size_t len) -{ - const size_t rowsize = 8 * sizeof(u32); - const void *prev = NULL; - bool skip = false; - size_t pos; - - for (pos = 0; pos < len; pos += rowsize) { - char line[128]; - - if (prev && !memcmp(prev, buf + pos, rowsize)) { - if (!skip) { - pr_info("*\n"); - skip = true; - } - continue; - } - - WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos, - rowsize, sizeof(u32), - line, sizeof(line), - false) >= sizeof(line)); - pr_info("[%04zx] %s\n", pos, line); - - prev = buf + pos; - skip = false; - } -} - static int emit_semaphore_signal(struct intel_context *ce, void *slot) { const u32 offset = @@ -4015,6 +4372,32 @@ static int emit_semaphore_signal(struct intel_context *ce, void *slot) return 0; } +static int context_flush(struct intel_context *ce, long timeout) +{ + struct i915_request *rq; + struct dma_fence *fence; + int err = 0; + + rq = intel_engine_create_kernel_request(ce->engine); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + fence = i915_active_fence_get(&ce->timeline->last_request); + if (fence) { + i915_request_await_dma_fence(rq, fence); + dma_fence_put(fence); + } + + rq = i915_request_get(rq); + i915_request_add(rq); + if (i915_request_wait(rq, 0, timeout) < 0) + err = -ETIME; + i915_request_put(rq); + + rmb(); /* We know the request is written, make sure all state is too! */ + return err; +} + static int live_lrc_layout(void *arg) { struct intel_gt *gt = arg; @@ -4040,13 +4423,12 @@ static int live_lrc_layout(void *arg) if (!engine->default_state) continue; - hw = i915_gem_object_pin_map(engine->default_state, - I915_MAP_WB); + hw = shmem_pin_map(engine->default_state); if (IS_ERR(hw)) { err = PTR_ERR(hw); break; } - hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw); + hw += LRC_STATE_OFFSET / sizeof(*hw); execlists_init_reg_state(memset(lrc, POISON_INUSE, PAGE_SIZE), engine->kernel_context, @@ -4107,13 +4489,13 @@ static int live_lrc_layout(void *arg) if (err) { pr_info("%s: HW register image:\n", engine->name); - hexdump(hw, PAGE_SIZE); + igt_hexdump(hw, PAGE_SIZE); pr_info("%s: SW register image:\n", engine->name); - hexdump(lrc, PAGE_SIZE); + igt_hexdump(lrc, PAGE_SIZE); } - i915_gem_object_unpin_map(engine->default_state); + shmem_unpin_map(engine->default_state, hw); if (err) break; } @@ -4182,10 +4564,35 @@ static int live_lrc_fixed(void *arg) "BB_STATE" }, { + i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), + lrc_ring_wa_bb_per_ctx(engine), + "RING_BB_PER_CTX_PTR" + }, + { + i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), + lrc_ring_indirect_ptr(engine), + "RING_INDIRECT_CTX_PTR" + }, + { + i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), + lrc_ring_indirect_offset(engine), + "RING_INDIRECT_CTX_OFFSET" + }, + { i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), CTX_TIMESTAMP - 1, "RING_CTX_TIMESTAMP" }, + { + i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), + lrc_ring_gpr0(engine), + "RING_CS_GPR0" + }, + { + i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), + lrc_ring_cmd_buf_cctl(engine), + "RING_CMD_BUF_CCTL" + }, { }, }, *t; u32 *hw; @@ -4193,13 +4600,12 @@ static int live_lrc_fixed(void *arg) if (!engine->default_state) continue; - hw = i915_gem_object_pin_map(engine->default_state, - I915_MAP_WB); + hw = shmem_pin_map(engine->default_state); if (IS_ERR(hw)) { err = PTR_ERR(hw); break; } - hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw); + hw += LRC_STATE_OFFSET / sizeof(*hw); for (t = tbl; t->name; t++) { int dw = find_offset(hw, t->reg); @@ -4215,7 +4621,7 @@ static int live_lrc_fixed(void *arg) } } - i915_gem_object_unpin_map(engine->default_state); + shmem_unpin_map(engine->default_state, hw); } return err; @@ -4638,18 +5044,10 @@ static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) wmb(); } - if (i915_request_wait(rq, 0, HZ / 2) < 0) { - err = -ETIME; - goto err; - } - - /* and wait for switch to kernel */ - if (igt_flush_test(arg->engine->i915)) { - err = -EIO; + /* And wait for switch to kernel (to save our context to memory) */ + err = context_flush(arg->ce[0], HZ / 2); + if (err) goto err; - } - - rmb(); if (!timestamp_advanced(arg->poison, slot[1])) { pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", @@ -4674,9 +5072,9 @@ err: static int live_lrc_timestamp(void *arg) { + struct lrc_timestamp data = {}; struct intel_gt *gt = arg; enum intel_engine_id id; - struct lrc_timestamp data; const u32 poison[] = { 0, S32_MAX, @@ -4748,6 +5146,860 @@ err: return 0; } +static struct i915_vma * +create_user_vma(struct i915_address_space *vm, unsigned long size) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + int err; + + obj = i915_gem_object_create_internal(vm->i915, size); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + vma = i915_vma_instance(obj, vm, NULL); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return vma; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) { + i915_gem_object_put(obj); + return ERR_PTR(err); + } + + return vma; +} + +static struct i915_vma * +store_context(struct intel_context *ce, struct i915_vma *scratch) +{ + struct i915_vma *batch; + u32 dw, x, *cs, *hw; + u32 *defaults; + + batch = create_user_vma(ce->vm, SZ_64K); + if (IS_ERR(batch)) + return batch; + + cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); + if (IS_ERR(cs)) { + i915_vma_put(batch); + return ERR_CAST(cs); + } + + defaults = shmem_pin_map(ce->engine->default_state); + if (!defaults) { + i915_gem_object_unpin_map(batch->obj); + i915_vma_put(batch); + return ERR_PTR(-ENOMEM); + } + + x = 0; + dw = 0; + hw = defaults; + hw += LRC_STATE_OFFSET / sizeof(*hw); + do { + u32 len = hw[dw] & 0x7f; + + if (hw[dw] == 0) { + dw++; + continue; + } + + if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + dw += len + 2; + continue; + } + + dw++; + len = (len + 1) / 2; + while (len--) { + *cs++ = MI_STORE_REGISTER_MEM_GEN8; + *cs++ = hw[dw]; + *cs++ = lower_32_bits(scratch->node.start + x); + *cs++ = upper_32_bits(scratch->node.start + x); + + dw += 2; + x += 4; + } + } while (dw < PAGE_SIZE / sizeof(u32) && + (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); + + *cs++ = MI_BATCH_BUFFER_END; + + shmem_unpin_map(ce->engine->default_state, defaults); + + i915_gem_object_flush_map(batch->obj); + i915_gem_object_unpin_map(batch->obj); + + return batch; +} + +static int move_to_active(struct i915_request *rq, + struct i915_vma *vma, + unsigned int flags) +{ + int err; + + i915_vma_lock(vma); + err = i915_request_await_object(rq, vma->obj, flags); + if (!err) + err = i915_vma_move_to_active(vma, rq, flags); + i915_vma_unlock(vma); + + return err; +} + +static struct i915_request * +record_registers(struct intel_context *ce, + struct i915_vma *before, + struct i915_vma *after, + u32 *sema) +{ + struct i915_vma *b_before, *b_after; + struct i915_request *rq; + u32 *cs; + int err; + + b_before = store_context(ce, before); + if (IS_ERR(b_before)) + return ERR_CAST(b_before); + + b_after = store_context(ce, after); + if (IS_ERR(b_after)) { + rq = ERR_CAST(b_after); + goto err_before; + } + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + goto err_after; + + err = move_to_active(rq, before, EXEC_OBJECT_WRITE); + if (err) + goto err_rq; + + err = move_to_active(rq, b_before, 0); + if (err) + goto err_rq; + + err = move_to_active(rq, after, EXEC_OBJECT_WRITE); + if (err) + goto err_rq; + + err = move_to_active(rq, b_after, 0); + if (err) + goto err_rq; + + cs = intel_ring_begin(rq, 14); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_rq; + } + + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); + *cs++ = lower_32_bits(b_before->node.start); + *cs++ = upper_32_bits(b_before->node.start); + + *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; + *cs++ = MI_SEMAPHORE_WAIT | + MI_SEMAPHORE_GLOBAL_GTT | + MI_SEMAPHORE_POLL | + MI_SEMAPHORE_SAD_NEQ_SDD; + *cs++ = 0; + *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(sema); + *cs++ = 0; + *cs++ = MI_NOOP; + + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); + *cs++ = lower_32_bits(b_after->node.start); + *cs++ = upper_32_bits(b_after->node.start); + + intel_ring_advance(rq, cs); + + WRITE_ONCE(*sema, 0); + i915_request_get(rq); + i915_request_add(rq); +err_after: + i915_vma_put(b_after); +err_before: + i915_vma_put(b_before); + return rq; + +err_rq: + i915_request_add(rq); + rq = ERR_PTR(err); + goto err_after; +} + +static struct i915_vma *load_context(struct intel_context *ce, u32 poison) +{ + struct i915_vma *batch; + u32 dw, *cs, *hw; + u32 *defaults; + + batch = create_user_vma(ce->vm, SZ_64K); + if (IS_ERR(batch)) + return batch; + + cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); + if (IS_ERR(cs)) { + i915_vma_put(batch); + return ERR_CAST(cs); + } + + defaults = shmem_pin_map(ce->engine->default_state); + if (!defaults) { + i915_gem_object_unpin_map(batch->obj); + i915_vma_put(batch); + return ERR_PTR(-ENOMEM); + } + + dw = 0; + hw = defaults; + hw += LRC_STATE_OFFSET / sizeof(*hw); + do { + u32 len = hw[dw] & 0x7f; + + if (hw[dw] == 0) { + dw++; + continue; + } + + if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + dw += len + 2; + continue; + } + + dw++; + len = (len + 1) / 2; + *cs++ = MI_LOAD_REGISTER_IMM(len); + while (len--) { + *cs++ = hw[dw]; + *cs++ = poison; + dw += 2; + } + } while (dw < PAGE_SIZE / sizeof(u32) && + (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); + + *cs++ = MI_BATCH_BUFFER_END; + + shmem_unpin_map(ce->engine->default_state, defaults); + + i915_gem_object_flush_map(batch->obj); + i915_gem_object_unpin_map(batch->obj); + + return batch; +} + +static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) +{ + struct i915_request *rq; + struct i915_vma *batch; + u32 *cs; + int err; + + batch = load_context(ce, poison); + if (IS_ERR(batch)) + return PTR_ERR(batch); + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_batch; + } + + err = move_to_active(rq, batch, 0); + if (err) + goto err_rq; + + cs = intel_ring_begin(rq, 8); + if (IS_ERR(cs)) { + err = PTR_ERR(cs); + goto err_rq; + } + + *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; + *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); + *cs++ = lower_32_bits(batch->node.start); + *cs++ = upper_32_bits(batch->node.start); + + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + + offset_in_page(sema); + *cs++ = 0; + *cs++ = 1; + + intel_ring_advance(rq, cs); + + rq->sched.attr.priority = I915_PRIORITY_BARRIER; +err_rq: + i915_request_add(rq); +err_batch: + i915_vma_put(batch); + return err; +} + +static bool is_moving(u32 a, u32 b) +{ + return a != b; +} + +static int compare_isolation(struct intel_engine_cs *engine, + struct i915_vma *ref[2], + struct i915_vma *result[2], + struct intel_context *ce, + u32 poison) +{ + u32 x, dw, *hw, *lrc; + u32 *A[2], *B[2]; + u32 *defaults; + int err = 0; + + A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC); + if (IS_ERR(A[0])) + return PTR_ERR(A[0]); + + A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC); + if (IS_ERR(A[1])) { + err = PTR_ERR(A[1]); + goto err_A0; + } + + B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC); + if (IS_ERR(B[0])) { + err = PTR_ERR(B[0]); + goto err_A1; + } + + B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC); + if (IS_ERR(B[1])) { + err = PTR_ERR(B[1]); + goto err_B0; + } + + lrc = i915_gem_object_pin_map(ce->state->obj, + i915_coherent_map_type(engine->i915)); + if (IS_ERR(lrc)) { + err = PTR_ERR(lrc); + goto err_B1; + } + lrc += LRC_STATE_OFFSET / sizeof(*hw); + + defaults = shmem_pin_map(ce->engine->default_state); + if (!defaults) { + err = -ENOMEM; + goto err_lrc; + } + + x = 0; + dw = 0; + hw = defaults; + hw += LRC_STATE_OFFSET / sizeof(*hw); + do { + u32 len = hw[dw] & 0x7f; + + if (hw[dw] == 0) { + dw++; + continue; + } + + if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { + dw += len + 2; + continue; + } + + dw++; + len = (len + 1) / 2; + while (len--) { + if (!is_moving(A[0][x], A[1][x]) && + (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { + switch (hw[dw] & 4095) { + case 0x30: /* RING_HEAD */ + case 0x34: /* RING_TAIL */ + break; + + default: + pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", + engine->name, dw, + hw[dw], hw[dw + 1], + A[0][x], B[0][x], B[1][x], + poison, lrc[dw + 1]); + err = -EINVAL; + } + } + dw += 2; + x++; + } + } while (dw < PAGE_SIZE / sizeof(u32) && + (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); + + shmem_unpin_map(ce->engine->default_state, defaults); +err_lrc: + i915_gem_object_unpin_map(ce->state->obj); +err_B1: + i915_gem_object_unpin_map(result[1]->obj); +err_B0: + i915_gem_object_unpin_map(result[0]->obj); +err_A1: + i915_gem_object_unpin_map(ref[1]->obj); +err_A0: + i915_gem_object_unpin_map(ref[0]->obj); + return err; +} + +static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) +{ + u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); + struct i915_vma *ref[2], *result[2]; + struct intel_context *A, *B; + struct i915_request *rq; + int err; + + A = intel_context_create(engine); + if (IS_ERR(A)) + return PTR_ERR(A); + + B = intel_context_create(engine); + if (IS_ERR(B)) { + err = PTR_ERR(B); + goto err_A; + } + + ref[0] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(ref[0])) { + err = PTR_ERR(ref[0]); + goto err_B; + } + + ref[1] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(ref[1])) { + err = PTR_ERR(ref[1]); + goto err_ref0; + } + + rq = record_registers(A, ref[0], ref[1], sema); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_ref1; + } + + WRITE_ONCE(*sema, 1); + wmb(); + + if (i915_request_wait(rq, 0, HZ / 2) < 0) { + i915_request_put(rq); + err = -ETIME; + goto err_ref1; + } + i915_request_put(rq); + + result[0] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(result[0])) { + err = PTR_ERR(result[0]); + goto err_ref1; + } + + result[1] = create_user_vma(A->vm, SZ_64K); + if (IS_ERR(result[1])) { + err = PTR_ERR(result[1]); + goto err_result0; + } + + rq = record_registers(A, result[0], result[1], sema); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_result1; + } + + err = poison_registers(B, poison, sema); + if (err) { + WRITE_ONCE(*sema, -1); + i915_request_put(rq); + goto err_result1; + } + + if (i915_request_wait(rq, 0, HZ / 2) < 0) { + i915_request_put(rq); + err = -ETIME; + goto err_result1; + } + i915_request_put(rq); + + err = compare_isolation(engine, ref, result, A, poison); + +err_result1: + i915_vma_put(result[1]); +err_result0: + i915_vma_put(result[0]); +err_ref1: + i915_vma_put(ref[1]); +err_ref0: + i915_vma_put(ref[0]); +err_B: + intel_context_put(B); +err_A: + intel_context_put(A); + return err; +} + +static bool skip_isolation(const struct intel_engine_cs *engine) +{ + if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9) + return true; + + if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11) + return true; + + return false; +} + +static int live_lrc_isolation(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + const u32 poison[] = { + STACK_MAGIC, + 0x3a3a3a3a, + 0x5c5c5c5c, + 0xffffffff, + 0xffff0000, + }; + int err = 0; + + /* + * Our goal is try and verify that per-context state cannot be + * tampered with by another non-privileged client. + * + * We take the list of context registers from the LRI in the default + * context image and attempt to modify that list from a remote context. + */ + + for_each_engine(engine, gt, id) { + int i; + + /* Just don't even ask */ + if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && + skip_isolation(engine)) + continue; + + intel_engine_pm_get(engine); + for (i = 0; i < ARRAY_SIZE(poison); i++) { + int result; + + result = __lrc_isolation(engine, poison[i]); + if (result && !err) + err = result; + + result = __lrc_isolation(engine, ~poison[i]); + if (result && !err) + err = result; + } + intel_engine_pm_put(engine); + if (igt_flush_test(gt->i915)) { + err = -EIO; + break; + } + } + + return err; +} + +static int indirect_ctx_submit_req(struct intel_context *ce) +{ + struct i915_request *rq; + int err = 0; + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + i915_request_get(rq); + i915_request_add(rq); + + if (i915_request_wait(rq, 0, HZ / 5) < 0) + err = -ETIME; + + i915_request_put(rq); + + return err; +} + +#define CTX_BB_CANARY_OFFSET (3 * 1024) +#define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) + +static u32 * +emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) +{ + *cs++ = MI_STORE_REGISTER_MEM_GEN8 | + MI_SRM_LRM_GLOBAL_GTT | + MI_LRI_LRM_CS_MMIO; + *cs++ = i915_mmio_reg_offset(RING_START(0)); + *cs++ = i915_ggtt_offset(ce->state) + + context_wa_bb_offset(ce) + + CTX_BB_CANARY_OFFSET; + *cs++ = 0; + + return cs; +} + +static void +indirect_ctx_bb_setup(struct intel_context *ce) +{ + u32 *cs = context_indirect_bb(ce); + + cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; + + setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); +} + +static bool check_ring_start(struct intel_context *ce) +{ + const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - + LRC_STATE_OFFSET + context_wa_bb_offset(ce); + + if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) + return true; + + pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", + ctx_bb[CTX_BB_CANARY_INDEX], + ce->lrc_reg_state[CTX_RING_START]); + + return false; +} + +static int indirect_ctx_bb_check(struct intel_context *ce) +{ + int err; + + err = indirect_ctx_submit_req(ce); + if (err) + return err; + + if (!check_ring_start(ce)) + return -EINVAL; + + return 0; +} + +static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) +{ + struct intel_context *a, *b; + int err; + + a = intel_context_create(engine); + if (IS_ERR(a)) + return PTR_ERR(a); + err = intel_context_pin(a); + if (err) + goto put_a; + + b = intel_context_create(engine); + if (IS_ERR(b)) { + err = PTR_ERR(b); + goto unpin_a; + } + err = intel_context_pin(b); + if (err) + goto put_b; + + /* We use the already reserved extra page in context state */ + if (!a->wa_bb_page) { + GEM_BUG_ON(b->wa_bb_page); + GEM_BUG_ON(INTEL_GEN(engine->i915) == 12); + goto unpin_b; + } + + /* + * In order to test that our per context bb is truly per context, + * and executes at the intended spot on context restoring process, + * make the batch store the ring start value to memory. + * As ring start is restored apriori of starting the indirect ctx bb and + * as it will be different for each context, it fits to this purpose. + */ + indirect_ctx_bb_setup(a); + indirect_ctx_bb_setup(b); + + err = indirect_ctx_bb_check(a); + if (err) + goto unpin_b; + + err = indirect_ctx_bb_check(b); + +unpin_b: + intel_context_unpin(b); +put_b: + intel_context_put(b); +unpin_a: + intel_context_unpin(a); +put_a: + intel_context_put(a); + + return err; +} + +static int live_lrc_indirect_ctx_bb(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + int err = 0; + + for_each_engine(engine, gt, id) { + intel_engine_pm_get(engine); + err = __live_lrc_indirect_ctx_bb(engine); + intel_engine_pm_put(engine); + + if (igt_flush_test(gt->i915)) + err = -EIO; + + if (err) + break; + } + + return err; +} + +static void garbage_reset(struct intel_engine_cs *engine, + struct i915_request *rq) +{ + const unsigned int bit = I915_RESET_ENGINE + engine->id; + unsigned long *lock = &engine->gt->reset.flags; + + if (test_and_set_bit(bit, lock)) + return; + + tasklet_disable(&engine->execlists.tasklet); + + if (!rq->fence.error) + intel_engine_reset(engine, NULL); + + tasklet_enable(&engine->execlists.tasklet); + clear_and_wake_up_bit(bit, lock); +} + +static struct i915_request *garbage(struct intel_context *ce, + struct rnd_state *prng) +{ + struct i915_request *rq; + int err; + + err = intel_context_pin(ce); + if (err) + return ERR_PTR(err); + + prandom_bytes_state(prng, + ce->lrc_reg_state, + ce->engine->context_size - + LRC_STATE_OFFSET); + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_unpin; + } + + i915_request_get(rq); + i915_request_add(rq); + return rq; + +err_unpin: + intel_context_unpin(ce); + return ERR_PTR(err); +} + +static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) +{ + struct intel_context *ce; + struct i915_request *hang; + int err = 0; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + hang = garbage(ce, prng); + if (IS_ERR(hang)) { + err = PTR_ERR(hang); + goto err_ce; + } + + if (wait_for_submit(engine, hang, HZ / 2)) { + i915_request_put(hang); + err = -ETIME; + goto err_ce; + } + + intel_context_set_banned(ce); + garbage_reset(engine, hang); + + intel_engine_flush_submission(engine); + if (!hang->fence.error) { + i915_request_put(hang); + pr_err("%s: corrupted context was not reset\n", + engine->name); + err = -EINVAL; + goto err_ce; + } + + if (i915_request_wait(hang, 0, HZ / 2) < 0) { + pr_err("%s: corrupted context did not recover\n", + engine->name); + i915_request_put(hang); + err = -EIO; + goto err_ce; + } + i915_request_put(hang); + +err_ce: + intel_context_put(ce); + return err; +} + +static int live_lrc_garbage(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* + * Verify that we can recover if one context state is completely + * corrupted. + */ + + if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) + return 0; + + for_each_engine(engine, gt, id) { + I915_RND_STATE(prng); + int err = 0, i; + + if (!intel_has_reset_engine(engine->gt)) + continue; + + intel_engine_pm_get(engine); + for (i = 0; i < 3; i++) { + err = __lrc_garbage(engine, &prng); + if (err) + break; + } + intel_engine_pm_put(engine); + + if (igt_flush_test(gt->i915)) + err = -EIO; + if (err) + return err; + } + + return 0; +} + static int __live_pphwsp_runtime(struct intel_engine_cs *engine) { struct intel_context *ce; @@ -4845,8 +6097,11 @@ int intel_lrc_live_selftests(struct drm_i915_private *i915) SUBTEST(live_lrc_fixed), SUBTEST(live_lrc_state), SUBTEST(live_lrc_gpr), + SUBTEST(live_lrc_isolation), SUBTEST(live_lrc_timestamp), + SUBTEST(live_lrc_garbage), SUBTEST(live_pphwsp_runtime), + SUBTEST(live_lrc_indirect_ctx_bb), }; if (!HAS_LOGICAL_RING_CONTEXTS(i915)) diff --git a/drivers/gpu/drm/i915/gt/selftest_rc6.c b/drivers/gpu/drm/i915/gt/selftest_rc6.c index 5f7e2dcf5686..2dc460624bbc 100644 --- a/drivers/gpu/drm/i915/gt/selftest_rc6.c +++ b/drivers/gpu/drm/i915/gt/selftest_rc6.c @@ -11,12 +11,30 @@ #include "selftest_rc6.h" #include "selftests/i915_random.h" +#include "selftests/librapl.h" + +static u64 rc6_residency(struct intel_rc6 *rc6) +{ + u64 result; + + /* XXX VLV_GT_MEDIA_RC6? */ + + result = intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6); + if (HAS_RC6p(rc6_to_i915(rc6))) + result += intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6p); + if (HAS_RC6pp(rc6_to_i915(rc6))) + result += intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6pp); + + return result; +} int live_rc6_manual(void *arg) { struct intel_gt *gt = arg; struct intel_rc6 *rc6 = >->rc6; + u64 rc0_power, rc6_power; intel_wakeref_t wakeref; + ktime_t dt; u64 res[2]; int err = 0; @@ -38,9 +56,14 @@ int live_rc6_manual(void *arg) __intel_rc6_disable(rc6); msleep(1); /* wakeup is not immediate, takes about 100us on icl */ - res[0] = intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6); + res[0] = rc6_residency(rc6); + + dt = ktime_get(); + rc0_power = librapl_energy_uJ(); msleep(250); - res[1] = intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6); + rc0_power = librapl_energy_uJ() - rc0_power; + dt = ktime_sub(ktime_get(), dt); + res[1] = rc6_residency(rc6); if ((res[1] - res[0]) >> 10) { pr_err("RC6 residency increased by %lldus while disabled for 250ms!\n", (res[1] - res[0]) >> 10); @@ -48,13 +71,24 @@ int live_rc6_manual(void *arg) goto out_unlock; } + rc0_power = div64_u64(NSEC_PER_SEC * rc0_power, ktime_to_ns(dt)); + if (!rc0_power) { + pr_err("No power measured while in RC0\n"); + err = -EINVAL; + goto out_unlock; + } + /* Manually enter RC6 */ intel_rc6_park(rc6); - res[0] = intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6); + res[0] = rc6_residency(rc6); + intel_uncore_forcewake_flush(rc6_to_uncore(rc6), FORCEWAKE_ALL); + dt = ktime_get(); + rc6_power = librapl_energy_uJ(); msleep(100); - res[1] = intel_rc6_residency_ns(rc6, GEN6_GT_GFX_RC6); - + rc6_power = librapl_energy_uJ() - rc6_power; + dt = ktime_sub(ktime_get(), dt); + res[1] = rc6_residency(rc6); if (res[1] == res[0]) { pr_err("Did not enter RC6! RC6_STATE=%08x, RC6_CONTROL=%08x, residency=%lld\n", intel_uncore_read_fw(gt->uncore, GEN6_RC_STATE), @@ -63,6 +97,15 @@ int live_rc6_manual(void *arg) err = -EINVAL; } + rc6_power = div64_u64(NSEC_PER_SEC * rc6_power, ktime_to_ns(dt)); + pr_info("GPU consumed %llduW in RC0 and %llduW in RC6\n", + rc0_power, rc6_power); + if (2 * rc6_power > rc0_power) { + pr_err("GPU leaked energy while in RC6!\n"); + err = -EINVAL; + goto out_unlock; + } + /* Restore what should have been the original state! */ intel_rc6_unpark(rc6); diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c index 6ad6aca315f6..35406ecdf0b2 100644 --- a/drivers/gpu/drm/i915/gt/selftest_reset.c +++ b/drivers/gpu/drm/i915/gt/selftest_reset.c @@ -115,7 +115,7 @@ static int igt_atomic_engine_reset(void *arg) if (!intel_has_reset_engine(gt)) return 0; - if (USES_GUC_SUBMISSION(gt->i915)) + if (intel_uc_uses_guc_submission(>->uc)) return 0; intel_gt_pm_get(gt); diff --git a/drivers/gpu/drm/i915/gt/selftest_ring_submission.c b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c new file mode 100644 index 000000000000..3350e7c995bc --- /dev/null +++ b/drivers/gpu/drm/i915/gt/selftest_ring_submission.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "intel_engine_pm.h" +#include "selftests/igt_flush_test.h" + +static struct i915_vma *create_wally(struct intel_engine_cs *engine) +{ + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + u32 *cs; + int err; + + obj = i915_gem_object_create_internal(engine->i915, 4096); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + vma = i915_vma_instance(obj, engine->gt->vm, NULL); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return vma; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH); + if (err) { + i915_gem_object_put(obj); + return ERR_PTR(err); + } + + err = i915_vma_sync(vma); + if (err) { + i915_gem_object_put(obj); + return ERR_PTR(err); + } + + cs = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(cs)) { + i915_gem_object_put(obj); + return ERR_CAST(cs); + } + + if (INTEL_GEN(engine->i915) >= 6) { + *cs++ = MI_STORE_DWORD_IMM_GEN4; + *cs++ = 0; + } else if (INTEL_GEN(engine->i915) >= 4) { + *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; + *cs++ = 0; + } else { + *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; + } + *cs++ = vma->node.start + 4000; + *cs++ = STACK_MAGIC; + + *cs++ = MI_BATCH_BUFFER_END; + + i915_gem_object_flush_map(obj); + i915_gem_object_unpin_map(obj); + + vma->private = intel_context_create(engine); /* dummy residuals */ + if (IS_ERR(vma->private)) { + vma = ERR_CAST(vma->private); + i915_gem_object_put(obj); + } + + return vma; +} + +static int context_sync(struct intel_context *ce) +{ + struct i915_request *rq; + int err = 0; + + rq = intel_context_create_request(ce); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + i915_request_get(rq); + i915_request_add(rq); + + if (i915_request_wait(rq, 0, HZ / 5) < 0) + err = -ETIME; + i915_request_put(rq); + + return err; +} + +static int new_context_sync(struct intel_engine_cs *engine) +{ + struct intel_context *ce; + int err; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + err = context_sync(ce); + intel_context_put(ce); + + return err; +} + +static int mixed_contexts_sync(struct intel_engine_cs *engine, u32 *result) +{ + int pass; + int err; + + for (pass = 0; pass < 2; pass++) { + WRITE_ONCE(*result, 0); + err = context_sync(engine->kernel_context); + if (err || READ_ONCE(*result)) { + if (!err) { + pr_err("pass[%d] wa_bb emitted for the kernel context\n", + pass); + err = -EINVAL; + } + return err; + } + + WRITE_ONCE(*result, 0); + err = new_context_sync(engine); + if (READ_ONCE(*result) != STACK_MAGIC) { + if (!err) { + pr_err("pass[%d] wa_bb *NOT* emitted after the kernel context\n", + pass); + err = -EINVAL; + } + return err; + } + + WRITE_ONCE(*result, 0); + err = new_context_sync(engine); + if (READ_ONCE(*result) != STACK_MAGIC) { + if (!err) { + pr_err("pass[%d] wa_bb *NOT* emitted for the user context switch\n", + pass); + err = -EINVAL; + } + return err; + } + } + + return 0; +} + +static int double_context_sync_00(struct intel_engine_cs *engine, u32 *result) +{ + struct intel_context *ce; + int err, i; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + for (i = 0; i < 2; i++) { + WRITE_ONCE(*result, 0); + err = context_sync(ce); + if (err) + break; + } + intel_context_put(ce); + if (err) + return err; + + if (READ_ONCE(*result)) { + pr_err("wa_bb emitted between the same user context\n"); + return -EINVAL; + } + + return 0; +} + +static int kernel_context_sync_00(struct intel_engine_cs *engine, u32 *result) +{ + struct intel_context *ce; + int err, i; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + for (i = 0; i < 2; i++) { + WRITE_ONCE(*result, 0); + err = context_sync(ce); + if (err) + break; + + err = context_sync(engine->kernel_context); + if (err) + break; + } + intel_context_put(ce); + if (err) + return err; + + if (READ_ONCE(*result)) { + pr_err("wa_bb emitted between the same user context [with intervening kernel]\n"); + return -EINVAL; + } + + return 0; +} + +static int __live_ctx_switch_wa(struct intel_engine_cs *engine) +{ + struct i915_vma *bb; + u32 *result; + int err; + + bb = create_wally(engine); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + result = i915_gem_object_pin_map(bb->obj, I915_MAP_WC); + if (IS_ERR(result)) { + intel_context_put(bb->private); + i915_vma_unpin_and_release(&bb, 0); + return PTR_ERR(result); + } + result += 1000; + + engine->wa_ctx.vma = bb; + + err = mixed_contexts_sync(engine, result); + if (err) + goto out; + + err = double_context_sync_00(engine, result); + if (err) + goto out; + + err = kernel_context_sync_00(engine, result); + if (err) + goto out; + +out: + intel_context_put(engine->wa_ctx.vma->private); + i915_vma_unpin_and_release(&engine->wa_ctx.vma, I915_VMA_RELEASE_MAP); + return err; +} + +static int live_ctx_switch_wa(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_engine_cs *engine; + enum intel_engine_id id; + + /* + * Exercise the inter-context wa batch. + * + * Between each user context we run a wa batch, and since it may + * have implications for user visible state, we have to check that + * we do actually execute it. + * + * The trick we use is to replace the normal wa batch with a custom + * one that writes to a marker within it, and we can then look for + * that marker to confirm if the batch was run when we expect it, + * and equally important it was wasn't run when we don't! + */ + + for_each_engine(engine, gt, id) { + struct i915_vma *saved_wa; + int err; + + if (!intel_engine_can_store_dword(engine)) + continue; + + if (IS_GEN_RANGE(gt->i915, 4, 5)) + continue; /* MI_STORE_DWORD is privileged! */ + + saved_wa = fetch_and_zero(&engine->wa_ctx.vma); + + intel_engine_pm_get(engine); + err = __live_ctx_switch_wa(engine); + intel_engine_pm_put(engine); + if (igt_flush_test(gt->i915)) + err = -EIO; + + engine->wa_ctx.vma = saved_wa; + if (err) + return err; + } + + return 0; +} + +int intel_ring_submission_live_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(live_ctx_switch_wa), + }; + + if (HAS_EXECLISTS(i915)) + return 0; + + return intel_gt_live_subtests(tests, &i915->gt); +} diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.c b/drivers/gpu/drm/i915/gt/selftest_rps.c new file mode 100644 index 000000000000..6275d69aa9cc --- /dev/null +++ b/drivers/gpu/drm/i915/gt/selftest_rps.c @@ -0,0 +1,1331 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <linux/pm_qos.h> +#include <linux/sort.h> + +#include "intel_engine_heartbeat.h" +#include "intel_engine_pm.h" +#include "intel_gpu_commands.h" +#include "intel_gt_clock_utils.h" +#include "intel_gt_pm.h" +#include "intel_rc6.h" +#include "selftest_rps.h" +#include "selftests/igt_flush_test.h" +#include "selftests/igt_spinner.h" +#include "selftests/librapl.h" + +/* Try to isolate the impact of cstates from determing frequency response */ +#define CPU_LATENCY 0 /* -1 to disable pm_qos, 0 to disable cstates */ + +static unsigned long engine_heartbeat_disable(struct intel_engine_cs *engine) +{ + unsigned long old; + + old = fetch_and_zero(&engine->props.heartbeat_interval_ms); + + intel_engine_pm_get(engine); + intel_engine_park_heartbeat(engine); + + return old; +} + +static void engine_heartbeat_enable(struct intel_engine_cs *engine, + unsigned long saved) +{ + intel_engine_pm_put(engine); + + engine->props.heartbeat_interval_ms = saved; +} + +static void dummy_rps_work(struct work_struct *wrk) +{ +} + +static int cmp_u64(const void *A, const void *B) +{ + const u64 *a = A, *b = B; + + if (a < b) + return -1; + else if (a > b) + return 1; + else + return 0; +} + +static int cmp_u32(const void *A, const void *B) +{ + const u32 *a = A, *b = B; + + if (a < b) + return -1; + else if (a > b) + return 1; + else + return 0; +} + +static struct i915_vma * +create_spin_counter(struct intel_engine_cs *engine, + struct i915_address_space *vm, + bool srm, + u32 **cancel, + u32 **counter) +{ + enum { + COUNT, + INC, + __NGPR__, + }; +#define CS_GPR(x) GEN8_RING_CS_GPR(engine->mmio_base, x) + struct drm_i915_gem_object *obj; + struct i915_vma *vma; + unsigned long end; + u32 *base, *cs; + int loop, i; + int err; + + obj = i915_gem_object_create_internal(vm->i915, 64 << 10); + if (IS_ERR(obj)) + return ERR_CAST(obj); + + end = obj->base.size / sizeof(u32) - 1; + + vma = i915_vma_instance(obj, vm, NULL); + if (IS_ERR(vma)) { + i915_gem_object_put(obj); + return vma; + } + + err = i915_vma_pin(vma, 0, 0, PIN_USER); + if (err) { + i915_vma_put(vma); + return ERR_PTR(err); + } + + base = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(base)) { + i915_gem_object_put(obj); + return ERR_CAST(base); + } + cs = base; + + *cs++ = MI_LOAD_REGISTER_IMM(__NGPR__ * 2); + for (i = 0; i < __NGPR__; i++) { + *cs++ = i915_mmio_reg_offset(CS_GPR(i)); + *cs++ = 0; + *cs++ = i915_mmio_reg_offset(CS_GPR(i)) + 4; + *cs++ = 0; + } + + *cs++ = MI_LOAD_REGISTER_IMM(1); + *cs++ = i915_mmio_reg_offset(CS_GPR(INC)); + *cs++ = 1; + + loop = cs - base; + + /* Unroll the loop to avoid MI_BB_START stalls impacting measurements */ + for (i = 0; i < 1024; i++) { + *cs++ = MI_MATH(4); + *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(COUNT)); + *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(INC)); + *cs++ = MI_MATH_ADD; + *cs++ = MI_MATH_STORE(MI_MATH_REG(COUNT), MI_MATH_REG_ACCU); + + if (srm) { + *cs++ = MI_STORE_REGISTER_MEM_GEN8; + *cs++ = i915_mmio_reg_offset(CS_GPR(COUNT)); + *cs++ = lower_32_bits(vma->node.start + end * sizeof(*cs)); + *cs++ = upper_32_bits(vma->node.start + end * sizeof(*cs)); + } + } + + *cs++ = MI_BATCH_BUFFER_START_GEN8; + *cs++ = lower_32_bits(vma->node.start + loop * sizeof(*cs)); + *cs++ = upper_32_bits(vma->node.start + loop * sizeof(*cs)); + GEM_BUG_ON(cs - base > end); + + i915_gem_object_flush_map(obj); + + *cancel = base + loop; + *counter = srm ? memset32(base + end, 0, 1) : NULL; + return vma; +} + +static u8 wait_for_freq(struct intel_rps *rps, u8 freq, int timeout_ms) +{ + u8 history[64], i; + unsigned long end; + int sleep; + + i = 0; + memset(history, freq, sizeof(history)); + sleep = 20; + + /* The PCU does not change instantly, but drifts towards the goal? */ + end = jiffies + msecs_to_jiffies(timeout_ms); + do { + u8 act; + + act = read_cagf(rps); + if (time_after(jiffies, end)) + return act; + + /* Target acquired */ + if (act == freq) + return act; + + /* Any change within the last N samples? */ + if (!memchr_inv(history, act, sizeof(history))) + return act; + + history[i] = act; + i = (i + 1) % ARRAY_SIZE(history); + + usleep_range(sleep, 2 * sleep); + sleep *= 2; + if (sleep > timeout_ms * 20) + sleep = timeout_ms * 20; + } while (1); +} + +static u8 rps_set_check(struct intel_rps *rps, u8 freq) +{ + mutex_lock(&rps->lock); + GEM_BUG_ON(!intel_rps_is_active(rps)); + intel_rps_set(rps, freq); + GEM_BUG_ON(rps->last_freq != freq); + mutex_unlock(&rps->lock); + + return wait_for_freq(rps, freq, 50); +} + +static void show_pstate_limits(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + + if (IS_BROXTON(i915)) { + pr_info("P_STATE_CAP[%x]: 0x%08x\n", + i915_mmio_reg_offset(BXT_RP_STATE_CAP), + intel_uncore_read(rps_to_uncore(rps), + BXT_RP_STATE_CAP)); + } else if (IS_GEN(i915, 9)) { + pr_info("P_STATE_LIMITS[%x]: 0x%08x\n", + i915_mmio_reg_offset(GEN9_RP_STATE_LIMITS), + intel_uncore_read(rps_to_uncore(rps), + GEN9_RP_STATE_LIMITS)); + } +} + +int live_rps_clock_interval(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + void (*saved_work)(struct work_struct *wrk); + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + int err = 0; + + if (!intel_rps_is_enabled(rps)) + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + intel_gt_pm_wait_for_idle(gt); + saved_work = rps->work.func; + rps->work.func = dummy_rps_work; + + intel_gt_pm_get(gt); + intel_rps_disable(>->rps); + + intel_gt_check_clock_frequency(gt); + + for_each_engine(engine, gt, id) { + unsigned long saved_heartbeat; + struct i915_request *rq; + u32 cycles; + u64 dt; + + if (!intel_engine_can_store_dword(engine)) + continue; + + saved_heartbeat = engine_heartbeat_disable(engine); + + rq = igt_spinner_create_request(&spin, + engine->kernel_context, + MI_NOOP); + if (IS_ERR(rq)) { + engine_heartbeat_enable(engine, saved_heartbeat); + err = PTR_ERR(rq); + break; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + pr_err("%s: RPS spinner did not start\n", + engine->name); + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + intel_gt_set_wedged(engine->gt); + err = -EIO; + break; + } + + intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); + + intel_uncore_write_fw(gt->uncore, GEN6_RP_CUR_UP_EI, 0); + + /* Set the evaluation interval to infinity! */ + intel_uncore_write_fw(gt->uncore, + GEN6_RP_UP_EI, 0xffffffff); + intel_uncore_write_fw(gt->uncore, + GEN6_RP_UP_THRESHOLD, 0xffffffff); + + intel_uncore_write_fw(gt->uncore, GEN6_RP_CONTROL, + GEN6_RP_ENABLE | GEN6_RP_UP_BUSY_AVG); + + if (wait_for(intel_uncore_read_fw(gt->uncore, + GEN6_RP_CUR_UP_EI), + 10)) { + /* Just skip the test; assume lack of HW support */ + pr_notice("%s: rps evaluation interval not ticking\n", + engine->name); + err = -ENODEV; + } else { + ktime_t dt_[5]; + u32 cycles_[5]; + int i; + + for (i = 0; i < 5; i++) { + preempt_disable(); + + dt_[i] = ktime_get(); + cycles_[i] = -intel_uncore_read_fw(gt->uncore, GEN6_RP_CUR_UP_EI); + + udelay(1000); + + dt_[i] = ktime_sub(ktime_get(), dt_[i]); + cycles_[i] += intel_uncore_read_fw(gt->uncore, GEN6_RP_CUR_UP_EI); + + preempt_enable(); + } + + /* Use the median of both cycle/dt; close enough */ + sort(cycles_, 5, sizeof(*cycles_), cmp_u32, NULL); + cycles = (cycles_[1] + 2 * cycles_[2] + cycles_[3]) / 4; + sort(dt_, 5, sizeof(*dt_), cmp_u64, NULL); + dt = div_u64(dt_[1] + 2 * dt_[2] + dt_[3], 4); + } + + intel_uncore_write_fw(gt->uncore, GEN6_RP_CONTROL, 0); + intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); + + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + + if (err == 0) { + u64 time = intel_gt_pm_interval_to_ns(gt, cycles); + u32 expected = + intel_gt_ns_to_pm_interval(gt, dt); + + pr_info("%s: rps counted %d C0 cycles [%lldns] in %lldns [%d cycles], using GT clock frequency of %uKHz\n", + engine->name, cycles, time, dt, expected, + gt->clock_frequency / 1000); + + if (10 * time < 8 * dt || + 8 * time > 10 * dt) { + pr_err("%s: rps clock time does not match walltime!\n", + engine->name); + err = -EINVAL; + } + + if (10 * expected < 8 * cycles || + 8 * expected > 10 * cycles) { + pr_err("%s: walltime does not match rps clock ticks!\n", + engine->name); + err = -EINVAL; + } + } + + if (igt_flush_test(gt->i915)) + err = -EIO; + + break; /* once is enough */ + } + + intel_rps_enable(>->rps); + intel_gt_pm_put(gt); + + igt_spinner_fini(&spin); + + intel_gt_pm_wait_for_idle(gt); + rps->work.func = saved_work; + + if (err == -ENODEV) /* skipped, don't report a fail */ + err = 0; + + return err; +} + +int live_rps_control(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + void (*saved_work)(struct work_struct *wrk); + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + int err = 0; + + /* + * Check that the actual frequency matches our requested frequency, + * to verify our control mechanism. We have to be careful that the + * PCU may throttle the GPU in which case the actual frequency used + * will be lowered than requested. + */ + + if (!intel_rps_is_enabled(rps)) + return 0; + + if (IS_CHERRYVIEW(gt->i915)) /* XXX fragile PCU */ + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + intel_gt_pm_wait_for_idle(gt); + saved_work = rps->work.func; + rps->work.func = dummy_rps_work; + + intel_gt_pm_get(gt); + for_each_engine(engine, gt, id) { + unsigned long saved_heartbeat; + struct i915_request *rq; + ktime_t min_dt, max_dt; + int f, limit; + int min, max; + + if (!intel_engine_can_store_dword(engine)) + continue; + + saved_heartbeat = engine_heartbeat_disable(engine); + + rq = igt_spinner_create_request(&spin, + engine->kernel_context, + MI_NOOP); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + pr_err("%s: RPS spinner did not start\n", + engine->name); + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + intel_gt_set_wedged(engine->gt); + err = -EIO; + break; + } + + if (rps_set_check(rps, rps->min_freq) != rps->min_freq) { + pr_err("%s: could not set minimum frequency [%x], only %x!\n", + engine->name, rps->min_freq, read_cagf(rps)); + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + show_pstate_limits(rps); + err = -EINVAL; + break; + } + + for (f = rps->min_freq + 1; f < rps->max_freq; f++) { + if (rps_set_check(rps, f) < f) + break; + } + + limit = rps_set_check(rps, f); + + if (rps_set_check(rps, rps->min_freq) != rps->min_freq) { + pr_err("%s: could not restore minimum frequency [%x], only %x!\n", + engine->name, rps->min_freq, read_cagf(rps)); + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + show_pstate_limits(rps); + err = -EINVAL; + break; + } + + max_dt = ktime_get(); + max = rps_set_check(rps, limit); + max_dt = ktime_sub(ktime_get(), max_dt); + + min_dt = ktime_get(); + min = rps_set_check(rps, rps->min_freq); + min_dt = ktime_sub(ktime_get(), min_dt); + + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + + pr_info("%s: range:[%x:%uMHz, %x:%uMHz] limit:[%x:%uMHz], %x:%x response %lluns:%lluns\n", + engine->name, + rps->min_freq, intel_gpu_freq(rps, rps->min_freq), + rps->max_freq, intel_gpu_freq(rps, rps->max_freq), + limit, intel_gpu_freq(rps, limit), + min, max, ktime_to_ns(min_dt), ktime_to_ns(max_dt)); + + if (limit == rps->min_freq) { + pr_err("%s: GPU throttled to minimum!\n", + engine->name); + show_pstate_limits(rps); + err = -ENODEV; + break; + } + + if (igt_flush_test(gt->i915)) { + err = -EIO; + break; + } + } + intel_gt_pm_put(gt); + + igt_spinner_fini(&spin); + + intel_gt_pm_wait_for_idle(gt); + rps->work.func = saved_work; + + return err; +} + +static void show_pcu_config(struct intel_rps *rps) +{ + struct drm_i915_private *i915 = rps_to_i915(rps); + unsigned int max_gpu_freq, min_gpu_freq; + intel_wakeref_t wakeref; + int gpu_freq; + + if (!HAS_LLC(i915)) + return; + + min_gpu_freq = rps->min_freq; + max_gpu_freq = rps->max_freq; + if (INTEL_GEN(i915) >= 9) { + /* Convert GT frequency to 50 HZ units */ + min_gpu_freq /= GEN9_FREQ_SCALER; + max_gpu_freq /= GEN9_FREQ_SCALER; + } + + wakeref = intel_runtime_pm_get(rps_to_uncore(rps)->rpm); + + pr_info("%5s %5s %5s\n", "GPU", "eCPU", "eRing"); + for (gpu_freq = min_gpu_freq; gpu_freq <= max_gpu_freq; gpu_freq++) { + int ia_freq = gpu_freq; + + sandybridge_pcode_read(i915, + GEN6_PCODE_READ_MIN_FREQ_TABLE, + &ia_freq, NULL); + + pr_info("%5d %5d %5d\n", + gpu_freq * 50, + ((ia_freq >> 0) & 0xff) * 100, + ((ia_freq >> 8) & 0xff) * 100); + } + + intel_runtime_pm_put(rps_to_uncore(rps)->rpm, wakeref); +} + +static u64 __measure_frequency(u32 *cntr, int duration_ms) +{ + u64 dc, dt; + + dt = ktime_get(); + dc = READ_ONCE(*cntr); + usleep_range(1000 * duration_ms, 2000 * duration_ms); + dc = READ_ONCE(*cntr) - dc; + dt = ktime_get() - dt; + + return div64_u64(1000 * 1000 * dc, dt); +} + +static u64 measure_frequency_at(struct intel_rps *rps, u32 *cntr, int *freq) +{ + u64 x[5]; + int i; + + *freq = rps_set_check(rps, *freq); + for (i = 0; i < 5; i++) + x[i] = __measure_frequency(cntr, 2); + *freq = (*freq + read_cagf(rps)) / 2; + + /* A simple triangle filter for better result stability */ + sort(x, 5, sizeof(*x), cmp_u64, NULL); + return div_u64(x[1] + 2 * x[2] + x[3], 4); +} + +static u64 __measure_cs_frequency(struct intel_engine_cs *engine, + int duration_ms) +{ + u64 dc, dt; + + dt = ktime_get(); + dc = intel_uncore_read_fw(engine->uncore, CS_GPR(0)); + usleep_range(1000 * duration_ms, 2000 * duration_ms); + dc = intel_uncore_read_fw(engine->uncore, CS_GPR(0)) - dc; + dt = ktime_get() - dt; + + return div64_u64(1000 * 1000 * dc, dt); +} + +static u64 measure_cs_frequency_at(struct intel_rps *rps, + struct intel_engine_cs *engine, + int *freq) +{ + u64 x[5]; + int i; + + *freq = rps_set_check(rps, *freq); + for (i = 0; i < 5; i++) + x[i] = __measure_cs_frequency(engine, 2); + *freq = (*freq + read_cagf(rps)) / 2; + + /* A simple triangle filter for better result stability */ + sort(x, 5, sizeof(*x), cmp_u64, NULL); + return div_u64(x[1] + 2 * x[2] + x[3], 4); +} + +static bool scaled_within(u64 x, u64 y, u32 f_n, u32 f_d) +{ + return f_d * x > f_n * y && f_n * x < f_d * y; +} + +int live_rps_frequency_cs(void *arg) +{ + void (*saved_work)(struct work_struct *wrk); + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + struct intel_engine_cs *engine; + struct pm_qos_request qos; + enum intel_engine_id id; + int err = 0; + + /* + * The premise is that the GPU does change freqency at our behest. + * Let's check there is a correspondence between the requested + * frequency, the actual frequency, and the observed clock rate. + */ + + if (!intel_rps_is_enabled(rps)) + return 0; + + if (INTEL_GEN(gt->i915) < 8) /* for CS simplicity */ + return 0; + + if (CPU_LATENCY >= 0) + cpu_latency_qos_add_request(&qos, CPU_LATENCY); + + intel_gt_pm_wait_for_idle(gt); + saved_work = rps->work.func; + rps->work.func = dummy_rps_work; + + for_each_engine(engine, gt, id) { + unsigned long saved_heartbeat; + struct i915_request *rq; + struct i915_vma *vma; + u32 *cancel, *cntr; + struct { + u64 count; + int freq; + } min, max; + + saved_heartbeat = engine_heartbeat_disable(engine); + + vma = create_spin_counter(engine, + engine->kernel_context->vm, false, + &cancel, &cntr); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + engine_heartbeat_enable(engine, saved_heartbeat); + break; + } + + rq = intel_engine_create_kernel_request(engine); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_vma; + } + + i915_vma_lock(vma); + err = i915_request_await_object(rq, vma->obj, false); + if (!err) + err = i915_vma_move_to_active(vma, rq, 0); + if (!err) + err = rq->engine->emit_bb_start(rq, + vma->node.start, + PAGE_SIZE, 0); + i915_vma_unlock(vma); + i915_request_add(rq); + if (err) + goto err_vma; + + if (wait_for(intel_uncore_read(engine->uncore, CS_GPR(0)), + 10)) { + pr_err("%s: timed loop did not start\n", + engine->name); + goto err_vma; + } + + min.freq = rps->min_freq; + min.count = measure_cs_frequency_at(rps, engine, &min.freq); + + max.freq = rps->max_freq; + max.count = measure_cs_frequency_at(rps, engine, &max.freq); + + pr_info("%s: min:%lluKHz @ %uMHz, max:%lluKHz @ %uMHz [%d%%]\n", + engine->name, + min.count, intel_gpu_freq(rps, min.freq), + max.count, intel_gpu_freq(rps, max.freq), + (int)DIV64_U64_ROUND_CLOSEST(100 * min.freq * max.count, + max.freq * min.count)); + + if (!scaled_within(max.freq * min.count, + min.freq * max.count, + 2, 3)) { + int f; + + pr_err("%s: CS did not scale with frequency! scaled min:%llu, max:%llu\n", + engine->name, + max.freq * min.count, + min.freq * max.count); + show_pcu_config(rps); + + for (f = min.freq + 1; f <= rps->max_freq; f++) { + int act = f; + u64 count; + + count = measure_cs_frequency_at(rps, engine, &act); + if (act < f) + break; + + pr_info("%s: %x:%uMHz: %lluKHz [%d%%]\n", + engine->name, + act, intel_gpu_freq(rps, act), count, + (int)DIV64_U64_ROUND_CLOSEST(100 * min.freq * count, + act * min.count)); + + f = act; /* may skip ahead [pcu granularity] */ + } + + err = -EINVAL; + } + +err_vma: + *cancel = MI_BATCH_BUFFER_END; + i915_gem_object_flush_map(vma->obj); + i915_gem_object_unpin_map(vma->obj); + i915_vma_unpin(vma); + i915_vma_put(vma); + + engine_heartbeat_enable(engine, saved_heartbeat); + if (igt_flush_test(gt->i915)) + err = -EIO; + if (err) + break; + } + + intel_gt_pm_wait_for_idle(gt); + rps->work.func = saved_work; + + if (CPU_LATENCY >= 0) + cpu_latency_qos_remove_request(&qos); + + return err; +} + +int live_rps_frequency_srm(void *arg) +{ + void (*saved_work)(struct work_struct *wrk); + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + struct intel_engine_cs *engine; + struct pm_qos_request qos; + enum intel_engine_id id; + int err = 0; + + /* + * The premise is that the GPU does change freqency at our behest. + * Let's check there is a correspondence between the requested + * frequency, the actual frequency, and the observed clock rate. + */ + + if (!intel_rps_is_enabled(rps)) + return 0; + + if (INTEL_GEN(gt->i915) < 8) /* for CS simplicity */ + return 0; + + if (CPU_LATENCY >= 0) + cpu_latency_qos_add_request(&qos, CPU_LATENCY); + + intel_gt_pm_wait_for_idle(gt); + saved_work = rps->work.func; + rps->work.func = dummy_rps_work; + + for_each_engine(engine, gt, id) { + unsigned long saved_heartbeat; + struct i915_request *rq; + struct i915_vma *vma; + u32 *cancel, *cntr; + struct { + u64 count; + int freq; + } min, max; + + saved_heartbeat = engine_heartbeat_disable(engine); + + vma = create_spin_counter(engine, + engine->kernel_context->vm, true, + &cancel, &cntr); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + engine_heartbeat_enable(engine, saved_heartbeat); + break; + } + + rq = intel_engine_create_kernel_request(engine); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err_vma; + } + + i915_vma_lock(vma); + err = i915_request_await_object(rq, vma->obj, false); + if (!err) + err = i915_vma_move_to_active(vma, rq, 0); + if (!err) + err = rq->engine->emit_bb_start(rq, + vma->node.start, + PAGE_SIZE, 0); + i915_vma_unlock(vma); + i915_request_add(rq); + if (err) + goto err_vma; + + if (wait_for(READ_ONCE(*cntr), 10)) { + pr_err("%s: timed loop did not start\n", + engine->name); + goto err_vma; + } + + min.freq = rps->min_freq; + min.count = measure_frequency_at(rps, cntr, &min.freq); + + max.freq = rps->max_freq; + max.count = measure_frequency_at(rps, cntr, &max.freq); + + pr_info("%s: min:%lluKHz @ %uMHz, max:%lluKHz @ %uMHz [%d%%]\n", + engine->name, + min.count, intel_gpu_freq(rps, min.freq), + max.count, intel_gpu_freq(rps, max.freq), + (int)DIV64_U64_ROUND_CLOSEST(100 * min.freq * max.count, + max.freq * min.count)); + + if (!scaled_within(max.freq * min.count, + min.freq * max.count, + 1, 2)) { + int f; + + pr_err("%s: CS did not scale with frequency! scaled min:%llu, max:%llu\n", + engine->name, + max.freq * min.count, + min.freq * max.count); + show_pcu_config(rps); + + for (f = min.freq + 1; f <= rps->max_freq; f++) { + int act = f; + u64 count; + + count = measure_frequency_at(rps, cntr, &act); + if (act < f) + break; + + pr_info("%s: %x:%uMHz: %lluKHz [%d%%]\n", + engine->name, + act, intel_gpu_freq(rps, act), count, + (int)DIV64_U64_ROUND_CLOSEST(100 * min.freq * count, + act * min.count)); + + f = act; /* may skip ahead [pcu granularity] */ + } + + err = -EINVAL; + } + +err_vma: + *cancel = MI_BATCH_BUFFER_END; + i915_gem_object_flush_map(vma->obj); + i915_gem_object_unpin_map(vma->obj); + i915_vma_unpin(vma); + i915_vma_put(vma); + + engine_heartbeat_enable(engine, saved_heartbeat); + if (igt_flush_test(gt->i915)) + err = -EIO; + if (err) + break; + } + + intel_gt_pm_wait_for_idle(gt); + rps->work.func = saved_work; + + if (CPU_LATENCY >= 0) + cpu_latency_qos_remove_request(&qos); + + return err; +} + +static void sleep_for_ei(struct intel_rps *rps, int timeout_us) +{ + /* Flush any previous EI */ + usleep_range(timeout_us, 2 * timeout_us); + + /* Reset the interrupt status */ + rps_disable_interrupts(rps); + GEM_BUG_ON(rps->pm_iir); + rps_enable_interrupts(rps); + + /* And then wait for the timeout, for real this time */ + usleep_range(2 * timeout_us, 3 * timeout_us); +} + +static int __rps_up_interrupt(struct intel_rps *rps, + struct intel_engine_cs *engine, + struct igt_spinner *spin) +{ + struct intel_uncore *uncore = engine->uncore; + struct i915_request *rq; + u32 timeout; + + if (!intel_engine_can_store_dword(engine)) + return 0; + + rps_set_check(rps, rps->min_freq); + + rq = igt_spinner_create_request(spin, engine->kernel_context, MI_NOOP); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + i915_request_get(rq); + i915_request_add(rq); + + if (!igt_wait_for_spinner(spin, rq)) { + pr_err("%s: RPS spinner did not start\n", + engine->name); + i915_request_put(rq); + intel_gt_set_wedged(engine->gt); + return -EIO; + } + + if (!intel_rps_is_active(rps)) { + pr_err("%s: RPS not enabled on starting spinner\n", + engine->name); + igt_spinner_end(spin); + i915_request_put(rq); + return -EINVAL; + } + + if (!(rps->pm_events & GEN6_PM_RP_UP_THRESHOLD)) { + pr_err("%s: RPS did not register UP interrupt\n", + engine->name); + i915_request_put(rq); + return -EINVAL; + } + + if (rps->last_freq != rps->min_freq) { + pr_err("%s: RPS did not program min frequency\n", + engine->name); + i915_request_put(rq); + return -EINVAL; + } + + timeout = intel_uncore_read(uncore, GEN6_RP_UP_EI); + timeout = intel_gt_pm_interval_to_ns(engine->gt, timeout); + timeout = DIV_ROUND_UP(timeout, 1000); + + sleep_for_ei(rps, timeout); + GEM_BUG_ON(i915_request_completed(rq)); + + igt_spinner_end(spin); + i915_request_put(rq); + + if (rps->cur_freq != rps->min_freq) { + pr_err("%s: Frequency unexpectedly changed [up], now %d!\n", + engine->name, intel_rps_read_actual_frequency(rps)); + return -EINVAL; + } + + if (!(rps->pm_iir & GEN6_PM_RP_UP_THRESHOLD)) { + pr_err("%s: UP interrupt not recorded for spinner, pm_iir:%x, prev_up:%x, up_threshold:%x, up_ei:%x\n", + engine->name, rps->pm_iir, + intel_uncore_read(uncore, GEN6_RP_PREV_UP), + intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD), + intel_uncore_read(uncore, GEN6_RP_UP_EI)); + return -EINVAL; + } + + return 0; +} + +static int __rps_down_interrupt(struct intel_rps *rps, + struct intel_engine_cs *engine) +{ + struct intel_uncore *uncore = engine->uncore; + u32 timeout; + + rps_set_check(rps, rps->max_freq); + + if (!(rps->pm_events & GEN6_PM_RP_DOWN_THRESHOLD)) { + pr_err("%s: RPS did not register DOWN interrupt\n", + engine->name); + return -EINVAL; + } + + if (rps->last_freq != rps->max_freq) { + pr_err("%s: RPS did not program max frequency\n", + engine->name); + return -EINVAL; + } + + timeout = intel_uncore_read(uncore, GEN6_RP_DOWN_EI); + timeout = intel_gt_pm_interval_to_ns(engine->gt, timeout); + timeout = DIV_ROUND_UP(timeout, 1000); + + sleep_for_ei(rps, timeout); + + if (rps->cur_freq != rps->max_freq) { + pr_err("%s: Frequency unexpectedly changed [down], now %d!\n", + engine->name, + intel_rps_read_actual_frequency(rps)); + return -EINVAL; + } + + if (!(rps->pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT))) { + pr_err("%s: DOWN interrupt not recorded for idle, pm_iir:%x, prev_down:%x, down_threshold:%x, down_ei:%x [prev_up:%x, up_threshold:%x, up_ei:%x]\n", + engine->name, rps->pm_iir, + intel_uncore_read(uncore, GEN6_RP_PREV_DOWN), + intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD), + intel_uncore_read(uncore, GEN6_RP_DOWN_EI), + intel_uncore_read(uncore, GEN6_RP_PREV_UP), + intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD), + intel_uncore_read(uncore, GEN6_RP_UP_EI)); + return -EINVAL; + } + + return 0; +} + +int live_rps_interrupt(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + void (*saved_work)(struct work_struct *wrk); + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + u32 pm_events; + int err = 0; + + /* + * First, let's check whether or not we are receiving interrupts. + */ + + if (!intel_rps_has_interrupts(rps)) + return 0; + + intel_gt_pm_get(gt); + pm_events = rps->pm_events; + intel_gt_pm_put(gt); + if (!pm_events) { + pr_err("No RPS PM events registered, but RPS is enabled?\n"); + return -ENODEV; + } + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + intel_gt_pm_wait_for_idle(gt); + saved_work = rps->work.func; + rps->work.func = dummy_rps_work; + + for_each_engine(engine, gt, id) { + /* Keep the engine busy with a spinner; expect an UP! */ + if (pm_events & GEN6_PM_RP_UP_THRESHOLD) { + unsigned long saved_heartbeat; + + intel_gt_pm_wait_for_idle(engine->gt); + GEM_BUG_ON(intel_rps_is_active(rps)); + + saved_heartbeat = engine_heartbeat_disable(engine); + + err = __rps_up_interrupt(rps, engine, &spin); + + engine_heartbeat_enable(engine, saved_heartbeat); + if (err) + goto out; + + intel_gt_pm_wait_for_idle(engine->gt); + } + + /* Keep the engine awake but idle and check for DOWN */ + if (pm_events & GEN6_PM_RP_DOWN_THRESHOLD) { + unsigned long saved_heartbeat; + + saved_heartbeat = engine_heartbeat_disable(engine); + intel_rc6_disable(>->rc6); + + err = __rps_down_interrupt(rps, engine); + + intel_rc6_enable(>->rc6); + engine_heartbeat_enable(engine, saved_heartbeat); + if (err) + goto out; + } + } + +out: + if (igt_flush_test(gt->i915)) + err = -EIO; + + igt_spinner_fini(&spin); + + intel_gt_pm_wait_for_idle(gt); + rps->work.func = saved_work; + + return err; +} + +static u64 __measure_power(int duration_ms) +{ + u64 dE, dt; + + dt = ktime_get(); + dE = librapl_energy_uJ(); + usleep_range(1000 * duration_ms, 2000 * duration_ms); + dE = librapl_energy_uJ() - dE; + dt = ktime_get() - dt; + + return div64_u64(1000 * 1000 * dE, dt); +} + +static u64 measure_power_at(struct intel_rps *rps, int *freq) +{ + u64 x[5]; + int i; + + *freq = rps_set_check(rps, *freq); + for (i = 0; i < 5; i++) + x[i] = __measure_power(5); + *freq = (*freq + read_cagf(rps)) / 2; + + /* A simple triangle filter for better result stability */ + sort(x, 5, sizeof(*x), cmp_u64, NULL); + return div_u64(x[1] + 2 * x[2] + x[3], 4); +} + +int live_rps_power(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + void (*saved_work)(struct work_struct *wrk); + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + int err = 0; + + /* + * Our fundamental assumption is that running at lower frequency + * actually saves power. Let's see if our RAPL measurement support + * that theory. + */ + + if (!intel_rps_is_enabled(rps)) + return 0; + + if (!librapl_energy_uJ()) + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + intel_gt_pm_wait_for_idle(gt); + saved_work = rps->work.func; + rps->work.func = dummy_rps_work; + + for_each_engine(engine, gt, id) { + unsigned long saved_heartbeat; + struct i915_request *rq; + struct { + u64 power; + int freq; + } min, max; + + if (!intel_engine_can_store_dword(engine)) + continue; + + saved_heartbeat = engine_heartbeat_disable(engine); + + rq = igt_spinner_create_request(&spin, + engine->kernel_context, + MI_NOOP); + if (IS_ERR(rq)) { + engine_heartbeat_enable(engine, saved_heartbeat); + err = PTR_ERR(rq); + break; + } + + i915_request_add(rq); + + if (!igt_wait_for_spinner(&spin, rq)) { + pr_err("%s: RPS spinner did not start\n", + engine->name); + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + intel_gt_set_wedged(engine->gt); + err = -EIO; + break; + } + + max.freq = rps->max_freq; + max.power = measure_power_at(rps, &max.freq); + + min.freq = rps->min_freq; + min.power = measure_power_at(rps, &min.freq); + + igt_spinner_end(&spin); + engine_heartbeat_enable(engine, saved_heartbeat); + + pr_info("%s: min:%llumW @ %uMHz, max:%llumW @ %uMHz\n", + engine->name, + min.power, intel_gpu_freq(rps, min.freq), + max.power, intel_gpu_freq(rps, max.freq)); + + if (10 * min.freq >= 9 * max.freq) { + pr_notice("Could not control frequency, ran at [%d:%uMHz, %d:%uMhz]\n", + min.freq, intel_gpu_freq(rps, min.freq), + max.freq, intel_gpu_freq(rps, max.freq)); + continue; + } + + if (11 * min.power > 10 * max.power) { + pr_err("%s: did not conserve power when setting lower frequency!\n", + engine->name); + err = -EINVAL; + break; + } + + if (igt_flush_test(gt->i915)) { + err = -EIO; + break; + } + } + + igt_spinner_fini(&spin); + + intel_gt_pm_wait_for_idle(gt); + rps->work.func = saved_work; + + return err; +} + +int live_rps_dynamic(void *arg) +{ + struct intel_gt *gt = arg; + struct intel_rps *rps = >->rps; + struct intel_engine_cs *engine; + enum intel_engine_id id; + struct igt_spinner spin; + int err = 0; + + /* + * We've looked at the bascs, and have established that we + * can change the clock frequency and that the HW will generate + * interrupts based on load. Now we check how we integrate those + * moving parts into dynamic reclocking based on load. + */ + + if (!intel_rps_is_enabled(rps)) + return 0; + + if (igt_spinner_init(&spin, gt)) + return -ENOMEM; + + for_each_engine(engine, gt, id) { + struct i915_request *rq; + struct { + ktime_t dt; + u8 freq; + } min, max; + + if (!intel_engine_can_store_dword(engine)) + continue; + + intel_gt_pm_wait_for_idle(gt); + GEM_BUG_ON(intel_rps_is_active(rps)); + rps->cur_freq = rps->min_freq; + + intel_engine_pm_get(engine); + intel_rc6_disable(>->rc6); + GEM_BUG_ON(rps->last_freq != rps->min_freq); + + rq = igt_spinner_create_request(&spin, + engine->kernel_context, + MI_NOOP); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + goto err; + } + + i915_request_add(rq); + + max.dt = ktime_get(); + max.freq = wait_for_freq(rps, rps->max_freq, 500); + max.dt = ktime_sub(ktime_get(), max.dt); + + igt_spinner_end(&spin); + + min.dt = ktime_get(); + min.freq = wait_for_freq(rps, rps->min_freq, 2000); + min.dt = ktime_sub(ktime_get(), min.dt); + + pr_info("%s: dynamically reclocked to %u:%uMHz while busy in %lluns, and %u:%uMHz while idle in %lluns\n", + engine->name, + max.freq, intel_gpu_freq(rps, max.freq), + ktime_to_ns(max.dt), + min.freq, intel_gpu_freq(rps, min.freq), + ktime_to_ns(min.dt)); + if (min.freq >= max.freq) { + pr_err("%s: dynamic reclocking of spinner failed\n!", + engine->name); + err = -EINVAL; + } + +err: + intel_rc6_enable(>->rc6); + intel_engine_pm_put(engine); + + if (igt_flush_test(gt->i915)) + err = -EIO; + if (err) + break; + } + + igt_spinner_fini(&spin); + + return err; +} diff --git a/drivers/gpu/drm/i915/gt/selftest_rps.h b/drivers/gpu/drm/i915/gt/selftest_rps.h new file mode 100644 index 000000000000..6e82a631cfa1 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/selftest_rps.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef SELFTEST_RPS_H +#define SELFTEST_RPS_H + +int live_rps_control(void *arg); +int live_rps_clock_interval(void *arg); +int live_rps_frequency_cs(void *arg); +int live_rps_frequency_srm(void *arg); +int live_rps_power(void *arg); +int live_rps_interrupt(void *arg); +int live_rps_dynamic(void *arg); + +#endif /* SELFTEST_RPS_H */ diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.c b/drivers/gpu/drm/i915/gt/shmem_utils.c new file mode 100644 index 000000000000..43c7acbdc79d --- /dev/null +++ b/drivers/gpu/drm/i915/gt/shmem_utils.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <linux/mm.h> +#include <linux/pagemap.h> +#include <linux/shmem_fs.h> + +#include "gem/i915_gem_object.h" +#include "shmem_utils.h" + +struct file *shmem_create_from_data(const char *name, void *data, size_t len) +{ + struct file *file; + int err; + + file = shmem_file_setup(name, PAGE_ALIGN(len), VM_NORESERVE); + if (IS_ERR(file)) + return file; + + err = shmem_write(file, 0, data, len); + if (err) { + fput(file); + return ERR_PTR(err); + } + + return file; +} + +struct file *shmem_create_from_object(struct drm_i915_gem_object *obj) +{ + struct file *file; + void *ptr; + + if (obj->ops == &i915_gem_shmem_ops) { + file = obj->base.filp; + atomic_long_inc(&file->f_count); + return file; + } + + ptr = i915_gem_object_pin_map(obj, I915_MAP_WB); + if (IS_ERR(ptr)) + return ERR_CAST(ptr); + + file = shmem_create_from_data("", ptr, obj->base.size); + i915_gem_object_unpin_map(obj); + + return file; +} + +static size_t shmem_npte(struct file *file) +{ + return file->f_mapping->host->i_size >> PAGE_SHIFT; +} + +static void __shmem_unpin_map(struct file *file, void *ptr, size_t n_pte) +{ + unsigned long pfn; + + vunmap(ptr); + + for (pfn = 0; pfn < n_pte; pfn++) { + struct page *page; + + page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, + GFP_KERNEL); + if (!WARN_ON(IS_ERR(page))) { + put_page(page); + put_page(page); + } + } +} + +void *shmem_pin_map(struct file *file) +{ + const size_t n_pte = shmem_npte(file); + pte_t *stack[32], **ptes, **mem; + struct vm_struct *area; + unsigned long pfn; + + mem = stack; + if (n_pte > ARRAY_SIZE(stack)) { + mem = kvmalloc_array(n_pte, sizeof(*mem), GFP_KERNEL); + if (!mem) + return NULL; + } + + area = alloc_vm_area(n_pte << PAGE_SHIFT, mem); + if (!area) { + if (mem != stack) + kvfree(mem); + return NULL; + } + + ptes = mem; + for (pfn = 0; pfn < n_pte; pfn++) { + struct page *page; + + page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, + GFP_KERNEL); + if (IS_ERR(page)) + goto err_page; + + **ptes++ = mk_pte(page, PAGE_KERNEL); + } + + if (mem != stack) + kvfree(mem); + + mapping_set_unevictable(file->f_mapping); + return area->addr; + +err_page: + if (mem != stack) + kvfree(mem); + + __shmem_unpin_map(file, area->addr, pfn); + return NULL; +} + +void shmem_unpin_map(struct file *file, void *ptr) +{ + mapping_clear_unevictable(file->f_mapping); + __shmem_unpin_map(file, ptr, shmem_npte(file)); +} + +static int __shmem_rw(struct file *file, loff_t off, + void *ptr, size_t len, + bool write) +{ + unsigned long pfn; + + for (pfn = off >> PAGE_SHIFT; len; pfn++) { + unsigned int this = + min_t(size_t, PAGE_SIZE - offset_in_page(off), len); + struct page *page; + void *vaddr; + + page = shmem_read_mapping_page_gfp(file->f_mapping, pfn, + GFP_KERNEL); + if (IS_ERR(page)) + return PTR_ERR(page); + + vaddr = kmap(page); + if (write) + memcpy(vaddr + offset_in_page(off), ptr, this); + else + memcpy(ptr, vaddr + offset_in_page(off), this); + kunmap(page); + put_page(page); + + len -= this; + ptr += this; + off = 0; + } + + return 0; +} + +int shmem_read(struct file *file, loff_t off, void *dst, size_t len) +{ + return __shmem_rw(file, off, dst, len, false); +} + +int shmem_write(struct file *file, loff_t off, void *src, size_t len) +{ + return __shmem_rw(file, off, src, len, true); +} + +#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) +#include "st_shmem_utils.c" +#endif diff --git a/drivers/gpu/drm/i915/gt/shmem_utils.h b/drivers/gpu/drm/i915/gt/shmem_utils.h new file mode 100644 index 000000000000..c1669170c351 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/shmem_utils.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef SHMEM_UTILS_H +#define SHMEM_UTILS_H + +#include <linux/types.h> + +struct drm_i915_gem_object; +struct file; + +struct file *shmem_create_from_data(const char *name, void *data, size_t len); +struct file *shmem_create_from_object(struct drm_i915_gem_object *obj); + +void *shmem_pin_map(struct file *file); +void shmem_unpin_map(struct file *file, void *ptr); + +int shmem_read(struct file *file, loff_t off, void *dst, size_t len); +int shmem_write(struct file *file, loff_t off, void *src, size_t len); + +#endif /* SHMEM_UTILS_H */ diff --git a/drivers/gpu/drm/i915/gt/st_shmem_utils.c b/drivers/gpu/drm/i915/gt/st_shmem_utils.c new file mode 100644 index 000000000000..b279fe88b70e --- /dev/null +++ b/drivers/gpu/drm/i915/gt/st_shmem_utils.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +/* Just a quick and causal check of the shmem_utils API */ + +static int igt_shmem_basic(void *ignored) +{ + u32 datum = 0xdeadbeef, result; + struct file *file; + u32 *map; + int err; + + file = shmem_create_from_data("mock", &datum, sizeof(datum)); + if (IS_ERR(file)) + return PTR_ERR(file); + + result = 0; + err = shmem_read(file, 0, &result, sizeof(result)); + if (err) + goto out_file; + + if (result != datum) { + pr_err("Incorrect read back from shmemfs: %x != %x\n", + result, datum); + err = -EINVAL; + goto out_file; + } + + result = 0xc0ffee; + err = shmem_write(file, 0, &result, sizeof(result)); + if (err) + goto out_file; + + map = shmem_pin_map(file); + if (!map) { + err = -ENOMEM; + goto out_file; + } + + if (*map != result) { + pr_err("Incorrect read back via mmap of last write: %x != %x\n", + *map, result); + err = -EINVAL; + goto out_map; + } + +out_map: + shmem_unpin_map(file, map); +out_file: + fput(file); + return err; +} + +int shmem_utils_mock_selftests(void) +{ + static const struct i915_subtest tests[] = { + SUBTEST(igt_shmem_basic), + }; + + return i915_subtests(tests, NULL); +} diff --git a/drivers/gpu/drm/i915/gt/sysfs_engines.c b/drivers/gpu/drm/i915/gt/sysfs_engines.c new file mode 100644 index 000000000000..535cc1169e54 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/sysfs_engines.c @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2019 Intel Corporation + */ + +#include <linux/kobject.h> +#include <linux/sysfs.h> + +#include "i915_drv.h" +#include "intel_engine.h" +#include "intel_engine_heartbeat.h" +#include "sysfs_engines.h" + +struct kobj_engine { + struct kobject base; + struct intel_engine_cs *engine; +}; + +static struct intel_engine_cs *kobj_to_engine(struct kobject *kobj) +{ + return container_of(kobj, struct kobj_engine, base)->engine; +} + +static ssize_t +name_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%s\n", kobj_to_engine(kobj)->name); +} + +static struct kobj_attribute name_attr = +__ATTR(name, 0444, name_show, NULL); + +static ssize_t +class_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", kobj_to_engine(kobj)->uabi_class); +} + +static struct kobj_attribute class_attr = +__ATTR(class, 0444, class_show, NULL); + +static ssize_t +inst_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", kobj_to_engine(kobj)->uabi_instance); +} + +static struct kobj_attribute inst_attr = +__ATTR(instance, 0444, inst_show, NULL); + +static ssize_t +mmio_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "0x%x\n", kobj_to_engine(kobj)->mmio_base); +} + +static struct kobj_attribute mmio_attr = +__ATTR(mmio_base, 0444, mmio_show, NULL); + +static const char * const vcs_caps[] = { + [ilog2(I915_VIDEO_CLASS_CAPABILITY_HEVC)] = "hevc", + [ilog2(I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC)] = "sfc", +}; + +static const char * const vecs_caps[] = { + [ilog2(I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC)] = "sfc", +}; + +static ssize_t repr_trim(char *buf, ssize_t len) +{ + /* Trim off the trailing space and replace with a newline */ + if (len > PAGE_SIZE) + len = PAGE_SIZE; + if (len > 0) + buf[len - 1] = '\n'; + + return len; +} + +static ssize_t +__caps_show(struct intel_engine_cs *engine, + u32 caps, char *buf, bool show_unknown) +{ + const char * const *repr; + int count, n; + ssize_t len; + + BUILD_BUG_ON(!typecheck(typeof(caps), engine->uabi_capabilities)); + + switch (engine->class) { + case VIDEO_DECODE_CLASS: + repr = vcs_caps; + count = ARRAY_SIZE(vcs_caps); + break; + + case VIDEO_ENHANCEMENT_CLASS: + repr = vecs_caps; + count = ARRAY_SIZE(vecs_caps); + break; + + default: + repr = NULL; + count = 0; + break; + } + GEM_BUG_ON(count > BITS_PER_TYPE(typeof(caps))); + + len = 0; + for_each_set_bit(n, + (unsigned long *)&caps, + show_unknown ? BITS_PER_TYPE(typeof(caps)) : count) { + if (n >= count || !repr[n]) { + if (GEM_WARN_ON(show_unknown)) + len += snprintf(buf + len, PAGE_SIZE - len, + "[%x] ", n); + } else { + len += snprintf(buf + len, PAGE_SIZE - len, + "%s ", repr[n]); + } + if (GEM_WARN_ON(len >= PAGE_SIZE)) + break; + } + return repr_trim(buf, len); +} + +static ssize_t +caps_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return __caps_show(engine, engine->uabi_capabilities, buf, true); +} + +static struct kobj_attribute caps_attr = +__ATTR(capabilities, 0444, caps_show, NULL); + +static ssize_t +all_caps_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + return __caps_show(kobj_to_engine(kobj), -1, buf, false); +} + +static struct kobj_attribute all_caps_attr = +__ATTR(known_capabilities, 0444, all_caps_show, NULL); + +static ssize_t +max_spin_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + unsigned long long duration; + int err; + + /* + * When waiting for a request, if is it currently being executed + * on the GPU, we busywait for a short while before sleeping. The + * premise is that most requests are short, and if it is already + * executing then there is a good chance that it will complete + * before we can setup the interrupt handler and go to sleep. + * We try to offset the cost of going to sleep, by first spinning + * on the request -- if it completed in less time than it would take + * to go sleep, process the interrupt and return back to the client, + * then we have saved the client some latency, albeit at the cost + * of spinning on an expensive CPU core. + * + * While we try to avoid waiting at all for a request that is unlikely + * to complete, deciding how long it is worth spinning is for is an + * arbitrary decision: trading off power vs latency. + */ + + err = kstrtoull(buf, 0, &duration); + if (err) + return err; + + if (duration > jiffies_to_nsecs(2)) + return -EINVAL; + + WRITE_ONCE(engine->props.max_busywait_duration_ns, duration); + + return count; +} + +static ssize_t +max_spin_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->props.max_busywait_duration_ns); +} + +static struct kobj_attribute max_spin_attr = +__ATTR(max_busywait_duration_ns, 0644, max_spin_show, max_spin_store); + +static ssize_t +max_spin_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->defaults.max_busywait_duration_ns); +} + +static struct kobj_attribute max_spin_def = +__ATTR(max_busywait_duration_ns, 0444, max_spin_default, NULL); + +static ssize_t +timeslice_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + unsigned long long duration; + int err; + + /* + * Execlists uses a scheduling quantum (a timeslice) to alternate + * execution between ready-to-run contexts of equal priority. This + * ensures that all users (though only if they of equal importance) + * have the opportunity to run and prevents livelocks where contexts + * may have implicit ordering due to userspace semaphores. + */ + + err = kstrtoull(buf, 0, &duration); + if (err) + return err; + + if (duration > jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)) + return -EINVAL; + + WRITE_ONCE(engine->props.timeslice_duration_ms, duration); + + if (execlists_active(&engine->execlists)) + set_timer_ms(&engine->execlists.timer, duration); + + return count; +} + +static ssize_t +timeslice_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->props.timeslice_duration_ms); +} + +static struct kobj_attribute timeslice_duration_attr = +__ATTR(timeslice_duration_ms, 0644, timeslice_show, timeslice_store); + +static ssize_t +timeslice_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->defaults.timeslice_duration_ms); +} + +static struct kobj_attribute timeslice_duration_def = +__ATTR(timeslice_duration_ms, 0444, timeslice_default, NULL); + +static ssize_t +stop_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + unsigned long long duration; + int err; + + /* + * When we allow ourselves to sleep before a GPU reset after disabling + * submission, even for a few milliseconds, gives an innocent context + * the opportunity to clear the GPU before the reset occurs. However, + * how long to sleep depends on the typical non-preemptible duration + * (a similar problem to determining the ideal preempt-reset timeout + * or even the heartbeat interval). + */ + + err = kstrtoull(buf, 0, &duration); + if (err) + return err; + + if (duration > jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)) + return -EINVAL; + + WRITE_ONCE(engine->props.stop_timeout_ms, duration); + return count; +} + +static ssize_t +stop_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->props.stop_timeout_ms); +} + +static struct kobj_attribute stop_timeout_attr = +__ATTR(stop_timeout_ms, 0644, stop_show, stop_store); + +static ssize_t +stop_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->defaults.stop_timeout_ms); +} + +static struct kobj_attribute stop_timeout_def = +__ATTR(stop_timeout_ms, 0444, stop_default, NULL); + +static ssize_t +preempt_timeout_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + unsigned long long timeout; + int err; + + /* + * After initialising a preemption request, we give the current + * resident a small amount of time to vacate the GPU. The preemption + * request is for a higher priority context and should be immediate to + * maintain high quality of service (and avoid priority inversion). + * However, the preemption granularity of the GPU can be quite coarse + * and so we need a compromise. + */ + + err = kstrtoull(buf, 0, &timeout); + if (err) + return err; + + if (timeout > jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)) + return -EINVAL; + + WRITE_ONCE(engine->props.preempt_timeout_ms, timeout); + + if (READ_ONCE(engine->execlists.pending[0])) + set_timer_ms(&engine->execlists.preempt, timeout); + + return count; +} + +static ssize_t +preempt_timeout_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->props.preempt_timeout_ms); +} + +static struct kobj_attribute preempt_timeout_attr = +__ATTR(preempt_timeout_ms, 0644, preempt_timeout_show, preempt_timeout_store); + +static ssize_t +preempt_timeout_default(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->defaults.preempt_timeout_ms); +} + +static struct kobj_attribute preempt_timeout_def = +__ATTR(preempt_timeout_ms, 0444, preempt_timeout_default, NULL); + +static ssize_t +heartbeat_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + unsigned long long delay; + int err; + + /* + * We monitor the health of the system via periodic heartbeat pulses. + * The pulses also provide the opportunity to perform garbage + * collection. However, we interpret an incomplete pulse (a missed + * heartbeat) as an indication that the system is no longer responsive, + * i.e. hung, and perform an engine or full GPU reset. Given that the + * preemption granularity can be very coarse on a system, the optimal + * value for any workload is unknowable! + */ + + err = kstrtoull(buf, 0, &delay); + if (err) + return err; + + if (delay >= jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT)) + return -EINVAL; + + err = intel_engine_set_heartbeat(engine, delay); + if (err) + return err; + + return count; +} + +static ssize_t +heartbeat_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->props.heartbeat_interval_ms); +} + +static struct kobj_attribute heartbeat_interval_attr = +__ATTR(heartbeat_interval_ms, 0644, heartbeat_show, heartbeat_store); + +static ssize_t +heartbeat_default(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct intel_engine_cs *engine = kobj_to_engine(kobj); + + return sprintf(buf, "%lu\n", engine->defaults.heartbeat_interval_ms); +} + +static struct kobj_attribute heartbeat_interval_def = +__ATTR(heartbeat_interval_ms, 0444, heartbeat_default, NULL); + +static void kobj_engine_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static struct kobj_type kobj_engine_type = { + .release = kobj_engine_release, + .sysfs_ops = &kobj_sysfs_ops +}; + +static struct kobject * +kobj_engine(struct kobject *dir, struct intel_engine_cs *engine) +{ + struct kobj_engine *ke; + + ke = kzalloc(sizeof(*ke), GFP_KERNEL); + if (!ke) + return NULL; + + kobject_init(&ke->base, &kobj_engine_type); + ke->engine = engine; + + if (kobject_add(&ke->base, dir, "%s", engine->name)) { + kobject_put(&ke->base); + return NULL; + } + + /* xfer ownership to sysfs tree */ + return &ke->base; +} + +static void add_defaults(struct kobj_engine *parent) +{ + static const struct attribute *files[] = { + &max_spin_def.attr, + &stop_timeout_def.attr, +#if CONFIG_DRM_I915_HEARTBEAT_INTERVAL + &heartbeat_interval_def.attr, +#endif + NULL + }; + struct kobj_engine *ke; + + ke = kzalloc(sizeof(*ke), GFP_KERNEL); + if (!ke) + return; + + kobject_init(&ke->base, &kobj_engine_type); + ke->engine = parent->engine; + + if (kobject_add(&ke->base, &parent->base, "%s", ".defaults")) { + kobject_put(&ke->base); + return; + } + + if (sysfs_create_files(&ke->base, files)) + return; + + if (intel_engine_has_timeslices(ke->engine) && + sysfs_create_file(&ke->base, ×lice_duration_def.attr)) + return; + + if (intel_engine_has_preempt_reset(ke->engine) && + sysfs_create_file(&ke->base, &preempt_timeout_def.attr)) + return; +} + +void intel_engines_add_sysfs(struct drm_i915_private *i915) +{ + static const struct attribute *files[] = { + &name_attr.attr, + &class_attr.attr, + &inst_attr.attr, + &mmio_attr.attr, + &caps_attr.attr, + &all_caps_attr.attr, + &max_spin_attr.attr, + &stop_timeout_attr.attr, +#if CONFIG_DRM_I915_HEARTBEAT_INTERVAL + &heartbeat_interval_attr.attr, +#endif + NULL + }; + + struct device *kdev = i915->drm.primary->kdev; + struct intel_engine_cs *engine; + struct kobject *dir; + + dir = kobject_create_and_add("engine", &kdev->kobj); + if (!dir) + return; + + for_each_uabi_engine(engine, i915) { + struct kobject *kobj; + + kobj = kobj_engine(dir, engine); + if (!kobj) + goto err_engine; + + if (sysfs_create_files(kobj, files)) + goto err_object; + + if (intel_engine_has_timeslices(engine) && + sysfs_create_file(kobj, ×lice_duration_attr.attr)) + goto err_engine; + + if (intel_engine_has_preempt_reset(engine) && + sysfs_create_file(kobj, &preempt_timeout_attr.attr)) + goto err_engine; + + add_defaults(container_of(kobj, struct kobj_engine, base)); + + if (0) { +err_object: + kobject_put(kobj); +err_engine: + dev_err(kdev, "Failed to add sysfs engine '%s'\n", + engine->name); + break; + } + } +} diff --git a/drivers/gpu/drm/i915/gt/sysfs_engines.h b/drivers/gpu/drm/i915/gt/sysfs_engines.h new file mode 100644 index 000000000000..9546fffe03a7 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/sysfs_engines.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2019 Intel Corporation + */ + +#ifndef INTEL_ENGINE_SYSFS_H +#define INTEL_ENGINE_SYSFS_H + +struct drm_i915_private; + +void intel_engines_add_sysfs(struct drm_i915_private *i915); + +#endif /* INTEL_ENGINE_SYSFS_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index c4c1523da7a6..861657897c0f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -169,7 +169,7 @@ void intel_guc_init_early(struct intel_guc *guc) { struct drm_i915_private *i915 = guc_to_gt(guc)->i915; - intel_guc_fw_init_early(guc); + intel_uc_fw_init_early(&guc->fw, INTEL_UC_FW_TYPE_GUC); intel_guc_ct_init_early(&guc->ct); intel_guc_log_init_early(&guc->log); intel_guc_submission_init_early(guc); @@ -207,7 +207,7 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc) { u32 flags = 0; - if (!intel_guc_is_submission_supported(guc)) + if (!intel_guc_submission_is_used(guc)) flags |= GUC_CTL_DISABLE_SCHEDULER; return flags; @@ -217,7 +217,7 @@ static u32 guc_ctl_ctxinfo_flags(struct intel_guc *guc) { u32 flags = 0; - if (intel_guc_is_submission_supported(guc)) { + if (intel_guc_submission_is_used(guc)) { u32 ctxnum, base; base = intel_guc_ggtt_offset(guc, guc->stage_desc_pool); @@ -333,7 +333,7 @@ int intel_guc_init(struct intel_guc *guc) ret = intel_uc_fw_init(&guc->fw); if (ret) - goto err_fetch; + goto out; ret = intel_guc_log_create(&guc->log); if (ret) @@ -348,7 +348,7 @@ int intel_guc_init(struct intel_guc *guc) if (ret) goto err_ads; - if (intel_guc_is_submission_supported(guc)) { + if (intel_guc_submission_is_used(guc)) { /* * This is stuff we need to have available at fw load time * if we are planning to enable submission later @@ -364,6 +364,8 @@ int intel_guc_init(struct intel_guc *guc) /* We need to notify the guc whenever we change the GGTT */ i915_ggtt_enable_guc(gt->ggtt); + intel_uc_fw_change_status(&guc->fw, INTEL_UC_FIRMWARE_LOADABLE); + return 0; err_ct: @@ -374,9 +376,8 @@ err_log: intel_guc_log_destroy(&guc->log); err_fw: intel_uc_fw_fini(&guc->fw); -err_fetch: - intel_uc_fw_cleanup_fetch(&guc->fw); - DRM_DEV_DEBUG_DRIVER(gt->i915->drm.dev, "failed with %d\n", ret); +out: + i915_probe_error(gt->i915, "failed with %d\n", ret); return ret; } @@ -384,12 +385,12 @@ void intel_guc_fini(struct intel_guc *guc) { struct intel_gt *gt = guc_to_gt(guc); - if (!intel_uc_fw_is_available(&guc->fw)) + if (!intel_uc_fw_is_loadable(&guc->fw)) return; i915_ggtt_disable_guc(gt->ggtt); - if (intel_guc_is_submission_supported(guc)) + if (intel_guc_submission_is_used(guc)) intel_guc_submission_fini(guc); intel_guc_ct_fini(&guc->ct); @@ -397,9 +398,6 @@ void intel_guc_fini(struct intel_guc *guc) intel_guc_ads_destroy(guc); intel_guc_log_destroy(&guc->log); intel_uc_fw_fini(&guc->fw); - intel_uc_fw_cleanup_fetch(&guc->fw); - - intel_uc_fw_change_status(&guc->fw, INTEL_UC_FIRMWARE_DISABLED); } /* @@ -544,7 +542,7 @@ int intel_guc_suspend(struct intel_guc *guc) * If GuC communication is enabled but submission is not supported, * we do not need to suspend the GuC. */ - if (!intel_guc_submission_is_enabled(guc)) + if (!intel_guc_submission_is_used(guc) || !intel_guc_is_ready(guc)) return 0; /* @@ -609,7 +607,7 @@ int intel_guc_resume(struct intel_guc *guc) * we do not need to resume the GuC but we do need to enable the * GuC communication on resume (above). */ - if (!intel_guc_submission_is_enabled(guc)) + if (!intel_guc_submission_is_used(guc) || !intel_guc_is_ready(guc)) return 0; return intel_guc_send(guc, action, ARRAY_SIZE(action)); @@ -725,3 +723,47 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, return 0; } + +/** + * intel_guc_load_status - dump information about GuC load status + * @guc: the GuC + * @p: the &drm_printer + * + * Pretty printer for GuC load status. + */ +void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p) +{ + struct intel_gt *gt = guc_to_gt(guc); + struct intel_uncore *uncore = gt->uncore; + intel_wakeref_t wakeref; + + if (!intel_guc_is_supported(guc)) { + drm_printf(p, "GuC not supported\n"); + return; + } + + if (!intel_guc_is_wanted(guc)) { + drm_printf(p, "GuC disabled\n"); + return; + } + + intel_uc_fw_dump(&guc->fw, p); + + with_intel_runtime_pm(uncore->rpm, wakeref) { + u32 status = intel_uncore_read(uncore, GUC_STATUS); + u32 i; + + drm_printf(p, "\nGuC status 0x%08x:\n", status); + drm_printf(p, "\tBootrom status = 0x%x\n", + (status & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); + drm_printf(p, "\tuKernel status = 0x%x\n", + (status & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); + drm_printf(p, "\tMIA Core status = 0x%x\n", + (status & GS_MIA_MASK) >> GS_MIA_SHIFT); + drm_puts(p, "\nScratch registers:\n"); + for (i = 0; i < 16; i++) { + drm_printf(p, "\t%2d: \t0x%x\n", + i, intel_uncore_read(uncore, SOFT_SCRATCH(i))); + } + } +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h index 668b067b71e2..e84ab67b317d 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h @@ -39,7 +39,7 @@ struct intel_guc { void (*disable)(struct intel_guc *guc); } interrupts; - bool submission_supported; + bool submission_selected; struct i915_vma *ads_vma; struct __guc_ads_blob *ads_blob; @@ -74,6 +74,11 @@ struct intel_guc { struct mutex send_mutex; }; +static inline struct intel_guc *log_to_guc(struct intel_guc_log *log) +{ + return container_of(log, struct intel_guc, log); +} + static inline int intel_guc_send(struct intel_guc *guc, const u32 *action, u32 len) { @@ -143,11 +148,17 @@ static inline bool intel_guc_is_supported(struct intel_guc *guc) return intel_uc_fw_is_supported(&guc->fw); } -static inline bool intel_guc_is_enabled(struct intel_guc *guc) +static inline bool intel_guc_is_wanted(struct intel_guc *guc) { return intel_uc_fw_is_enabled(&guc->fw); } +static inline bool intel_guc_is_used(struct intel_guc *guc) +{ + GEM_BUG_ON(__intel_uc_fw_status(&guc->fw) == INTEL_UC_FIRMWARE_SELECTED); + return intel_uc_fw_is_available(&guc->fw); +} + static inline bool intel_guc_is_fw_running(struct intel_guc *guc) { return intel_uc_fw_is_running(&guc->fw); @@ -167,11 +178,6 @@ static inline int intel_guc_sanitize(struct intel_guc *guc) return 0; } -static inline bool intel_guc_is_submission_supported(struct intel_guc *guc) -{ - return guc->submission_supported; -} - static inline void intel_guc_enable_msg(struct intel_guc *guc, u32 mask) { spin_lock_irq(&guc->irq_lock); @@ -189,4 +195,6 @@ static inline void intel_guc_disable_msg(struct intel_guc *guc, u32 mask) int intel_guc_reset_engine(struct intel_guc *guc, struct intel_engine_cs *engine); +void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p); + #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c new file mode 100644 index 000000000000..fe7cb7b29a1e --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <drm/drm_print.h> + +#include "gt/debugfs_gt.h" +#include "intel_guc.h" +#include "intel_guc_debugfs.h" +#include "intel_guc_log_debugfs.h" + +static int guc_info_show(struct seq_file *m, void *data) +{ + struct intel_guc *guc = m->private; + struct drm_printer p = drm_seq_file_printer(m); + + if (!intel_guc_is_supported(guc)) + return -ENODEV; + + intel_guc_load_status(guc, &p); + drm_puts(&p, "\n"); + intel_guc_log_info(&guc->log, &p); + + /* Add more as required ... */ + + return 0; +} +DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_info); + +void intel_guc_debugfs_register(struct intel_guc *guc, struct dentry *root) +{ + static const struct debugfs_gt_file files[] = { + { "guc_info", &guc_info_fops, NULL }, + }; + + if (!intel_guc_is_supported(guc)) + return; + + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), guc); + intel_guc_log_debugfs_register(&guc->log, root); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.h new file mode 100644 index 000000000000..424c26665cf1 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef DEBUGFS_GUC_H +#define DEBUGFS_GUC_H + +struct intel_guc; +struct dentry; + +void intel_guc_debugfs_register(struct intel_guc *guc, struct dentry *root); + +#endif /* DEBUGFS_GUC_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c index 3a1c47d600ea..d4a87f4c9421 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.c @@ -13,20 +13,6 @@ #include "intel_guc_fw.h" #include "i915_drv.h" -/** - * intel_guc_fw_init_early() - initializes GuC firmware struct - * @guc: intel_guc struct - * - * On platforms with GuC selects firmware for uploading - */ -void intel_guc_fw_init_early(struct intel_guc *guc) -{ - struct drm_i915_private *i915 = guc_to_gt(guc)->i915; - - intel_uc_fw_init_early(&guc->fw, INTEL_UC_FW_TYPE_GUC, HAS_GT_UC(i915), - INTEL_INFO(i915)->platform, INTEL_REVID(i915)); -} - static void guc_prepare_xfer(struct intel_uncore *uncore) { u32 shim_flags = GUC_DISABLE_SRAM_INIT_TO_ZEROES | diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.h index b5ab639d7259..0b4d2a9c9435 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fw.h @@ -8,7 +8,6 @@ struct intel_guc; -void intel_guc_fw_init_early(struct intel_guc *guc); int intel_guc_fw_upload(struct intel_guc *guc); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c index caed0d57e704..fb10f3597ea5 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c @@ -55,11 +55,6 @@ static int guc_action_control_log(struct intel_guc *guc, bool enable, return intel_guc_send(guc, action, ARRAY_SIZE(action)); } -static inline struct intel_guc *log_to_guc(struct intel_guc_log *log) -{ - return container_of(log, struct intel_guc, log); -} - static void guc_log_enable_flush_events(struct intel_guc_log *log) { intel_guc_enable_msg(log_to_guc(log), @@ -672,3 +667,95 @@ void intel_guc_log_handle_flush_event(struct intel_guc_log *log) { queue_work(system_highpri_wq, &log->relay.flush_work); } + +static const char * +stringify_guc_log_type(enum guc_log_buffer_type type) +{ + switch (type) { + case GUC_ISR_LOG_BUFFER: + return "ISR"; + case GUC_DPC_LOG_BUFFER: + return "DPC"; + case GUC_CRASH_DUMP_LOG_BUFFER: + return "CRASH"; + default: + MISSING_CASE(type); + } + + return ""; +} + +/** + * intel_guc_log_info - dump information about GuC log relay + * @log: the GuC log + * @p: the &drm_printer + * + * Pretty printer for GuC log info + */ +void intel_guc_log_info(struct intel_guc_log *log, struct drm_printer *p) +{ + enum guc_log_buffer_type type; + + if (!intel_guc_log_relay_created(log)) { + drm_puts(p, "GuC log relay not created\n"); + return; + } + + drm_puts(p, "GuC logging stats:\n"); + + drm_printf(p, "\tRelay full count: %u\n", log->relay.full_count); + + for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { + drm_printf(p, "\t%s:\tflush count %10u, overflow count %10u\n", + stringify_guc_log_type(type), + log->stats[type].flush, + log->stats[type].sampled_overflow); + } +} + +/** + * intel_guc_log_dump - dump the contents of the GuC log + * @log: the GuC log + * @p: the &drm_printer + * @dump_load_err: dump the log saved on GuC load error + * + * Pretty printer for the GuC log + */ +int intel_guc_log_dump(struct intel_guc_log *log, struct drm_printer *p, + bool dump_load_err) +{ + struct intel_guc *guc = log_to_guc(log); + struct intel_uc *uc = container_of(guc, struct intel_uc, guc); + struct drm_i915_gem_object *obj = NULL; + u32 *map; + int i = 0; + + if (!intel_guc_is_supported(guc)) + return -ENODEV; + + if (dump_load_err) + obj = uc->load_err_log; + else if (guc->log.vma) + obj = guc->log.vma->obj; + + if (!obj) + return 0; + + map = i915_gem_object_pin_map(obj, I915_MAP_WC); + if (IS_ERR(map)) { + DRM_DEBUG("Failed to pin object\n"); + drm_puts(p, "(log data unaccessible)\n"); + return PTR_ERR(map); + } + + for (i = 0; i < obj->base.size / sizeof(u32); i += 4) + drm_printf(p, "0x%08x 0x%08x 0x%08x 0x%08x\n", + *(map + i), *(map + i + 1), + *(map + i + 2), *(map + i + 3)); + + drm_puts(p, "\n"); + + i915_gem_object_unpin_map(obj); + + return 0; +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h index c252c022c5fc..11fccd0b2294 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h @@ -79,4 +79,8 @@ static inline u32 intel_guc_log_get_level(struct intel_guc_log *log) return log->level; } +void intel_guc_log_info(struct intel_guc_log *log, struct drm_printer *p); +int intel_guc_log_dump(struct intel_guc_log *log, struct drm_printer *p, + bool dump_load_err); + #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c new file mode 100644 index 000000000000..129e0cf7dfe2 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <linux/fs.h> +#include <drm/drm_print.h> + +#include "gt/debugfs_gt.h" +#include "intel_guc.h" +#include "intel_guc_log.h" +#include "intel_guc_log_debugfs.h" + +static int guc_log_dump_show(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + + return intel_guc_log_dump(m->private, &p, false); +} +DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_log_dump); + +static int guc_load_err_log_dump_show(struct seq_file *m, void *data) +{ + struct drm_printer p = drm_seq_file_printer(m); + + return intel_guc_log_dump(m->private, &p, true); +} +DEFINE_GT_DEBUGFS_ATTRIBUTE(guc_load_err_log_dump); + +static int guc_log_level_get(void *data, u64 *val) +{ + struct intel_guc_log *log = data; + + if (!intel_guc_is_used(log_to_guc(log))) + return -ENODEV; + + *val = intel_guc_log_get_level(log); + + return 0; +} + +static int guc_log_level_set(void *data, u64 val) +{ + struct intel_guc_log *log = data; + + if (!intel_guc_is_used(log_to_guc(log))) + return -ENODEV; + + return intel_guc_log_set_level(log, val); +} + +DEFINE_SIMPLE_ATTRIBUTE(guc_log_level_fops, + guc_log_level_get, guc_log_level_set, + "%lld\n"); + +static int guc_log_relay_open(struct inode *inode, struct file *file) +{ + struct intel_guc_log *log = inode->i_private; + + if (!intel_guc_is_ready(log_to_guc(log))) + return -ENODEV; + + file->private_data = log; + + return intel_guc_log_relay_open(log); +} + +static ssize_t +guc_log_relay_write(struct file *filp, + const char __user *ubuf, + size_t cnt, + loff_t *ppos) +{ + struct intel_guc_log *log = filp->private_data; + int val; + int ret; + + ret = kstrtoint_from_user(ubuf, cnt, 0, &val); + if (ret < 0) + return ret; + + /* + * Enable and start the guc log relay on value of 1. + * Flush log relay for any other value. + */ + if (val == 1) + ret = intel_guc_log_relay_start(log); + else + intel_guc_log_relay_flush(log); + + return ret ?: cnt; +} + +static int guc_log_relay_release(struct inode *inode, struct file *file) +{ + struct intel_guc_log *log = inode->i_private; + + intel_guc_log_relay_close(log); + return 0; +} + +static const struct file_operations guc_log_relay_fops = { + .owner = THIS_MODULE, + .open = guc_log_relay_open, + .write = guc_log_relay_write, + .release = guc_log_relay_release, +}; + +void intel_guc_log_debugfs_register(struct intel_guc_log *log, + struct dentry *root) +{ + static const struct debugfs_gt_file files[] = { + { "guc_log_dump", &guc_log_dump_fops, NULL }, + { "guc_load_err_log_dump", &guc_load_err_log_dump_fops, NULL }, + { "guc_log_level", &guc_log_level_fops, NULL }, + { "guc_log_relay", &guc_log_relay_fops, NULL }, + }; + + if (!intel_guc_is_supported(log_to_guc(log))) + return; + + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), log); +} + diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.h new file mode 100644 index 000000000000..e8900e3d74ea --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log_debugfs.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef DEBUGFS_GUC_LOG_H +#define DEBUGFS_GUC_LOG_H + +struct intel_guc_log; +struct dentry; + +void intel_guc_log_debugfs_register(struct intel_guc_log *log, + struct dentry *root); + +#endif /* DEBUGFS_GUC_LOG_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 9e42324fdecd..94eb63f309ce 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -217,7 +217,7 @@ static void guc_wq_item_append(struct intel_guc *guc, static void guc_add_request(struct intel_guc *guc, struct i915_request *rq) { struct intel_engine_cs *engine = rq->engine; - u32 ctx_desc = lower_32_bits(rq->context->lrc_desc); + u32 ctx_desc = rq->context->lrc.ccid; u32 ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64); guc_wq_item_append(guc, engine->guc_id, ctx_desc, @@ -258,7 +258,7 @@ static void guc_submit(struct intel_engine_cs *engine, static inline int rq_prio(const struct i915_request *rq) { - return rq->sched.attr.priority | __NO_PREEMPTION; + return rq->sched.attr.priority; } static struct i915_request *schedule_in(struct i915_request *rq, int idx) @@ -456,9 +456,7 @@ static void guc_reset_cancel(struct intel_engine_cs *engine) /* Mark all executing requests as skipped. */ list_for_each_entry(rq, &engine->active.requests, sched.link) { - if (!i915_request_signaled(rq)) - dma_fence_set_error(&rq->fence, -EIO); - + i915_request_set_error_once(rq, -EIO); i915_request_mark_complete(rq); } @@ -660,12 +658,9 @@ void intel_guc_submission_disable(struct intel_guc *guc) guc_proc_desc_fini(guc); } -static bool __guc_submission_support(struct intel_guc *guc) +static bool __guc_submission_selected(struct intel_guc *guc) { - /* XXX: GuC submission is unavailable for now */ - return false; - - if (!intel_guc_is_supported(guc)) + if (!intel_guc_submission_is_supported(guc)) return false; return i915_modparams.enable_guc & ENABLE_GUC_SUBMISSION; @@ -673,7 +668,7 @@ static bool __guc_submission_support(struct intel_guc *guc) void intel_guc_submission_init_early(struct intel_guc *guc) { - guc->submission_supported = __guc_submission_support(guc); + guc->submission_selected = __guc_submission_selected(guc); } bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs *engine) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h index e402a2932592..4cf9d3e50263 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.h @@ -8,7 +8,8 @@ #include <linux/types.h> -struct intel_guc; +#include "intel_guc.h" + struct intel_engine_cs; void intel_guc_submission_init_early(struct intel_guc *guc); @@ -20,4 +21,20 @@ int intel_guc_preempt_work_create(struct intel_guc *guc); void intel_guc_preempt_work_destroy(struct intel_guc *guc); bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs *engine); +static inline bool intel_guc_submission_is_supported(struct intel_guc *guc) +{ + /* XXX: GuC submission is unavailable for now */ + return false; +} + +static inline bool intel_guc_submission_is_wanted(struct intel_guc *guc) +{ + return guc->submission_selected; +} + +static inline bool intel_guc_submission_is_used(struct intel_guc *guc) +{ + return intel_guc_is_used(guc) && intel_guc_submission_is_wanted(guc); +} + #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.c b/drivers/gpu/drm/i915/gt/uc/intel_huc.c index 32a069841c14..65eeb44b397d 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.c @@ -41,7 +41,7 @@ void intel_huc_init_early(struct intel_huc *huc) { struct drm_i915_private *i915 = huc_to_gt(huc)->i915; - intel_huc_fw_init_early(huc); + intel_uc_fw_init_early(&huc->fw, INTEL_UC_FW_TYPE_HUC); if (INTEL_GEN(i915) >= 11) { huc->status.reg = GEN11_HUC_KERNEL_LOAD_INFO; @@ -121,19 +121,20 @@ int intel_huc_init(struct intel_huc *huc) if (err) goto out_fini; + intel_uc_fw_change_status(&huc->fw, INTEL_UC_FIRMWARE_LOADABLE); + return 0; out_fini: intel_uc_fw_fini(&huc->fw); out: - intel_uc_fw_cleanup_fetch(&huc->fw); - DRM_DEV_DEBUG_DRIVER(i915->drm.dev, "failed with %d\n", err); + i915_probe_error(i915, "failed with %d\n", err); return err; } void intel_huc_fini(struct intel_huc *huc) { - if (!intel_uc_fw_is_available(&huc->fw)) + if (!intel_uc_fw_is_loadable(&huc->fw)) return; intel_huc_rsa_data_destroy(huc); @@ -199,9 +200,13 @@ fail: * This function reads status register to verify if HuC * firmware was successfully loaded. * - * Returns: 1 if HuC firmware is loaded and verified, - * 0 if HuC firmware is not loaded and -ENODEV if HuC - * is not present on this platform. + * Returns: + * * -ENODEV if HuC is not present on this platform, + * * -EOPNOTSUPP if HuC firmware is disabled, + * * -ENOPKG if HuC firmware was not installed, + * * -ENOEXEC if HuC firmware is invalid or mismatched, + * * 0 if HuC firmware is not running, + * * 1 if HuC firmware is authenticated and running. */ int intel_huc_check_status(struct intel_huc *huc) { @@ -209,11 +214,50 @@ int intel_huc_check_status(struct intel_huc *huc) intel_wakeref_t wakeref; u32 status = 0; - if (!intel_huc_is_supported(huc)) + switch (__intel_uc_fw_status(&huc->fw)) { + case INTEL_UC_FIRMWARE_NOT_SUPPORTED: return -ENODEV; + case INTEL_UC_FIRMWARE_DISABLED: + return -EOPNOTSUPP; + case INTEL_UC_FIRMWARE_MISSING: + return -ENOPKG; + case INTEL_UC_FIRMWARE_ERROR: + return -ENOEXEC; + default: + break; + } with_intel_runtime_pm(gt->uncore->rpm, wakeref) status = intel_uncore_read(gt->uncore, huc->status.reg); return (status & huc->status.mask) == huc->status.value; } + +/** + * intel_huc_load_status - dump information about HuC load status + * @huc: the HuC + * @p: the &drm_printer + * + * Pretty printer for HuC load status. + */ +void intel_huc_load_status(struct intel_huc *huc, struct drm_printer *p) +{ + struct intel_gt *gt = huc_to_gt(huc); + intel_wakeref_t wakeref; + + if (!intel_huc_is_supported(huc)) { + drm_printf(p, "HuC not supported\n"); + return; + } + + if (!intel_huc_is_wanted(huc)) { + drm_printf(p, "HuC disabled\n"); + return; + } + + intel_uc_fw_dump(&huc->fw, p); + + with_intel_runtime_pm(gt->uncore->rpm, wakeref) + drm_printf(p, "HuC status: 0x%08x\n", + intel_uncore_read(gt->uncore, huc->status.reg)); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc.h b/drivers/gpu/drm/i915/gt/uc/intel_huc.h index 644c059fe01d..daee43b661d4 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc.h @@ -41,14 +41,22 @@ static inline bool intel_huc_is_supported(struct intel_huc *huc) return intel_uc_fw_is_supported(&huc->fw); } -static inline bool intel_huc_is_enabled(struct intel_huc *huc) +static inline bool intel_huc_is_wanted(struct intel_huc *huc) { return intel_uc_fw_is_enabled(&huc->fw); } +static inline bool intel_huc_is_used(struct intel_huc *huc) +{ + GEM_BUG_ON(__intel_uc_fw_status(&huc->fw) == INTEL_UC_FIRMWARE_SELECTED); + return intel_uc_fw_is_available(&huc->fw); +} + static inline bool intel_huc_is_authenticated(struct intel_huc *huc) { return intel_uc_fw_is_running(&huc->fw); } +void intel_huc_load_status(struct intel_huc *huc, struct drm_printer *p); + #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c new file mode 100644 index 000000000000..5733c15fd123 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <drm/drm_print.h> + +#include "gt/debugfs_gt.h" +#include "intel_huc.h" +#include "intel_huc_debugfs.h" + +static int huc_info_show(struct seq_file *m, void *data) +{ + struct intel_huc *huc = m->private; + struct drm_printer p = drm_seq_file_printer(m); + + if (!intel_huc_is_supported(huc)) + return -ENODEV; + + intel_huc_load_status(huc, &p); + + return 0; +} +DEFINE_GT_DEBUGFS_ATTRIBUTE(huc_info); + +void intel_huc_debugfs_register(struct intel_huc *huc, struct dentry *root) +{ + static const struct debugfs_gt_file files[] = { + { "huc_info", &huc_info_fops, NULL }, + }; + + if (!intel_huc_is_supported(huc)) + return; + + intel_gt_debugfs_register_files(root, files, ARRAY_SIZE(files), huc); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.h b/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.h new file mode 100644 index 000000000000..be79e992f976 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef DEBUGFS_HUC_H +#define DEBUGFS_HUC_H + +struct intel_huc; +struct dentry; + +void intel_huc_debugfs_register(struct intel_huc *huc, struct dentry *root); + +#endif /* DEBUGFS_HUC_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c index eee193bf2cc4..e5ef509c70e8 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.c @@ -8,23 +8,6 @@ #include "i915_drv.h" /** - * intel_huc_fw_init_early() - initializes HuC firmware struct - * @huc: intel_huc struct - * - * On platforms with HuC selects firmware for uploading - */ -void intel_huc_fw_init_early(struct intel_huc *huc) -{ - struct intel_gt *gt = huc_to_gt(huc); - struct intel_uc *uc = >->uc; - struct drm_i915_private *i915 = gt->i915; - - intel_uc_fw_init_early(&huc->fw, INTEL_UC_FW_TYPE_HUC, - intel_uc_uses_guc(uc), - INTEL_INFO(i915)->platform, INTEL_REVID(i915)); -} - -/** * intel_huc_fw_upload() - load HuC uCode to device * @huc: intel_huc structure * diff --git a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h index b791269ce923..12f264ee3e0b 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_huc_fw.h @@ -8,7 +8,6 @@ struct intel_huc; -void intel_huc_fw_init_early(struct intel_huc *huc); int intel_huc_fw_upload(struct intel_huc *huc); #endif diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c index affc4d6f9ead..f518fe05c6f9 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c @@ -45,43 +45,43 @@ static void __confirm_options(struct intel_uc *uc) { struct drm_i915_private *i915 = uc_to_gt(uc)->i915; - DRM_DEV_DEBUG_DRIVER(i915->drm.dev, - "enable_guc=%d (guc:%s submission:%s huc:%s)\n", - i915_modparams.enable_guc, - yesno(intel_uc_uses_guc(uc)), - yesno(intel_uc_uses_guc_submission(uc)), - yesno(intel_uc_uses_huc(uc))); + drm_dbg(&i915->drm, + "enable_guc=%d (guc:%s submission:%s huc:%s)\n", + i915_modparams.enable_guc, + yesno(intel_uc_wants_guc(uc)), + yesno(intel_uc_wants_guc_submission(uc)), + yesno(intel_uc_wants_huc(uc))); if (i915_modparams.enable_guc == -1) return; if (i915_modparams.enable_guc == 0) { - GEM_BUG_ON(intel_uc_uses_guc(uc)); - GEM_BUG_ON(intel_uc_uses_guc_submission(uc)); - GEM_BUG_ON(intel_uc_uses_huc(uc)); + GEM_BUG_ON(intel_uc_wants_guc(uc)); + GEM_BUG_ON(intel_uc_wants_guc_submission(uc)); + GEM_BUG_ON(intel_uc_wants_huc(uc)); return; } if (!intel_uc_supports_guc(uc)) - dev_info(i915->drm.dev, + drm_info(&i915->drm, "Incompatible option enable_guc=%d - %s\n", i915_modparams.enable_guc, "GuC is not supported!"); if (i915_modparams.enable_guc & ENABLE_GUC_LOAD_HUC && !intel_uc_supports_huc(uc)) - dev_info(i915->drm.dev, + drm_info(&i915->drm, "Incompatible option enable_guc=%d - %s\n", i915_modparams.enable_guc, "HuC is not supported!"); if (i915_modparams.enable_guc & ENABLE_GUC_SUBMISSION && !intel_uc_supports_guc_submission(uc)) - dev_info(i915->drm.dev, + drm_info(&i915->drm, "Incompatible option enable_guc=%d - %s\n", i915_modparams.enable_guc, "GuC submission is N/A"); if (i915_modparams.enable_guc & ~(ENABLE_GUC_SUBMISSION | ENABLE_GUC_LOAD_HUC)) - dev_info(i915->drm.dev, + drm_info(&i915->drm, "Incompatible option enable_guc=%d - %s\n", i915_modparams.enable_guc, "undocumented flag"); } @@ -93,7 +93,7 @@ void intel_uc_init_early(struct intel_uc *uc) __confirm_options(uc); - if (intel_uc_uses_guc(uc)) + if (intel_uc_wants_guc(uc)) uc->ops = &uc_ops_on; else uc->ops = &uc_ops_off; @@ -131,6 +131,13 @@ static void __uc_free_load_err_log(struct intel_uc *uc) i915_gem_object_put(log); } +void intel_uc_driver_remove(struct intel_uc *uc) +{ + intel_uc_fini_hw(uc); + intel_uc_fini(uc); + __uc_free_load_err_log(uc); +} + static inline bool guc_communication_enabled(struct intel_guc *guc) { return intel_guc_ct_enabled(&guc->ct); @@ -257,13 +264,13 @@ static void __uc_fetch_firmwares(struct intel_uc *uc) { int err; - GEM_BUG_ON(!intel_uc_uses_guc(uc)); + GEM_BUG_ON(!intel_uc_wants_guc(uc)); err = intel_uc_fw_fetch(&uc->guc.fw); if (err) return; - if (intel_uc_uses_huc(uc)) + if (intel_uc_wants_huc(uc)) intel_uc_fw_fetch(&uc->huc.fw); } @@ -273,33 +280,44 @@ static void __uc_cleanup_firmwares(struct intel_uc *uc) intel_uc_fw_cleanup_fetch(&uc->guc.fw); } -static void __uc_init(struct intel_uc *uc) +static int __uc_init(struct intel_uc *uc) { struct intel_guc *guc = &uc->guc; struct intel_huc *huc = &uc->huc; int ret; - GEM_BUG_ON(!intel_uc_uses_guc(uc)); + GEM_BUG_ON(!intel_uc_wants_guc(uc)); + + if (!intel_uc_uses_guc(uc)) + return 0; + + if (i915_inject_probe_failure(uc_to_gt(uc)->i915)) + return -ENOMEM; /* XXX: GuC submission is unavailable for now */ - GEM_BUG_ON(intel_uc_supports_guc_submission(uc)); + GEM_BUG_ON(intel_uc_uses_guc_submission(uc)); ret = intel_guc_init(guc); - if (ret) { - intel_uc_fw_cleanup_fetch(&huc->fw); - return; + if (ret) + return ret; + + if (intel_uc_uses_huc(uc)) { + ret = intel_huc_init(huc); + if (ret) + goto out_guc; } - if (intel_uc_uses_huc(uc)) - intel_huc_init(huc); + return 0; + +out_guc: + intel_guc_fini(guc); + return ret; } static void __uc_fini(struct intel_uc *uc) { intel_huc_fini(&uc->huc); intel_guc_fini(&uc->guc); - - __uc_free_load_err_log(uc); } static int __uc_sanitize(struct intel_uc *uc) @@ -402,12 +420,12 @@ static int __uc_init_hw(struct intel_uc *uc) int ret, attempts; GEM_BUG_ON(!intel_uc_supports_guc(uc)); - GEM_BUG_ON(!intel_uc_uses_guc(uc)); + GEM_BUG_ON(!intel_uc_wants_guc(uc)); - if (!intel_uc_fw_is_available(&guc->fw)) { + if (!intel_uc_fw_is_loadable(&guc->fw)) { ret = __uc_check_hw(uc) || intel_uc_fw_is_overridden(&guc->fw) || - intel_uc_supports_guc_submission(uc) ? + intel_uc_wants_guc_submission(uc) ? intel_uc_fw_status_to_error(guc->fw.status) : 0; goto err_out; } @@ -459,17 +477,17 @@ static int __uc_init_hw(struct intel_uc *uc) if (ret) goto err_communication; - if (intel_uc_supports_guc_submission(uc)) + if (intel_uc_uses_guc_submission(uc)) intel_guc_submission_enable(guc); - dev_info(i915->drm.dev, "%s firmware %s version %u.%u %s:%s\n", + drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_GUC), guc->fw.path, guc->fw.major_ver_found, guc->fw.minor_ver_found, "submission", - enableddisabled(intel_uc_supports_guc_submission(uc))); + enableddisabled(intel_uc_uses_guc_submission(uc))); if (intel_uc_uses_huc(uc)) { - dev_info(i915->drm.dev, "%s firmware %s version %u.%u %s:%s\n", + drm_info(&i915->drm, "%s firmware %s version %u.%u %s:%s\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_HUC), huc->fw.path, huc->fw.major_ver_found, huc->fw.minor_ver_found, @@ -490,7 +508,7 @@ err_out: __uc_sanitize(uc); if (!ret) { - dev_notice(i915->drm.dev, "GuC is uninitialized\n"); + drm_notice(&i915->drm, "GuC is uninitialized\n"); /* We want to run without GuC submission */ return 0; } @@ -508,7 +526,7 @@ static void __uc_fini_hw(struct intel_uc *uc) if (!intel_guc_is_fw_running(guc)) return; - if (intel_uc_supports_guc_submission(uc)) + if (intel_uc_uses_guc_submission(uc)) intel_guc_submission_disable(guc); if (guc_communication_enabled(guc)) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.h b/drivers/gpu/drm/i915/gt/uc/intel_uc.h index 49c913524686..9c954c589edf 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.h @@ -7,6 +7,7 @@ #define _INTEL_UC_H_ #include "intel_guc.h" +#include "intel_guc_submission.h" #include "intel_huc.h" #include "i915_params.h" @@ -16,7 +17,7 @@ struct intel_uc_ops { int (*sanitize)(struct intel_uc *uc); void (*init_fw)(struct intel_uc *uc); void (*fini_fw)(struct intel_uc *uc); - void (*init)(struct intel_uc *uc); + int (*init)(struct intel_uc *uc); void (*fini)(struct intel_uc *uc); int (*init_hw)(struct intel_uc *uc); void (*fini_hw)(struct intel_uc *uc); @@ -33,6 +34,7 @@ struct intel_uc { void intel_uc_init_early(struct intel_uc *uc); void intel_uc_driver_late_release(struct intel_uc *uc); +void intel_uc_driver_remove(struct intel_uc *uc); void intel_uc_init_mmio(struct intel_uc *uc); void intel_uc_reset_prepare(struct intel_uc *uc); void intel_uc_suspend(struct intel_uc *uc); @@ -40,35 +42,44 @@ void intel_uc_runtime_suspend(struct intel_uc *uc); int intel_uc_resume(struct intel_uc *uc); int intel_uc_runtime_resume(struct intel_uc *uc); -static inline bool intel_uc_supports_guc(struct intel_uc *uc) -{ - return intel_guc_is_supported(&uc->guc); -} - -static inline bool intel_uc_uses_guc(struct intel_uc *uc) -{ - return intel_guc_is_enabled(&uc->guc); -} +/* + * We need to know as early as possible if we're going to use GuC or not to + * take the correct setup paths. Additionally, once we've started loading the + * GuC, it is unsafe to keep executing without it because some parts of the HW, + * a subset of which is not cleaned on GT reset, will start expecting the GuC FW + * to be running. + * To solve both these requirements, we commit to using the microcontrollers if + * the relevant modparam is set and the blobs are found on the system. At this + * stage, the only thing that can stop us from attempting to load the blobs on + * the HW and use them is a fundamental issue (e.g. no memory for our + * structures); if we hit such a problem during driver load we're broken even + * without GuC, so there is no point in trying to fall back. + * + * Given the above, we can be in one of 4 states, with the last one implying + * we're committed to using the microcontroller: + * - Not supported: not available in HW and/or firmware not defined. + * - Supported: available in HW and firmware defined. + * - Wanted: supported + enabled in modparam. + * - In use: wanted + firmware found on the system and successfully fetched. + */ -static inline bool intel_uc_supports_guc_submission(struct intel_uc *uc) -{ - return intel_guc_is_submission_supported(&uc->guc); +#define __uc_state_checker(x, func, state, required) \ +static inline bool intel_uc_##state##_##func(struct intel_uc *uc) \ +{ \ + return intel_##func##_is_##required(&uc->x); \ } -static inline bool intel_uc_uses_guc_submission(struct intel_uc *uc) -{ - return intel_guc_is_submission_supported(&uc->guc); -} +#define uc_state_checkers(x, func) \ +__uc_state_checker(x, func, supports, supported) \ +__uc_state_checker(x, func, wants, wanted) \ +__uc_state_checker(x, func, uses, used) -static inline bool intel_uc_supports_huc(struct intel_uc *uc) -{ - return intel_uc_supports_guc(uc); -} +uc_state_checkers(guc, guc); +uc_state_checkers(huc, huc); +uc_state_checkers(guc, guc_submission); -static inline bool intel_uc_uses_huc(struct intel_uc *uc) -{ - return intel_huc_is_enabled(&uc->huc); -} +#undef uc_state_checkers +#undef __uc_state_checker #define intel_uc_ops_function(_NAME, _OPS, _TYPE, _RET) \ static inline _TYPE intel_uc_##_NAME(struct intel_uc *uc) \ @@ -80,7 +91,7 @@ static inline _TYPE intel_uc_##_NAME(struct intel_uc *uc) \ intel_uc_ops_function(sanitize, sanitize, int, 0); intel_uc_ops_function(fetch_firmwares, init_fw, void, ); intel_uc_ops_function(cleanup_firmwares, fini_fw, void, ); -intel_uc_ops_function(init, init, void, ); +intel_uc_ops_function(init, init, int, 0); intel_uc_ops_function(fini, fini, void, ); intel_uc_ops_function(init_hw, init_hw, int, 0); intel_uc_ops_function(fini_hw, fini_hw, void, ); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c new file mode 100644 index 000000000000..9d16b784aa0d --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <linux/debugfs.h> + +#include "intel_guc_debugfs.h" +#include "intel_huc_debugfs.h" +#include "intel_uc.h" +#include "intel_uc_debugfs.h" + +void intel_uc_debugfs_register(struct intel_uc *uc, struct dentry *gt_root) +{ + struct dentry *root; + + if (!gt_root) + return; + + /* GuC and HuC go always in pair, no need to check both */ + if (!intel_uc_supports_guc(uc)) + return; + + root = debugfs_create_dir("uc", gt_root); + if (IS_ERR(root)) + return; + + intel_guc_debugfs_register(&uc->guc, root); + intel_huc_debugfs_register(&uc->huc, root); +} diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.h new file mode 100644 index 000000000000..010ce250d223 --- /dev/null +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef DEBUGFS_UC_H +#define DEBUGFS_UC_H + +struct intel_uc; +struct dentry; + +void intel_uc_debugfs_register(struct intel_uc *uc, struct dentry *gt_root); + +#endif /* DEBUGFS_UC_H */ diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c index 8ee0a0c7f447..e1caae93996d 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c @@ -11,26 +11,32 @@ #include "intel_uc_fw_abi.h" #include "i915_drv.h" -static inline struct intel_gt *__uc_fw_to_gt(struct intel_uc_fw *uc_fw) +static inline struct intel_gt * +____uc_fw_to_gt(struct intel_uc_fw *uc_fw, enum intel_uc_fw_type type) { - GEM_BUG_ON(uc_fw->status == INTEL_UC_FIRMWARE_UNINITIALIZED); - if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) + if (type == INTEL_UC_FW_TYPE_GUC) return container_of(uc_fw, struct intel_gt, uc.guc.fw); - GEM_BUG_ON(uc_fw->type != INTEL_UC_FW_TYPE_HUC); + GEM_BUG_ON(type != INTEL_UC_FW_TYPE_HUC); return container_of(uc_fw, struct intel_gt, uc.huc.fw); } +static inline struct intel_gt *__uc_fw_to_gt(struct intel_uc_fw *uc_fw) +{ + GEM_BUG_ON(uc_fw->status == INTEL_UC_FIRMWARE_UNINITIALIZED); + return ____uc_fw_to_gt(uc_fw, uc_fw->type); +} + #ifdef CONFIG_DRM_I915_DEBUG_GUC void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, enum intel_uc_fw_status status) { uc_fw->__status = status; - DRM_DEV_DEBUG_DRIVER(__uc_fw_to_gt(uc_fw)->i915->drm.dev, - "%s firmware -> %s\n", - intel_uc_fw_type_repr(uc_fw->type), - status == INTEL_UC_FIRMWARE_SELECTED ? - uc_fw->path : intel_uc_fw_status_repr(status)); + drm_dbg(&__uc_fw_to_gt(uc_fw)->i915->drm, + "%s firmware -> %s\n", + intel_uc_fw_type_repr(uc_fw->type), + status == INTEL_UC_FIRMWARE_SELECTED ? + uc_fw->path : intel_uc_fw_status_repr(status)); } #endif @@ -43,7 +49,7 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw, * features. */ #define INTEL_UC_FIRMWARE_DEFS(fw_def, guc_def, huc_def) \ - fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 0, 3)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 35, 2, 0), huc_def(tgl, 7, 0, 12)) \ fw_def(ELKHARTLAKE, 0, guc_def(ehl, 33, 0, 4), huc_def(ehl, 9, 0, 0)) \ fw_def(ICELAKE, 0, guc_def(icl, 33, 0, 0), huc_def(icl, 9, 0, 0)) \ fw_def(COFFEELAKE, 5, guc_def(cml, 33, 0, 0), huc_def(cml, 4, 0, 0)) \ @@ -187,17 +193,15 @@ static void __uc_fw_user_override(struct intel_uc_fw *uc_fw) * intel_uc_fw_init_early - initialize the uC object and select the firmware * @uc_fw: uC firmware * @type: type of uC - * @supported: is uC support possible - * @platform: platform identifier - * @rev: hardware revision * * Initialize the state of our uC object and relevant tracking and select the * firmware to fetch and load. */ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw, - enum intel_uc_fw_type type, bool supported, - enum intel_platform platform, u8 rev) + enum intel_uc_fw_type type) { + struct drm_i915_private *i915 = ____uc_fw_to_gt(uc_fw, type)->i915; + /* * we use FIRMWARE_UNINITIALIZED to detect checks against uc_fw->status * before we're looked at the HW caps to see if we have uc support @@ -208,8 +212,10 @@ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw, uc_fw->type = type; - if (supported) { - __uc_fw_auto_select(uc_fw, platform, rev); + if (HAS_GT_UC(i915)) { + __uc_fw_auto_select(uc_fw, + INTEL_INFO(i915)->platform, + INTEL_REVID(i915)); __uc_fw_user_override(uc_fw); } @@ -279,7 +285,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) err = i915_inject_probe_error(i915, -ENXIO); if (err) - return err; + goto fail; __force_fw_fetch_failures(uc_fw, -EINVAL); __force_fw_fetch_failures(uc_fw, -ESTALE); @@ -290,7 +296,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) /* Check the size of the blob before examining buffer contents */ if (unlikely(fw->size < sizeof(struct uc_css_header))) { - dev_warn(dev, "%s firmware %s: invalid size: %zu < %zu\n", + drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, fw->size, sizeof(struct uc_css_header)); err = -ENODATA; @@ -303,7 +309,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) size = (css->header_size_dw - css->key_size_dw - css->modulus_size_dw - css->exponent_size_dw) * sizeof(u32); if (unlikely(size != sizeof(struct uc_css_header))) { - dev_warn(dev, + drm_warn(&i915->drm, "%s firmware %s: unexpected header size: %zu != %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, fw->size, sizeof(struct uc_css_header)); @@ -316,7 +322,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) /* now RSA */ if (unlikely(css->key_size_dw != UOS_RSA_SCRATCH_COUNT)) { - dev_warn(dev, "%s firmware %s: unexpected key size: %u != %u\n", + drm_warn(&i915->drm, "%s firmware %s: unexpected key size: %u != %u\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, css->key_size_dw, UOS_RSA_SCRATCH_COUNT); err = -EPROTO; @@ -327,7 +333,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) /* At least, it should have header, uCode and RSA. Size of all three. */ size = sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->rsa_size; if (unlikely(fw->size < size)) { - dev_warn(dev, "%s firmware %s: invalid size: %zu < %zu\n", + drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, fw->size, size); err = -ENOEXEC; @@ -337,7 +343,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) /* Sanity check whether this fw is not larger than whole WOPCM memory */ size = __intel_uc_fw_get_upload_size(uc_fw); if (unlikely(size >= i915->wopcm.size)) { - dev_warn(dev, "%s firmware %s: invalid size: %zu > %zu\n", + drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu > %zu\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, size, (size_t)i915->wopcm.size); err = -E2BIG; @@ -352,7 +358,7 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw) if (uc_fw->major_ver_found != uc_fw->major_ver_wanted || uc_fw->minor_ver_found < uc_fw->minor_ver_wanted) { - dev_notice(dev, "%s firmware %s: unexpected version: %u.%u != %u.%u\n", + drm_notice(&i915->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, uc_fw->major_ver_found, uc_fw->minor_ver_found, uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted); @@ -380,9 +386,9 @@ fail: INTEL_UC_FIRMWARE_MISSING : INTEL_UC_FIRMWARE_ERROR); - dev_notice(dev, "%s firmware %s: fetch failed with error %d\n", + drm_notice(&i915->drm, "%s firmware %s: fetch failed with error %d\n", intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, err); - dev_info(dev, "%s firmware(s) can be downloaded from %s\n", + drm_info(&i915->drm, "%s firmware(s) can be downloaded from %s\n", intel_uc_fw_type_repr(uc_fw->type), INTEL_UC_FIRMWARE_URL); release_firmware(fw); /* OK even if fw is NULL */ @@ -467,7 +473,7 @@ static int uc_fw_xfer(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags) /* Wait for DMA to finish */ ret = intel_wait_for_register_fw(uncore, DMA_CTRL, START_DMA, 0, 100); if (ret) - dev_err(gt->i915->drm.dev, "DMA for %s fw failed, DMA_CTRL=%u\n", + drm_err(>->i915->drm, "DMA for %s fw failed, DMA_CTRL=%u\n", intel_uc_fw_type_repr(uc_fw->type), intel_uncore_read_fw(uncore, DMA_CTRL)); @@ -501,7 +507,7 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags) if (err) return err; - if (!intel_uc_fw_is_available(uc_fw)) + if (!intel_uc_fw_is_loadable(uc_fw)) return -ENOEXEC; /* Call custom loader */ @@ -544,7 +550,10 @@ int intel_uc_fw_init(struct intel_uc_fw *uc_fw) void intel_uc_fw_fini(struct intel_uc_fw *uc_fw) { - intel_uc_fw_cleanup_fetch(uc_fw); + if (i915_gem_object_has_pinned_pages(uc_fw->obj)) + i915_gem_object_unpin_pages(uc_fw->obj); + + intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE); } /** diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h index 1f30543d0d2d..23d3a423ac0f 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h @@ -29,8 +29,11 @@ struct intel_gt; * | | SELECTED | * +------------+- / | \ -+ * | | MISSING <--/ | \--> ERROR | - * | fetch | | | - * | | /------> AVAILABLE <---<-----------\ | + * | fetch | V | + * | | AVAILABLE | + * +------------+- | -+ + * | init | V | + * | | /------> LOADABLE <----<-----------\ | * +------------+- \ / \ \ \ -+ * | | FAIL <--< \--> TRANSFERRED \ | * | upload | \ / \ / | @@ -46,6 +49,7 @@ enum intel_uc_fw_status { INTEL_UC_FIRMWARE_MISSING, /* blob not found on the system */ INTEL_UC_FIRMWARE_ERROR, /* invalid format or version */ INTEL_UC_FIRMWARE_AVAILABLE, /* blob found and copied in mem */ + INTEL_UC_FIRMWARE_LOADABLE, /* all fw-required objects are ready */ INTEL_UC_FIRMWARE_FAIL, /* failed to xfer or init/auth the fw */ INTEL_UC_FIRMWARE_TRANSFERRED, /* dma xfer done */ INTEL_UC_FIRMWARE_RUNNING /* init/auth done */ @@ -115,6 +119,8 @@ const char *intel_uc_fw_status_repr(enum intel_uc_fw_status status) return "ERROR"; case INTEL_UC_FIRMWARE_AVAILABLE: return "AVAILABLE"; + case INTEL_UC_FIRMWARE_LOADABLE: + return "LOADABLE"; case INTEL_UC_FIRMWARE_FAIL: return "FAIL"; case INTEL_UC_FIRMWARE_TRANSFERRED: @@ -143,6 +149,7 @@ static inline int intel_uc_fw_status_to_error(enum intel_uc_fw_status status) case INTEL_UC_FIRMWARE_SELECTED: return -ESTALE; case INTEL_UC_FIRMWARE_AVAILABLE: + case INTEL_UC_FIRMWARE_LOADABLE: case INTEL_UC_FIRMWARE_TRANSFERRED: case INTEL_UC_FIRMWARE_RUNNING: return 0; @@ -184,6 +191,11 @@ static inline bool intel_uc_fw_is_available(struct intel_uc_fw *uc_fw) return __intel_uc_fw_status(uc_fw) >= INTEL_UC_FIRMWARE_AVAILABLE; } +static inline bool intel_uc_fw_is_loadable(struct intel_uc_fw *uc_fw) +{ + return __intel_uc_fw_status(uc_fw) >= INTEL_UC_FIRMWARE_LOADABLE; +} + static inline bool intel_uc_fw_is_loaded(struct intel_uc_fw *uc_fw) { return __intel_uc_fw_status(uc_fw) >= INTEL_UC_FIRMWARE_TRANSFERRED; @@ -202,7 +214,7 @@ static inline bool intel_uc_fw_is_overridden(const struct intel_uc_fw *uc_fw) static inline void intel_uc_fw_sanitize(struct intel_uc_fw *uc_fw) { if (intel_uc_fw_is_loaded(uc_fw)) - intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_AVAILABLE); + intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_LOADABLE); } static inline u32 __intel_uc_fw_get_upload_size(struct intel_uc_fw *uc_fw) @@ -227,8 +239,7 @@ static inline u32 intel_uc_fw_get_upload_size(struct intel_uc_fw *uc_fw) } void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw, - enum intel_uc_fw_type type, bool supported, - enum intel_platform platform, u8 rev); + enum intel_uc_fw_type type); int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw); void intel_uc_fw_cleanup_fetch(struct intel_uc_fw *uc_fw); int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 offset, u32 dma_flags); diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c index 8b13f091cee2..0d6d59871308 100644 --- a/drivers/gpu/drm/i915/gvt/aperture_gm.c +++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c @@ -35,7 +35,7 @@ */ #include "i915_drv.h" -#include "i915_gem_fence_reg.h" +#include "gt/intel_ggtt_fencing.h" #include "gvt.h" static int alloc_gm(struct intel_vgpu *vgpu, bool high_gm) diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index 956e5a9fd11b..8b87f130f7f1 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -164,6 +164,7 @@ struct decode_info { #define OP_STATE_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x01) #define OP_STATE_SIP OP_3D_MEDIA(0x0, 0x1, 0x02) #define OP_3D_MEDIA_0_1_4 OP_3D_MEDIA(0x0, 0x1, 0x04) +#define OP_SWTESS_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x03) #define OP_3DSTATE_VF_STATISTICS_GM45 OP_3D_MEDIA(0x1, 0x0, 0x0B) @@ -1012,18 +1013,6 @@ static int cmd_handler_lri(struct parser_exec_state *s) { int i, ret = 0; int cmd_len = cmd_length(s); - u32 valid_len = CMD_LEN(1); - - /* - * Official intel docs are somewhat sloppy , check the definition of - * MI_LOAD_REGISTER_IMM. - */ - #define MAX_VALID_LEN 127 - if ((cmd_len < valid_len) || (cmd_len > MAX_VALID_LEN)) { - gvt_err("len is not valid: len=%u valid_len=%u\n", - cmd_len, valid_len); - return -EFAULT; - } for (i = 1; i < cmd_len; i += 2) { if (IS_BROADWELL(s->engine->i915) && s->engine->id != RCS0) { @@ -2530,6 +2519,9 @@ static const struct cmd_info cmd_info[] = { {"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL, ADDR_FIX_1(1), 8, NULL}, + {"OP_SWTESS_BASE_ADDRESS", OP_SWTESS_BASE_ADDRESS, + F_LEN_VAR, R_RCS, D_ALL, ADDR_FIX_2(1, 2), 3, NULL}, + {"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, {"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index 6e5c9885d9fe..a83df2f84eb9 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -221,7 +221,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= - (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_B << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { @@ -241,7 +241,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= - (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_C << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { @@ -261,7 +261,7 @@ static void emulate_monitor_status_change(struct intel_vgpu *vgpu) ~(TRANS_DDI_BPC_MASK | TRANS_DDI_MODE_SELECT_MASK | TRANS_DDI_PORT_MASK); vgpu_vreg_t(vgpu, TRANS_DDI_FUNC_CTL(TRANSCODER_A)) |= - (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DVI | + (TRANS_DDI_BPC_8 | TRANS_DDI_MODE_SELECT_DP_SST | (PORT_D << TRANS_DDI_PORT_SHIFT) | TRANS_DDI_FUNC_ENABLE); if (IS_BROADWELL(dev_priv)) { diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c index 4ff60c793a21..37fc460414a8 100644 --- a/drivers/gpu/drm/i915/gvt/dmabuf.c +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -151,12 +151,12 @@ static void dmabuf_gem_object_free(struct kref *kref) dmabuf_obj = container_of(pos, struct intel_vgpu_dmabuf_obj, list); if (dmabuf_obj == obj) { + list_del(pos); intel_gvt_hypervisor_put_vfio_device(vgpu); idr_remove(&vgpu->object_idr, dmabuf_obj->dmabuf_id); kfree(dmabuf_obj->info); kfree(dmabuf_obj); - list_del(pos); break; } } diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c index 7aaae9c562f8..990a181094e3 100644 --- a/drivers/gpu/drm/i915/gvt/firmware.c +++ b/drivers/gpu/drm/i915/gvt/firmware.c @@ -144,7 +144,7 @@ void intel_gvt_free_firmware(struct intel_gvt *gvt) clean_firmware_sysfs(gvt); kfree(gvt->firmware.cfg_space); - kfree(gvt->firmware.mmio); + vfree(gvt->firmware.mmio); } static int verify_firmware(struct intel_gvt *gvt, @@ -225,7 +225,7 @@ int intel_gvt_load_firmware(struct intel_gvt *gvt) firmware->cfg_space = mem; - mem = kmalloc(info->mmio_size, GFP_KERNEL); + mem = vmalloc(info->mmio_size); if (!mem) { kfree(path); kfree(firmware->cfg_space); diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index 1959a6aea5bb..210016192ce7 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -1963,7 +1963,11 @@ void _intel_vgpu_mm_release(struct kref *mm_ref) if (mm->type == INTEL_GVT_MM_PPGTT) { list_del(&mm->ppgtt_mm.list); + + mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); list_del(&mm->ppgtt_mm.lru_list); + mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); + invalidate_ppgtt_mm(mm); } else { vfree(mm->ggtt_mm.virtual_ggtt); diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index ed7be1b39a06..c7c561237883 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -34,6 +34,7 @@ #include <linux/kthread.h> #include "i915_drv.h" +#include "intel_gvt.h" #include "gvt.h" #include <linux/vfio.h> #include <linux/mdev.h> diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index 23a3193a6654..3e88e3b5c43a 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -462,11 +462,14 @@ static int pipeconf_mmio_write(struct intel_vgpu *vgpu, unsigned int offset, return 0; } -/* ascendingly sorted */ +/* sorted in ascending order */ static i915_reg_t force_nonpriv_white_list[] = { + _MMIO(0xd80), GEN9_CS_DEBUG_MODE1, //_MMIO(0x20ec) GEN9_CTX_PREEMPT_REG,//_MMIO(0x2248) - PS_INVOCATION_COUNT,//_MMIO(0x2348) + CL_PRIMITIVES_COUNT, //_MMIO(0x2340) + PS_INVOCATION_COUNT, //_MMIO(0x2348) + PS_DEPTH_COUNT, //_MMIO(0x2350) GEN8_CS_CHICKEN1,//_MMIO(0x2580) _MMIO(0x2690), _MMIO(0x2694), @@ -491,6 +494,7 @@ static i915_reg_t force_nonpriv_white_list[] = { _MMIO(0xe18c), _MMIO(0xe48c), _MMIO(0xe5f4), + _MMIO(0x64844), }; /* a simple bsearch */ diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c index 867e7629025b..33569b910ed5 100644 --- a/drivers/gpu/drm/i915/gvt/opregion.c +++ b/drivers/gpu/drm/i915/gvt/opregion.c @@ -147,15 +147,14 @@ static void virt_vbt_generation(struct vbt *v) /* there's features depending on version! */ v->header.version = 155; v->header.header_size = sizeof(v->header); - v->header.vbt_size = sizeof(struct vbt) - sizeof(v->header); + v->header.vbt_size = sizeof(struct vbt); v->header.bdb_offset = offsetof(struct vbt, bdb_header); strcpy(&v->bdb_header.signature[0], "BIOS_DATA_BLOCK"); v->bdb_header.version = 186; /* child_dev_size = 33 */ v->bdb_header.header_size = sizeof(v->bdb_header); - v->bdb_header.bdb_size = sizeof(struct vbt) - sizeof(struct vbt_header) - - sizeof(struct bdb_header); + v->bdb_header.bdb_size = sizeof(struct vbt) - sizeof(struct vbt_header); /* general features */ v->general_features_header.id = BDB_GENERAL_FEATURES; diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index 54c10dfcd3d2..c00189432b58 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -327,17 +327,17 @@ static void shadow_context_descriptor_update(struct intel_context *ce, struct intel_vgpu_workload *workload) { - u64 desc = ce->lrc_desc; + u64 desc = ce->lrc.desc; /* * Update bits 0-11 of the context descriptor which includes flags * like GEN8_CTX_* cached in desc_template */ - desc &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT); - desc |= workload->ctx_desc.addressing_mode << + desc &= ~(0x3ull << GEN8_CTX_ADDRESSING_MODE_SHIFT); + desc |= (u64)workload->ctx_desc.addressing_mode << GEN8_CTX_ADDRESSING_MODE_SHIFT; - ce->lrc_desc = desc; + ce->lrc.desc = desc; } static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) @@ -632,10 +632,9 @@ static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) if (bb->va && !IS_ERR(bb->va)) i915_gem_object_unpin_map(bb->obj); - if (bb->vma && !IS_ERR(bb->vma)) { + if (bb->vma && !IS_ERR(bb->vma)) i915_vma_unpin(bb->vma); - i915_vma_close(bb->vma); - } + i915_gem_object_put(bb->obj); } list_del(&bb->list); @@ -1373,7 +1372,8 @@ int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) ce->vm = i915_vm_get(&ppgtt->vm); intel_context_set_single_submission(ce); - if (!USES_GUC_SUBMISSION(i915)) { /* Max ring buffer size */ + /* Max ring buffer size */ + if (!intel_uc_wants_guc_submission(&engine->gt->uc)) { const unsigned int ring_size = 512 * SZ_4K; ce->ring = __intel_context_ring_size(ring_size); diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index abcde8ce1a9a..1d5ff88078bd 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -274,10 +274,17 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu) struct intel_gvt *gvt = vgpu->gvt; struct drm_i915_private *i915 = gvt->gt->i915; - mutex_lock(&vgpu->vgpu_lock); - drm_WARN(&i915->drm, vgpu->active, "vGPU is still active!\n"); + /* + * remove idr first so later clean can judge if need to stop + * service if no active vgpu. + */ + mutex_lock(&gvt->lock); + idr_remove(&gvt->vgpu_idr, vgpu->id); + mutex_unlock(&gvt->lock); + + mutex_lock(&vgpu->vgpu_lock); intel_gvt_debugfs_remove_vgpu(vgpu); intel_vgpu_clean_sched_policy(vgpu); intel_vgpu_clean_submission(vgpu); @@ -292,7 +299,6 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu) mutex_unlock(&vgpu->vgpu_lock); mutex_lock(&gvt->lock); - idr_remove(&gvt->vgpu_idr, vgpu->id); if (idr_is_empty(&gvt->vgpu_idr)) intel_gvt_clean_irq(gvt); intel_gvt_update_vgpu_types(gvt); @@ -560,9 +566,9 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, intel_vgpu_reset_mmio(vgpu, dmlr); populate_pvinfo_page(vgpu); - intel_vgpu_reset_display(vgpu); if (dmlr) { + intel_vgpu_reset_display(vgpu); intel_vgpu_reset_cfg_space(vgpu); /* only reset the failsafe mode when dmlr reset */ vgpu->failsafe = false; diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index 9ccb931a733e..d960d0be5bd2 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -7,6 +7,7 @@ #include <linux/debugobjects.h> #include "gt/intel_context.h" +#include "gt/intel_engine_heartbeat.h" #include "gt/intel_engine_pm.h" #include "gt/intel_ring.h" @@ -452,6 +453,9 @@ static void enable_signaling(struct i915_active_fence *active) { struct dma_fence *fence; + if (unlikely(is_barrier(active))) + return; + fence = i915_active_fence_get(active); if (!fence) return; @@ -460,58 +464,193 @@ static void enable_signaling(struct i915_active_fence *active) dma_fence_put(fence); } -int i915_active_wait(struct i915_active *ref) +static int flush_barrier(struct active_node *it) { - struct active_node *it, *n; - int err = 0; + struct intel_engine_cs *engine; - might_sleep(); + if (likely(!is_barrier(&it->base))) + return 0; - if (!i915_active_acquire_if_busy(ref)) + engine = __barrier_to_engine(it); + smp_rmb(); /* serialise with add_active_barriers */ + if (!is_barrier(&it->base)) return 0; - /* Flush lazy signals */ + return intel_engine_flush_barriers(engine); +} + +static int flush_lazy_signals(struct i915_active *ref) +{ + struct active_node *it, *n; + int err = 0; + enable_signaling(&ref->excl); rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { - if (is_barrier(&it->base)) /* unconnected idle barrier */ - continue; + err = flush_barrier(it); /* unconnected idle barrier? */ + if (err) + break; enable_signaling(&it->base); } - /* Any fence added after the wait begins will not be auto-signaled */ + return err; +} + +int __i915_active_wait(struct i915_active *ref, int state) +{ + int err; + + might_sleep(); + + if (!i915_active_acquire_if_busy(ref)) + return 0; + + /* Any fence added after the wait begins will not be auto-signaled */ + err = flush_lazy_signals(ref); i915_active_release(ref); if (err) return err; - if (wait_var_event_interruptible(ref, i915_active_is_idle(ref))) + if (!i915_active_is_idle(ref) && + ___wait_var_event(ref, i915_active_is_idle(ref), + state, 0, 0, schedule())) return -EINTR; flush_work(&ref->work); return 0; } -int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) +static int __await_active(struct i915_active_fence *active, + int (*fn)(void *arg, struct dma_fence *fence), + void *arg) +{ + struct dma_fence *fence; + + if (is_barrier(active)) /* XXX flush the barrier? */ + return 0; + + fence = i915_active_fence_get(active); + if (fence) { + int err; + + err = fn(arg, fence); + dma_fence_put(fence); + if (err < 0) + return err; + } + + return 0; +} + +struct wait_barrier { + struct wait_queue_entry base; + struct i915_active *ref; +}; + +static int +barrier_wake(wait_queue_entry_t *wq, unsigned int mode, int flags, void *key) +{ + struct wait_barrier *wb = container_of(wq, typeof(*wb), base); + + if (i915_active_is_idle(wb->ref)) { + list_del(&wq->entry); + i915_sw_fence_complete(wq->private); + kfree(wq); + } + + return 0; +} + +static int __await_barrier(struct i915_active *ref, struct i915_sw_fence *fence) +{ + struct wait_barrier *wb; + + wb = kmalloc(sizeof(*wb), GFP_KERNEL); + if (unlikely(!wb)) + return -ENOMEM; + + GEM_BUG_ON(i915_active_is_idle(ref)); + if (!i915_sw_fence_await(fence)) { + kfree(wb); + return -EINVAL; + } + + wb->base.flags = 0; + wb->base.func = barrier_wake; + wb->base.private = fence; + wb->ref = ref; + + add_wait_queue(__var_waitqueue(ref), &wb->base); + return 0; +} + +static int await_active(struct i915_active *ref, + unsigned int flags, + int (*fn)(void *arg, struct dma_fence *fence), + void *arg, struct i915_sw_fence *barrier) { int err = 0; - if (rcu_access_pointer(ref->excl.fence)) { - struct dma_fence *fence; + if (!i915_active_acquire_if_busy(ref)) + return 0; - rcu_read_lock(); - fence = dma_fence_get_rcu_safe(&ref->excl.fence); - rcu_read_unlock(); - if (fence) { - err = i915_request_await_dma_fence(rq, fence); - dma_fence_put(fence); + if (flags & I915_ACTIVE_AWAIT_EXCL && + rcu_access_pointer(ref->excl.fence)) { + err = __await_active(&ref->excl, fn, arg); + if (err) + goto out; + } + + if (flags & I915_ACTIVE_AWAIT_ACTIVE) { + struct active_node *it, *n; + + rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { + err = __await_active(&it->base, fn, arg); + if (err) + goto out; } } - /* In the future we may choose to await on all fences */ + if (flags & I915_ACTIVE_AWAIT_BARRIER) { + err = flush_lazy_signals(ref); + if (err) + goto out; + + err = __await_barrier(ref, barrier); + if (err) + goto out; + } +out: + i915_active_release(ref); return err; } +static int rq_await_fence(void *arg, struct dma_fence *fence) +{ + return i915_request_await_dma_fence(arg, fence); +} + +int i915_request_await_active(struct i915_request *rq, + struct i915_active *ref, + unsigned int flags) +{ + return await_active(ref, flags, rq_await_fence, rq, &rq->submit); +} + +static int sw_await_fence(void *arg, struct dma_fence *fence) +{ + return i915_sw_fence_await_dma_fence(arg, fence, 0, + GFP_NOWAIT | __GFP_NOWARN); +} + +int i915_sw_fence_await_active(struct i915_sw_fence *fence, + struct i915_active *ref, + unsigned int flags) +{ + return await_active(ref, flags, sw_await_fence, fence, fence); +} + #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) void i915_active_fini(struct i915_active *ref) { @@ -735,7 +874,7 @@ void i915_active_acquire_barrier(struct i915_active *ref) GEM_BUG_ON(!intel_engine_pm_is_awake(engine)); llist_add(barrier_to_ll(node), &engine->barrier_tasks); - intel_engine_pm_put(engine); + intel_engine_pm_put_delay(engine, 1); } } @@ -823,7 +962,6 @@ __i915_active_fence_set(struct i915_active_fence *active, __list_del_entry(&active->cb.node); spin_unlock(prev->lock); /* serialise with prev->cb_list */ } - GEM_BUG_ON(rcu_access_pointer(active->fence) != fence); list_add_tail(&active->cb.node, &fence->cb_list); spin_unlock_irqrestore(fence->lock, flags); @@ -855,6 +993,59 @@ void i915_active_noop(struct dma_fence *fence, struct dma_fence_cb *cb) active_fence_cb(fence, cb); } +struct auto_active { + struct i915_active base; + struct kref ref; +}; + +struct i915_active *i915_active_get(struct i915_active *ref) +{ + struct auto_active *aa = container_of(ref, typeof(*aa), base); + + kref_get(&aa->ref); + return &aa->base; +} + +static void auto_release(struct kref *ref) +{ + struct auto_active *aa = container_of(ref, typeof(*aa), ref); + + i915_active_fini(&aa->base); + kfree(aa); +} + +void i915_active_put(struct i915_active *ref) +{ + struct auto_active *aa = container_of(ref, typeof(*aa), base); + + kref_put(&aa->ref, auto_release); +} + +static int auto_active(struct i915_active *ref) +{ + i915_active_get(ref); + return 0; +} + +static void auto_retire(struct i915_active *ref) +{ + i915_active_put(ref); +} + +struct i915_active *i915_active_create(void) +{ + struct auto_active *aa; + + aa = kmalloc(sizeof(*aa), GFP_KERNEL); + if (!aa) + return NULL; + + kref_init(&aa->ref); + i915_active_init(&aa->base, auto_active, auto_retire); + + return &aa->base; +} + #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/i915_active.c" #endif diff --git a/drivers/gpu/drm/i915/i915_active.h b/drivers/gpu/drm/i915/i915_active.h index 973ff0447c6c..cf4058150966 100644 --- a/drivers/gpu/drm/i915/i915_active.h +++ b/drivers/gpu/drm/i915/i915_active.h @@ -181,9 +181,21 @@ static inline bool i915_active_has_exclusive(struct i915_active *ref) return rcu_access_pointer(ref->excl.fence); } -int i915_active_wait(struct i915_active *ref); +int __i915_active_wait(struct i915_active *ref, int state); +static inline int i915_active_wait(struct i915_active *ref) +{ + return __i915_active_wait(ref, TASK_INTERRUPTIBLE); +} -int i915_request_await_active(struct i915_request *rq, struct i915_active *ref); +int i915_sw_fence_await_active(struct i915_sw_fence *fence, + struct i915_active *ref, + unsigned int flags); +int i915_request_await_active(struct i915_request *rq, + struct i915_active *ref, + unsigned int flags); +#define I915_ACTIVE_AWAIT_EXCL BIT(0) +#define I915_ACTIVE_AWAIT_ACTIVE BIT(1) +#define I915_ACTIVE_AWAIT_BARRIER BIT(2) int i915_active_acquire(struct i915_active *ref); bool i915_active_acquire_if_busy(struct i915_active *ref); @@ -215,4 +227,8 @@ void i915_request_add_active_barriers(struct i915_request *rq); void i915_active_print(struct i915_active *ref, struct drm_printer *m); void i915_active_unlock_wait(struct i915_active *ref); +struct i915_active *i915_active_create(void); +struct i915_active *i915_active_get(struct i915_active *ref); +void i915_active_put(struct i915_active *ref); + #endif /* _I915_ACTIVE_H_ */ diff --git a/drivers/gpu/drm/i915/i915_buddy.c b/drivers/gpu/drm/i915/i915_buddy.c index 66883af64ca1..20babbdb297d 100644 --- a/drivers/gpu/drm/i915/i915_buddy.c +++ b/drivers/gpu/drm/i915/i915_buddy.c @@ -312,7 +312,8 @@ i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order) return block; out_free: - __i915_buddy_free(mm, block); + if (i != order) + __i915_buddy_free(mm, block); return ERR_PTR(err); } diff --git a/drivers/gpu/drm/i915/i915_config.c b/drivers/gpu/drm/i915/i915_config.c new file mode 100644 index 000000000000..b79b5f6d2cfa --- /dev/null +++ b/drivers/gpu/drm/i915/i915_config.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "i915_drv.h" + +unsigned long +i915_fence_context_timeout(const struct drm_i915_private *i915, u64 context) +{ + if (context && IS_ACTIVE(CONFIG_DRM_I915_FENCE_TIMEOUT)) + return msecs_to_jiffies_timeout(CONFIG_DRM_I915_FENCE_TIMEOUT); + + return 0; +} diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index e5eea915bd0d..c01837a16a4e 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -32,12 +32,13 @@ #include <drm/drm_debugfs.h> #include "gem/i915_gem_context.h" +#include "gt/intel_gt_buffer_pool.h" +#include "gt/intel_gt_clock_utils.h" #include "gt/intel_gt_pm.h" #include "gt/intel_gt_requests.h" #include "gt/intel_reset.h" #include "gt/intel_rc6.h" #include "gt/intel_rps.h" -#include "gt/uc/intel_guc_submission.h" #include "i915_debugfs.h" #include "i915_debugfs_params.h" @@ -218,7 +219,7 @@ i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) struct file_stats { struct i915_address_space *vm; unsigned long count; - u64 total, unbound; + u64 total; u64 active, inactive; u64 closed; }; @@ -234,8 +235,6 @@ static int per_file_stats(int id, void *ptr, void *data) stats->count++; stats->total += obj->base.size; - if (!atomic_read(&obj->bind_count)) - stats->unbound += obj->base.size; spin_lock(&obj->vma.lock); if (!stats->vm) { @@ -285,13 +284,12 @@ static int per_file_stats(int id, void *ptr, void *data) #define print_file_stats(m, name, stats) do { \ if (stats.count) \ - seq_printf(m, "%s: %lu objects, %llu bytes (%llu active, %llu inactive, %llu unbound, %llu closed)\n", \ + seq_printf(m, "%s: %lu objects, %llu bytes (%llu active, %llu inactive, %llu closed)\n", \ name, \ stats.count, \ stats.total, \ stats.active, \ stats.inactive, \ - stats.unbound, \ stats.closed); \ } while (0) @@ -745,7 +743,7 @@ i915_error_state_write(struct file *filp, if (!error) return 0; - DRM_DEBUG_DRIVER("Resetting error state\n"); + drm_dbg(&error->i915->drm, "Resetting error state\n"); i915_reset_error_state(error->i915); return cnt; @@ -930,21 +928,30 @@ static int i915_frequency_info(struct seq_file *m, void *unused) seq_printf(m, "RPDECLIMIT: 0x%08x\n", rpdeclimit); seq_printf(m, "RPNSWREQ: %dMHz\n", reqf); seq_printf(m, "CAGF: %dMHz\n", cagf); - seq_printf(m, "RP CUR UP EI: %d (%dus)\n", - rpupei, GT_PM_INTERVAL_TO_US(dev_priv, rpupei)); - seq_printf(m, "RP CUR UP: %d (%dus)\n", - rpcurup, GT_PM_INTERVAL_TO_US(dev_priv, rpcurup)); - seq_printf(m, "RP PREV UP: %d (%dus)\n", - rpprevup, GT_PM_INTERVAL_TO_US(dev_priv, rpprevup)); + seq_printf(m, "RP CUR UP EI: %d (%dns)\n", + rpupei, + intel_gt_pm_interval_to_ns(&dev_priv->gt, rpupei)); + seq_printf(m, "RP CUR UP: %d (%dun)\n", + rpcurup, + intel_gt_pm_interval_to_ns(&dev_priv->gt, rpcurup)); + seq_printf(m, "RP PREV UP: %d (%dns)\n", + rpprevup, + intel_gt_pm_interval_to_ns(&dev_priv->gt, rpprevup)); seq_printf(m, "Up threshold: %d%%\n", rps->power.up_threshold); - seq_printf(m, "RP CUR DOWN EI: %d (%dus)\n", - rpdownei, GT_PM_INTERVAL_TO_US(dev_priv, rpdownei)); - seq_printf(m, "RP CUR DOWN: %d (%dus)\n", - rpcurdown, GT_PM_INTERVAL_TO_US(dev_priv, rpcurdown)); - seq_printf(m, "RP PREV DOWN: %d (%dus)\n", - rpprevdown, GT_PM_INTERVAL_TO_US(dev_priv, rpprevdown)); + seq_printf(m, "RP CUR DOWN EI: %d (%dns)\n", + rpdownei, + intel_gt_pm_interval_to_ns(&dev_priv->gt, + rpdownei)); + seq_printf(m, "RP CUR DOWN: %d (%dns)\n", + rpcurdown, + intel_gt_pm_interval_to_ns(&dev_priv->gt, + rpcurdown)); + seq_printf(m, "RP PREV DOWN: %d (%dns)\n", + rpprevdown, + intel_gt_pm_interval_to_ns(&dev_priv->gt, + rpprevdown)); seq_printf(m, "Down threshold: %d%%\n", rps->power.down_threshold); @@ -996,220 +1003,6 @@ static int i915_frequency_info(struct seq_file *m, void *unused) return ret; } -static int ilk_drpc_info(struct seq_file *m) -{ - struct drm_i915_private *i915 = node_to_i915(m->private); - struct intel_uncore *uncore = &i915->uncore; - u32 rgvmodectl, rstdbyctl; - u16 crstandvid; - - rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); - rstdbyctl = intel_uncore_read(uncore, RSTDBYCTL); - crstandvid = intel_uncore_read16(uncore, CRSTANDVID); - - seq_printf(m, "HD boost: %s\n", yesno(rgvmodectl & MEMMODE_BOOST_EN)); - seq_printf(m, "Boost freq: %d\n", - (rgvmodectl & MEMMODE_BOOST_FREQ_MASK) >> - MEMMODE_BOOST_FREQ_SHIFT); - seq_printf(m, "HW control enabled: %s\n", - yesno(rgvmodectl & MEMMODE_HWIDLE_EN)); - seq_printf(m, "SW control enabled: %s\n", - yesno(rgvmodectl & MEMMODE_SWMODE_EN)); - seq_printf(m, "Gated voltage change: %s\n", - yesno(rgvmodectl & MEMMODE_RCLK_GATE)); - seq_printf(m, "Starting frequency: P%d\n", - (rgvmodectl & MEMMODE_FSTART_MASK) >> MEMMODE_FSTART_SHIFT); - seq_printf(m, "Max P-state: P%d\n", - (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT); - seq_printf(m, "Min P-state: P%d\n", (rgvmodectl & MEMMODE_FMIN_MASK)); - seq_printf(m, "RS1 VID: %d\n", (crstandvid & 0x3f)); - seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f)); - seq_printf(m, "Render standby enabled: %s\n", - yesno(!(rstdbyctl & RCX_SW_EXIT))); - seq_puts(m, "Current RS state: "); - switch (rstdbyctl & RSX_STATUS_MASK) { - case RSX_STATUS_ON: - seq_puts(m, "on\n"); - break; - case RSX_STATUS_RC1: - seq_puts(m, "RC1\n"); - break; - case RSX_STATUS_RC1E: - seq_puts(m, "RC1E\n"); - break; - case RSX_STATUS_RS1: - seq_puts(m, "RS1\n"); - break; - case RSX_STATUS_RS2: - seq_puts(m, "RS2 (RC6)\n"); - break; - case RSX_STATUS_RS3: - seq_puts(m, "RC3 (RC6+)\n"); - break; - default: - seq_puts(m, "unknown\n"); - break; - } - - return 0; -} - -static int i915_forcewake_domains(struct seq_file *m, void *data) -{ - struct drm_i915_private *i915 = node_to_i915(m->private); - struct intel_uncore *uncore = &i915->uncore; - struct intel_uncore_forcewake_domain *fw_domain; - unsigned int tmp; - - seq_printf(m, "user.bypass_count = %u\n", - uncore->user_forcewake_count); - - for_each_fw_domain(fw_domain, uncore, tmp) - seq_printf(m, "%s.wake_count = %u\n", - intel_uncore_forcewake_domain_to_str(fw_domain->id), - READ_ONCE(fw_domain->wake_count)); - - return 0; -} - -static void print_rc6_res(struct seq_file *m, - const char *title, - const i915_reg_t reg) -{ - struct drm_i915_private *i915 = node_to_i915(m->private); - intel_wakeref_t wakeref; - - with_intel_runtime_pm(&i915->runtime_pm, wakeref) - seq_printf(m, "%s %u (%llu us)\n", title, - intel_uncore_read(&i915->uncore, reg), - intel_rc6_residency_us(&i915->gt.rc6, reg)); -} - -static int vlv_drpc_info(struct seq_file *m) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - u32 rcctl1, pw_status; - - pw_status = I915_READ(VLV_GTLC_PW_STATUS); - rcctl1 = I915_READ(GEN6_RC_CONTROL); - - seq_printf(m, "RC6 Enabled: %s\n", - yesno(rcctl1 & (GEN7_RC_CTL_TO_MODE | - GEN6_RC_CTL_EI_MODE(1)))); - seq_printf(m, "Render Power Well: %s\n", - (pw_status & VLV_GTLC_PW_RENDER_STATUS_MASK) ? "Up" : "Down"); - seq_printf(m, "Media Power Well: %s\n", - (pw_status & VLV_GTLC_PW_MEDIA_STATUS_MASK) ? "Up" : "Down"); - - print_rc6_res(m, "Render RC6 residency since boot:", VLV_GT_RENDER_RC6); - print_rc6_res(m, "Media RC6 residency since boot:", VLV_GT_MEDIA_RC6); - - return i915_forcewake_domains(m, NULL); -} - -static int gen6_drpc_info(struct seq_file *m) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - u32 gt_core_status, rcctl1, rc6vids = 0; - u32 gen9_powergate_enable = 0, gen9_powergate_status = 0; - - gt_core_status = I915_READ_FW(GEN6_GT_CORE_STATUS); - trace_i915_reg_rw(false, GEN6_GT_CORE_STATUS, gt_core_status, 4, true); - - rcctl1 = I915_READ(GEN6_RC_CONTROL); - if (INTEL_GEN(dev_priv) >= 9) { - gen9_powergate_enable = I915_READ(GEN9_PG_ENABLE); - gen9_powergate_status = I915_READ(GEN9_PWRGT_DOMAIN_STATUS); - } - - if (INTEL_GEN(dev_priv) <= 7) - sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, - &rc6vids, NULL); - - seq_printf(m, "RC1e Enabled: %s\n", - yesno(rcctl1 & GEN6_RC_CTL_RC1e_ENABLE)); - seq_printf(m, "RC6 Enabled: %s\n", - yesno(rcctl1 & GEN6_RC_CTL_RC6_ENABLE)); - if (INTEL_GEN(dev_priv) >= 9) { - seq_printf(m, "Render Well Gating Enabled: %s\n", - yesno(gen9_powergate_enable & GEN9_RENDER_PG_ENABLE)); - seq_printf(m, "Media Well Gating Enabled: %s\n", - yesno(gen9_powergate_enable & GEN9_MEDIA_PG_ENABLE)); - } - seq_printf(m, "Deep RC6 Enabled: %s\n", - yesno(rcctl1 & GEN6_RC_CTL_RC6p_ENABLE)); - seq_printf(m, "Deepest RC6 Enabled: %s\n", - yesno(rcctl1 & GEN6_RC_CTL_RC6pp_ENABLE)); - seq_puts(m, "Current RC state: "); - switch (gt_core_status & GEN6_RCn_MASK) { - case GEN6_RC0: - if (gt_core_status & GEN6_CORE_CPD_STATE_MASK) - seq_puts(m, "Core Power Down\n"); - else - seq_puts(m, "on\n"); - break; - case GEN6_RC3: - seq_puts(m, "RC3\n"); - break; - case GEN6_RC6: - seq_puts(m, "RC6\n"); - break; - case GEN6_RC7: - seq_puts(m, "RC7\n"); - break; - default: - seq_puts(m, "Unknown\n"); - break; - } - - seq_printf(m, "Core Power Down: %s\n", - yesno(gt_core_status & GEN6_CORE_CPD_STATE_MASK)); - if (INTEL_GEN(dev_priv) >= 9) { - seq_printf(m, "Render Power Well: %s\n", - (gen9_powergate_status & - GEN9_PWRGT_RENDER_STATUS_MASK) ? "Up" : "Down"); - seq_printf(m, "Media Power Well: %s\n", - (gen9_powergate_status & - GEN9_PWRGT_MEDIA_STATUS_MASK) ? "Up" : "Down"); - } - - /* Not exactly sure what this is */ - print_rc6_res(m, "RC6 \"Locked to RPn\" residency since boot:", - GEN6_GT_GFX_RC6_LOCKED); - print_rc6_res(m, "RC6 residency since boot:", GEN6_GT_GFX_RC6); - print_rc6_res(m, "RC6+ residency since boot:", GEN6_GT_GFX_RC6p); - print_rc6_res(m, "RC6++ residency since boot:", GEN6_GT_GFX_RC6pp); - - if (INTEL_GEN(dev_priv) <= 7) { - seq_printf(m, "RC6 voltage: %dmV\n", - GEN6_DECODE_RC6_VID(((rc6vids >> 0) & 0xff))); - seq_printf(m, "RC6+ voltage: %dmV\n", - GEN6_DECODE_RC6_VID(((rc6vids >> 8) & 0xff))); - seq_printf(m, "RC6++ voltage: %dmV\n", - GEN6_DECODE_RC6_VID(((rc6vids >> 16) & 0xff))); - } - - return i915_forcewake_domains(m, NULL); -} - -static int i915_drpc_info(struct seq_file *m, void *unused) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - intel_wakeref_t wakeref; - int err = -ENODEV; - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - err = vlv_drpc_info(m); - else if (INTEL_GEN(dev_priv) >= 6) - err = gen6_drpc_info(m); - else - err = ilk_drpc_info(m); - } - - return err; -} - static int i915_ring_freq_table(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -1407,7 +1200,8 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) struct drm_i915_private *dev_priv = node_to_i915(m->private); struct intel_rps *rps = &dev_priv->gt.rps; - seq_printf(m, "RPS enabled? %d\n", rps->enabled); + seq_printf(m, "RPS enabled? %s\n", yesno(intel_rps_is_enabled(rps))); + seq_printf(m, "RPS active? %s\n", yesno(intel_rps_is_active(rps))); seq_printf(m, "GPU busy? %s\n", yesno(dev_priv->gt.awake)); seq_printf(m, "Boosts outstanding? %d\n", atomic_read(&rps->num_waiters)); @@ -1427,7 +1221,7 @@ static int i915_rps_boost_info(struct seq_file *m, void *data) seq_printf(m, "Wait boosts: %d\n", atomic_read(&rps->boosts)); - if (INTEL_GEN(dev_priv) >= 6 && rps->enabled && dev_priv->gt.awake) { + if (INTEL_GEN(dev_priv) >= 6 && intel_rps_is_active(rps)) { u32 rpup, rpupei; u32 rpdown, rpdownei; @@ -1465,285 +1259,6 @@ static int i915_llc(struct seq_file *m, void *data) return 0; } -static int i915_huc_load_status_info(struct seq_file *m, void *data) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - intel_wakeref_t wakeref; - struct drm_printer p; - - if (!HAS_GT_UC(dev_priv)) - return -ENODEV; - - p = drm_seq_file_printer(m); - intel_uc_fw_dump(&dev_priv->gt.uc.huc.fw, &p); - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) - seq_printf(m, "\nHuC status 0x%08x:\n", I915_READ(HUC_STATUS2)); - - return 0; -} - -static int i915_guc_load_status_info(struct seq_file *m, void *data) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - intel_wakeref_t wakeref; - struct drm_printer p; - - if (!HAS_GT_UC(dev_priv)) - return -ENODEV; - - p = drm_seq_file_printer(m); - intel_uc_fw_dump(&dev_priv->gt.uc.guc.fw, &p); - - with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) { - u32 tmp = I915_READ(GUC_STATUS); - u32 i; - - seq_printf(m, "\nGuC status 0x%08x:\n", tmp); - seq_printf(m, "\tBootrom status = 0x%x\n", - (tmp & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); - seq_printf(m, "\tuKernel status = 0x%x\n", - (tmp & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); - seq_printf(m, "\tMIA Core status = 0x%x\n", - (tmp & GS_MIA_MASK) >> GS_MIA_SHIFT); - seq_puts(m, "\nScratch registers:\n"); - for (i = 0; i < 16; i++) { - seq_printf(m, "\t%2d: \t0x%x\n", - i, I915_READ(SOFT_SCRATCH(i))); - } - } - - return 0; -} - -static const char * -stringify_guc_log_type(enum guc_log_buffer_type type) -{ - switch (type) { - case GUC_ISR_LOG_BUFFER: - return "ISR"; - case GUC_DPC_LOG_BUFFER: - return "DPC"; - case GUC_CRASH_DUMP_LOG_BUFFER: - return "CRASH"; - default: - MISSING_CASE(type); - } - - return ""; -} - -static void i915_guc_log_info(struct seq_file *m, - struct drm_i915_private *dev_priv) -{ - struct intel_guc_log *log = &dev_priv->gt.uc.guc.log; - enum guc_log_buffer_type type; - - if (!intel_guc_log_relay_created(log)) { - seq_puts(m, "GuC log relay not created\n"); - return; - } - - seq_puts(m, "GuC logging stats:\n"); - - seq_printf(m, "\tRelay full count: %u\n", - log->relay.full_count); - - for (type = GUC_ISR_LOG_BUFFER; type < GUC_MAX_LOG_BUFFER; type++) { - seq_printf(m, "\t%s:\tflush count %10u, overflow count %10u\n", - stringify_guc_log_type(type), - log->stats[type].flush, - log->stats[type].sampled_overflow); - } -} - -static int i915_guc_info(struct seq_file *m, void *data) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - - if (!USES_GUC(dev_priv)) - return -ENODEV; - - i915_guc_log_info(m, dev_priv); - - /* Add more as required ... */ - - return 0; -} - -static int i915_guc_stage_pool(struct seq_file *m, void *data) -{ - struct drm_i915_private *dev_priv = node_to_i915(m->private); - const struct intel_guc *guc = &dev_priv->gt.uc.guc; - struct guc_stage_desc *desc = guc->stage_desc_pool_vaddr; - int index; - - if (!USES_GUC_SUBMISSION(dev_priv)) - return -ENODEV; - - for (index = 0; index < GUC_MAX_STAGE_DESCRIPTORS; index++, desc++) { - struct intel_engine_cs *engine; - - if (!(desc->attribute & GUC_STAGE_DESC_ATTR_ACTIVE)) - continue; - - seq_printf(m, "GuC stage descriptor %u:\n", index); - seq_printf(m, "\tIndex: %u\n", desc->stage_id); - seq_printf(m, "\tAttribute: 0x%x\n", desc->attribute); - seq_printf(m, "\tPriority: %d\n", desc->priority); - seq_printf(m, "\tDoorbell id: %d\n", desc->db_id); - seq_printf(m, "\tEngines used: 0x%x\n", - desc->engines_used); - seq_printf(m, "\tDoorbell trigger phy: 0x%llx, cpu: 0x%llx, uK: 0x%x\n", - desc->db_trigger_phy, - desc->db_trigger_cpu, - desc->db_trigger_uk); - seq_printf(m, "\tProcess descriptor: 0x%x\n", - desc->process_desc); - seq_printf(m, "\tWorkqueue address: 0x%x, size: 0x%x\n", - desc->wq_addr, desc->wq_size); - seq_putc(m, '\n'); - - for_each_uabi_engine(engine, dev_priv) { - u32 guc_engine_id = engine->guc_id; - struct guc_execlist_context *lrc = - &desc->lrc[guc_engine_id]; - - seq_printf(m, "\t%s LRC:\n", engine->name); - seq_printf(m, "\t\tContext desc: 0x%x\n", - lrc->context_desc); - seq_printf(m, "\t\tContext id: 0x%x\n", lrc->context_id); - seq_printf(m, "\t\tLRCA: 0x%x\n", lrc->ring_lrca); - seq_printf(m, "\t\tRing begin: 0x%x\n", lrc->ring_begin); - seq_printf(m, "\t\tRing end: 0x%x\n", lrc->ring_end); - seq_putc(m, '\n'); - } - } - - return 0; -} - -static int i915_guc_log_dump(struct seq_file *m, void *data) -{ - struct drm_info_node *node = m->private; - struct drm_i915_private *dev_priv = node_to_i915(node); - bool dump_load_err = !!node->info_ent->data; - struct drm_i915_gem_object *obj = NULL; - u32 *log; - int i = 0; - - if (!HAS_GT_UC(dev_priv)) - return -ENODEV; - - if (dump_load_err) - obj = dev_priv->gt.uc.load_err_log; - else if (dev_priv->gt.uc.guc.log.vma) - obj = dev_priv->gt.uc.guc.log.vma->obj; - - if (!obj) - return 0; - - log = i915_gem_object_pin_map(obj, I915_MAP_WC); - if (IS_ERR(log)) { - DRM_DEBUG("Failed to pin object\n"); - seq_puts(m, "(log data unaccessible)\n"); - return PTR_ERR(log); - } - - for (i = 0; i < obj->base.size / sizeof(u32); i += 4) - seq_printf(m, "0x%08x 0x%08x 0x%08x 0x%08x\n", - *(log + i), *(log + i + 1), - *(log + i + 2), *(log + i + 3)); - - seq_putc(m, '\n'); - - i915_gem_object_unpin_map(obj); - - return 0; -} - -static int i915_guc_log_level_get(void *data, u64 *val) -{ - struct drm_i915_private *dev_priv = data; - - if (!USES_GUC(dev_priv)) - return -ENODEV; - - *val = intel_guc_log_get_level(&dev_priv->gt.uc.guc.log); - - return 0; -} - -static int i915_guc_log_level_set(void *data, u64 val) -{ - struct drm_i915_private *dev_priv = data; - - if (!USES_GUC(dev_priv)) - return -ENODEV; - - return intel_guc_log_set_level(&dev_priv->gt.uc.guc.log, val); -} - -DEFINE_SIMPLE_ATTRIBUTE(i915_guc_log_level_fops, - i915_guc_log_level_get, i915_guc_log_level_set, - "%lld\n"); - -static int i915_guc_log_relay_open(struct inode *inode, struct file *file) -{ - struct drm_i915_private *i915 = inode->i_private; - struct intel_guc *guc = &i915->gt.uc.guc; - struct intel_guc_log *log = &guc->log; - - if (!intel_guc_is_ready(guc)) - return -ENODEV; - - file->private_data = log; - - return intel_guc_log_relay_open(log); -} - -static ssize_t -i915_guc_log_relay_write(struct file *filp, - const char __user *ubuf, - size_t cnt, - loff_t *ppos) -{ - struct intel_guc_log *log = filp->private_data; - int val; - int ret; - - ret = kstrtoint_from_user(ubuf, cnt, 0, &val); - if (ret < 0) - return ret; - - /* - * Enable and start the guc log relay on value of 1. - * Flush log relay for any other value. - */ - if (val == 1) - ret = intel_guc_log_relay_start(log); - else - intel_guc_log_relay_flush(log); - - return ret ?: cnt; -} - -static int i915_guc_log_relay_release(struct inode *inode, struct file *file) -{ - struct drm_i915_private *i915 = inode->i_private; - struct intel_guc *guc = &i915->gt.uc.guc; - - intel_guc_log_relay_close(&guc->log); - return 0; -} - -static const struct file_operations i915_guc_log_relay_fops = { - .owner = THIS_MODULE, - .open = i915_guc_log_relay_open, - .write = i915_guc_log_relay_write, - .release = i915_guc_log_relay_release, -}; - static int i915_runtime_pm_status(struct seq_file *m, void *unused) { struct drm_i915_private *dev_priv = node_to_i915(m->private); @@ -1970,6 +1485,9 @@ gt_drop_caches(struct intel_gt *gt, u64 val) if (val & DROP_RESET_ACTIVE && intel_gt_terminally_wedged(gt)) intel_gt_handle_error(gt, ALL_ENGINES, 0, NULL); + if (val & DROP_FREED) + intel_gt_flush_buffer_pool(gt); + return 0; } @@ -2352,17 +1870,9 @@ static const struct drm_info_list i915_debugfs_list[] = { {"i915_gem_objects", i915_gem_object_info, 0}, {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0}, {"i915_gem_interrupt", i915_interrupt_info, 0}, - {"i915_guc_info", i915_guc_info, 0}, - {"i915_guc_load_status", i915_guc_load_status_info, 0}, - {"i915_guc_log_dump", i915_guc_log_dump, 0}, - {"i915_guc_load_err_log_dump", i915_guc_log_dump, 0, (void *)1}, - {"i915_guc_stage_pool", i915_guc_stage_pool, 0}, - {"i915_huc_load_status", i915_huc_load_status_info, 0}, {"i915_frequency_info", i915_frequency_info, 0}, - {"i915_drpc_info", i915_drpc_info, 0}, {"i915_ring_freq_table", i915_ring_freq_table, 0}, {"i915_context_status", i915_context_status, 0}, - {"i915_forcewake_domains", i915_forcewake_domains, 0}, {"i915_swizzle_info", i915_swizzle_info, 0}, {"i915_llc", i915_llc, 0}, {"i915_runtime_pm_status", i915_runtime_pm_status, 0}, @@ -2387,8 +1897,6 @@ static const struct i915_debugfs_files { {"i915_error_state", &i915_error_state_fops}, {"i915_gpu_info", &i915_gpu_info_fops}, #endif - {"i915_guc_log_level", &i915_guc_log_level_fops}, - {"i915_guc_log_relay", &i915_guc_log_relay_fops}, }; int i915_debugfs_register(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 4dd8294b68e1..c08b165a9cb4 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -44,7 +44,6 @@ #include <drm/drm_ioctl.h> #include <drm/drm_irq.h> #include <drm/drm_probe_helper.h> -#include <drm/i915_drm.h> #include "display/intel_acpi.h" #include "display/intel_audio.h" @@ -58,6 +57,7 @@ #include "display/intel_hotplug.h" #include "display/intel_overlay.h" #include "display/intel_pipe_crc.h" +#include "display/intel_psr.h" #include "display/intel_sprite.h" #include "display/intel_vga.h" @@ -70,6 +70,7 @@ #include "i915_debugfs.h" #include "i915_drv.h" +#include "i915_ioc32.h" #include "i915_irq.h" #include "i915_memcpy.h" #include "i915_perf.h" @@ -79,6 +80,8 @@ #include "i915_sysfs.h" #include "i915_trace.h" #include "i915_vgpu.h" +#include "intel_dram.h" +#include "intel_gvt.h" #include "intel_memory_region.h" #include "intel_pm.h" #include "vlv_suspend.h" @@ -212,7 +215,8 @@ intel_teardown_mchbar(struct drm_i915_private *dev_priv) release_resource(&dev_priv->mch_res); } -static int i915_driver_modeset_probe(struct drm_i915_private *i915) +/* part #1: call before irq install */ +static int i915_driver_modeset_probe_noirq(struct drm_i915_private *i915) { int ret; @@ -223,28 +227,44 @@ static int i915_driver_modeset_probe(struct drm_i915_private *i915) ret = drm_vblank_init(&i915->drm, INTEL_NUM_PIPES(i915)); if (ret) - goto out; + return ret; } intel_bios_init(i915); ret = intel_vga_register(i915); if (ret) - goto out; + goto cleanup_bios; intel_power_domains_init_hw(i915, false); intel_csr_ucode_init(i915); - ret = intel_irq_install(i915); + ret = intel_modeset_init_noirq(i915); if (ret) - goto cleanup_csr; + goto cleanup_vga_client_pw_domain_csr; + + return 0; + +cleanup_vga_client_pw_domain_csr: + intel_csr_ucode_fini(i915); + intel_power_domains_driver_remove(i915); + intel_vga_unregister(i915); +cleanup_bios: + intel_bios_driver_remove(i915); + return ret; +} + +/* part #2: call after irq install */ +static int i915_driver_modeset_probe(struct drm_i915_private *i915) +{ + int ret; /* Important: The output setup functions called by modeset_init need * working irqs for e.g. gmbus and dp aux transfers. */ ret = intel_modeset_init(i915); if (ret) - goto cleanup_irq; + goto out; ret = i915_gem_init(i915); if (ret) @@ -264,6 +284,8 @@ static int i915_driver_modeset_probe(struct drm_i915_private *i915) intel_init_ipc(i915); + intel_psr_set_force_mode_changed(i915->psr.dp); + return 0; cleanup_gem: @@ -271,16 +293,10 @@ cleanup_gem: i915_gem_driver_remove(i915); i915_gem_driver_release(i915); cleanup_modeset: + /* FIXME */ intel_modeset_driver_remove(i915); intel_irq_uninstall(i915); intel_modeset_driver_remove_noirq(i915); - goto cleanup_csr; -cleanup_irq: - intel_irq_uninstall(i915); -cleanup_csr: - intel_csr_ucode_fini(i915); - intel_power_domains_driver_remove(i915); - intel_vga_unregister(i915); out: return ret; } @@ -294,13 +310,13 @@ static void i915_driver_modeset_remove(struct drm_i915_private *i915) /* part #2: call after irq uninstall */ static void i915_driver_modeset_remove_noirq(struct drm_i915_private *i915) { - intel_modeset_driver_remove_noirq(i915); + intel_csr_ucode_fini(i915); - intel_bios_driver_remove(i915); + intel_power_domains_driver_remove(i915); intel_vga_unregister(i915); - intel_csr_ucode_fini(i915); + intel_bios_driver_remove(i915); } static void intel_init_dpio(struct drm_i915_private *dev_priv) @@ -376,6 +392,7 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) pre |= IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0); pre |= IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST); pre |= IS_KBL_REVID(dev_priv, 0, KBL_REVID_A0); + pre |= IS_GLK_REVID(dev_priv, 0, GLK_REVID_A2); if (pre) { drm_err(&dev_priv->drm, "This is a pre-production stepping. " @@ -417,8 +434,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) mutex_init(&dev_priv->backlight_lock); mutex_init(&dev_priv->sb_lock); - pm_qos_add_request(&dev_priv->sb_qos, - PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE); + cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE); mutex_init(&dev_priv->av_mutex); mutex_init(&dev_priv->wm.wm_mutex); @@ -454,7 +470,6 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) intel_init_display_hooks(dev_priv); intel_init_clock_gating_hooks(dev_priv); intel_init_audio_hooks(dev_priv); - intel_display_crc_init(dev_priv); intel_detect_preproduction_hw(dev_priv); @@ -483,7 +498,7 @@ static void i915_driver_late_release(struct drm_i915_private *dev_priv) vlv_suspend_cleanup(dev_priv); i915_workqueues_cleanup(dev_priv); - pm_qos_remove_request(&dev_priv->sb_qos); + cpu_latency_qos_remove_request(&dev_priv->sb_qos); mutex_destroy(&dev_priv->sb_lock); } @@ -553,492 +568,60 @@ static void intel_sanitize_options(struct drm_i915_private *dev_priv) intel_gvt_sanitize_options(dev_priv); } -#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type - -static const char *intel_dram_type_str(enum intel_dram_type type) -{ - static const char * const str[] = { - DRAM_TYPE_STR(UNKNOWN), - DRAM_TYPE_STR(DDR3), - DRAM_TYPE_STR(DDR4), - DRAM_TYPE_STR(LPDDR3), - DRAM_TYPE_STR(LPDDR4), - }; - - if (type >= ARRAY_SIZE(str)) - type = INTEL_DRAM_UNKNOWN; - - return str[type]; -} - -#undef DRAM_TYPE_STR - -static int intel_dimm_num_devices(const struct dram_dimm_info *dimm) -{ - return dimm->ranks * 64 / (dimm->width ?: 1); -} - -/* Returns total GB for the whole DIMM */ -static int skl_get_dimm_size(u16 val) -{ - return val & SKL_DRAM_SIZE_MASK; -} - -static int skl_get_dimm_width(u16 val) -{ - if (skl_get_dimm_size(val) == 0) - return 0; - - switch (val & SKL_DRAM_WIDTH_MASK) { - case SKL_DRAM_WIDTH_X8: - case SKL_DRAM_WIDTH_X16: - case SKL_DRAM_WIDTH_X32: - val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT; - return 8 << val; - default: - MISSING_CASE(val); - return 0; - } -} - -static int skl_get_dimm_ranks(u16 val) -{ - if (skl_get_dimm_size(val) == 0) - return 0; - - val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT; - - return val + 1; -} - -/* Returns total GB for the whole DIMM */ -static int cnl_get_dimm_size(u16 val) -{ - return (val & CNL_DRAM_SIZE_MASK) / 2; -} - -static int cnl_get_dimm_width(u16 val) -{ - if (cnl_get_dimm_size(val) == 0) - return 0; - - switch (val & CNL_DRAM_WIDTH_MASK) { - case CNL_DRAM_WIDTH_X8: - case CNL_DRAM_WIDTH_X16: - case CNL_DRAM_WIDTH_X32: - val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT; - return 8 << val; - default: - MISSING_CASE(val); - return 0; - } -} - -static int cnl_get_dimm_ranks(u16 val) -{ - if (cnl_get_dimm_size(val) == 0) - return 0; - - val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT; - - return val + 1; -} - -static bool -skl_is_16gb_dimm(const struct dram_dimm_info *dimm) -{ - /* Convert total GB to Gb per DRAM device */ - return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16; -} - -static void -skl_dram_get_dimm_info(struct drm_i915_private *dev_priv, - struct dram_dimm_info *dimm, - int channel, char dimm_name, u16 val) -{ - if (INTEL_GEN(dev_priv) >= 10) { - dimm->size = cnl_get_dimm_size(val); - dimm->width = cnl_get_dimm_width(val); - dimm->ranks = cnl_get_dimm_ranks(val); - } else { - dimm->size = skl_get_dimm_size(val); - dimm->width = skl_get_dimm_width(val); - dimm->ranks = skl_get_dimm_ranks(val); - } - - drm_dbg_kms(&dev_priv->drm, - "CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n", - channel, dimm_name, dimm->size, dimm->width, dimm->ranks, - yesno(skl_is_16gb_dimm(dimm))); -} - -static int -skl_dram_get_channel_info(struct drm_i915_private *dev_priv, - struct dram_channel_info *ch, - int channel, u32 val) -{ - skl_dram_get_dimm_info(dev_priv, &ch->dimm_l, - channel, 'L', val & 0xffff); - skl_dram_get_dimm_info(dev_priv, &ch->dimm_s, - channel, 'S', val >> 16); - - if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) { - drm_dbg_kms(&dev_priv->drm, "CH%u not populated\n", channel); - return -EINVAL; - } - - if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2) - ch->ranks = 2; - else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1) - ch->ranks = 2; - else - ch->ranks = 1; - - ch->is_16gb_dimm = - skl_is_16gb_dimm(&ch->dimm_l) || - skl_is_16gb_dimm(&ch->dimm_s); - - drm_dbg_kms(&dev_priv->drm, "CH%u ranks: %u, 16Gb DIMMs: %s\n", - channel, ch->ranks, yesno(ch->is_16gb_dimm)); - - return 0; -} - -static bool -intel_is_dram_symmetric(const struct dram_channel_info *ch0, - const struct dram_channel_info *ch1) -{ - return !memcmp(ch0, ch1, sizeof(*ch0)) && - (ch0->dimm_s.size == 0 || - !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l))); -} - -static int -skl_dram_get_channels_info(struct drm_i915_private *dev_priv) +/** + * i915_set_dma_info - set all relevant PCI dma info as configured for the + * platform + * @i915: valid i915 instance + * + * Set the dma max segment size, device and coherent masks. The dma mask set + * needs to occur before i915_ggtt_probe_hw. + * + * A couple of platforms have special needs. Address them as well. + * + */ +static int i915_set_dma_info(struct drm_i915_private *i915) { - struct dram_info *dram_info = &dev_priv->dram_info; - struct dram_channel_info ch0 = {}, ch1 = {}; - u32 val; + struct pci_dev *pdev = i915->drm.pdev; + unsigned int mask_size = INTEL_INFO(i915)->dma_mask_size; int ret; - val = I915_READ(SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN); - ret = skl_dram_get_channel_info(dev_priv, &ch0, 0, val); - if (ret == 0) - dram_info->num_channels++; - - val = I915_READ(SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN); - ret = skl_dram_get_channel_info(dev_priv, &ch1, 1, val); - if (ret == 0) - dram_info->num_channels++; - - if (dram_info->num_channels == 0) { - drm_info(&dev_priv->drm, - "Number of memory channels is zero\n"); - return -EINVAL; - } + GEM_BUG_ON(!mask_size); /* - * If any of the channel is single rank channel, worst case output - * will be same as if single rank memory, so consider single rank - * memory. + * We don't have a max segment size, so set it to the max so sg's + * debugging layer doesn't complain */ - if (ch0.ranks == 1 || ch1.ranks == 1) - dram_info->ranks = 1; - else - dram_info->ranks = max(ch0.ranks, ch1.ranks); - - if (dram_info->ranks == 0) { - drm_info(&dev_priv->drm, - "couldn't get memory rank information\n"); - return -EINVAL; - } - - dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm; - - dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1); - - drm_dbg_kms(&dev_priv->drm, "Memory configuration is symmetric? %s\n", - yesno(dram_info->symmetric_memory)); - return 0; -} - -static enum intel_dram_type -skl_get_dram_type(struct drm_i915_private *dev_priv) -{ - u32 val; - - val = I915_READ(SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN); - - switch (val & SKL_DRAM_DDR_TYPE_MASK) { - case SKL_DRAM_DDR_TYPE_DDR3: - return INTEL_DRAM_DDR3; - case SKL_DRAM_DDR_TYPE_DDR4: - return INTEL_DRAM_DDR4; - case SKL_DRAM_DDR_TYPE_LPDDR3: - return INTEL_DRAM_LPDDR3; - case SKL_DRAM_DDR_TYPE_LPDDR4: - return INTEL_DRAM_LPDDR4; - default: - MISSING_CASE(val); - return INTEL_DRAM_UNKNOWN; - } -} - -static int -skl_get_dram_info(struct drm_i915_private *dev_priv) -{ - struct dram_info *dram_info = &dev_priv->dram_info; - u32 mem_freq_khz, val; - int ret; - - dram_info->type = skl_get_dram_type(dev_priv); - drm_dbg_kms(&dev_priv->drm, "DRAM type: %s\n", - intel_dram_type_str(dram_info->type)); + dma_set_max_seg_size(&pdev->dev, UINT_MAX); - ret = skl_dram_get_channels_info(dev_priv); + ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(mask_size)); if (ret) - return ret; - - val = I915_READ(SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); - mem_freq_khz = DIV_ROUND_UP((val & SKL_REQ_DATA_MASK) * - SKL_MEMORY_FREQ_MULTIPLIER_HZ, 1000); - - dram_info->bandwidth_kbps = dram_info->num_channels * - mem_freq_khz * 8; - - if (dram_info->bandwidth_kbps == 0) { - drm_info(&dev_priv->drm, - "Couldn't get system memory bandwidth\n"); - return -EINVAL; - } - - dram_info->valid = true; - return 0; -} + goto mask_err; -/* Returns Gb per DRAM device */ -static int bxt_get_dimm_size(u32 val) -{ - switch (val & BXT_DRAM_SIZE_MASK) { - case BXT_DRAM_SIZE_4GBIT: - return 4; - case BXT_DRAM_SIZE_6GBIT: - return 6; - case BXT_DRAM_SIZE_8GBIT: - return 8; - case BXT_DRAM_SIZE_12GBIT: - return 12; - case BXT_DRAM_SIZE_16GBIT: - return 16; - default: - MISSING_CASE(val); - return 0; - } -} - -static int bxt_get_dimm_width(u32 val) -{ - if (!bxt_get_dimm_size(val)) - return 0; - - val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT; - - return 8 << val; -} - -static int bxt_get_dimm_ranks(u32 val) -{ - if (!bxt_get_dimm_size(val)) - return 0; - - switch (val & BXT_DRAM_RANK_MASK) { - case BXT_DRAM_RANK_SINGLE: - return 1; - case BXT_DRAM_RANK_DUAL: - return 2; - default: - MISSING_CASE(val); - return 0; - } -} - -static enum intel_dram_type bxt_get_dimm_type(u32 val) -{ - if (!bxt_get_dimm_size(val)) - return INTEL_DRAM_UNKNOWN; - - switch (val & BXT_DRAM_TYPE_MASK) { - case BXT_DRAM_TYPE_DDR3: - return INTEL_DRAM_DDR3; - case BXT_DRAM_TYPE_LPDDR3: - return INTEL_DRAM_LPDDR3; - case BXT_DRAM_TYPE_DDR4: - return INTEL_DRAM_DDR4; - case BXT_DRAM_TYPE_LPDDR4: - return INTEL_DRAM_LPDDR4; - default: - MISSING_CASE(val); - return INTEL_DRAM_UNKNOWN; - } -} - -static void bxt_get_dimm_info(struct dram_dimm_info *dimm, - u32 val) -{ - dimm->width = bxt_get_dimm_width(val); - dimm->ranks = bxt_get_dimm_ranks(val); - - /* - * Size in register is Gb per DRAM device. Convert to total - * GB to match the way we report this for non-LP platforms. - */ - dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8; -} - -static int -bxt_get_dram_info(struct drm_i915_private *dev_priv) -{ - struct dram_info *dram_info = &dev_priv->dram_info; - u32 dram_channels; - u32 mem_freq_khz, val; - u8 num_active_channels; - int i; - - val = I915_READ(BXT_P_CR_MC_BIOS_REQ_0_0_0); - mem_freq_khz = DIV_ROUND_UP((val & BXT_REQ_DATA_MASK) * - BXT_MEMORY_FREQ_MULTIPLIER_HZ, 1000); - - dram_channels = val & BXT_DRAM_CHANNEL_ACTIVE_MASK; - num_active_channels = hweight32(dram_channels); - - /* Each active bit represents 4-byte channel */ - dram_info->bandwidth_kbps = (mem_freq_khz * num_active_channels * 4); - - if (dram_info->bandwidth_kbps == 0) { - drm_info(&dev_priv->drm, - "Couldn't get system memory bandwidth\n"); - return -EINVAL; - } - - /* - * Now read each DUNIT8/9/10/11 to check the rank of each dimms. - */ - for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) { - struct dram_dimm_info dimm; - enum intel_dram_type type; - - val = I915_READ(BXT_D_CR_DRP0_DUNIT(i)); - if (val == 0xFFFFFFFF) - continue; - - dram_info->num_channels++; - - bxt_get_dimm_info(&dimm, val); - type = bxt_get_dimm_type(val); - - drm_WARN_ON(&dev_priv->drm, type != INTEL_DRAM_UNKNOWN && - dram_info->type != INTEL_DRAM_UNKNOWN && - dram_info->type != type); - - drm_dbg_kms(&dev_priv->drm, - "CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n", - i - BXT_D_CR_DRP0_DUNIT_START, - dimm.size, dimm.width, dimm.ranks, - intel_dram_type_str(type)); - - /* - * If any of the channel is single rank channel, - * worst case output will be same as if single rank - * memory, so consider single rank memory. - */ - if (dram_info->ranks == 0) - dram_info->ranks = dimm.ranks; - else if (dimm.ranks == 1) - dram_info->ranks = 1; - - if (type != INTEL_DRAM_UNKNOWN) - dram_info->type = type; - } - - if (dram_info->type == INTEL_DRAM_UNKNOWN || - dram_info->ranks == 0) { - drm_info(&dev_priv->drm, "couldn't get memory information\n"); - return -EINVAL; - } - - dram_info->valid = true; - return 0; -} - -static void -intel_get_dram_info(struct drm_i915_private *dev_priv) -{ - struct dram_info *dram_info = &dev_priv->dram_info; - int ret; + /* overlay on gen2 is broken and can't address above 1G */ + if (IS_GEN(i915, 2)) + mask_size = 30; /* - * Assume 16Gb DIMMs are present until proven otherwise. - * This is only used for the level 0 watermark latency - * w/a which does not apply to bxt/glk. + * 965GM sometimes incorrectly writes to hardware status page (HWS) + * using 32bit addressing, overwriting memory if HWS is located + * above 4GB. + * + * The documentation also mentions an issue with undefined + * behaviour if any general state is accessed within a page above 4GB, + * which also needs to be handled carefully. */ - dram_info->is_16gb_dimm = !IS_GEN9_LP(dev_priv); - - if (INTEL_GEN(dev_priv) < 9 || !HAS_DISPLAY(dev_priv)) - return; + if (IS_I965G(i915) || IS_I965GM(i915)) + mask_size = 32; - if (IS_GEN9_LP(dev_priv)) - ret = bxt_get_dram_info(dev_priv); - else - ret = skl_get_dram_info(dev_priv); + ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(mask_size)); if (ret) - return; - - drm_dbg_kms(&dev_priv->drm, "DRAM bandwidth: %u kBps, channels: %u\n", - dram_info->bandwidth_kbps, - dram_info->num_channels); - - drm_dbg_kms(&dev_priv->drm, "DRAM ranks: %u, 16Gb DIMMs: %s\n", - dram_info->ranks, yesno(dram_info->is_16gb_dimm)); -} - -static u32 gen9_edram_size_mb(struct drm_i915_private *dev_priv, u32 cap) -{ - static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; - static const u8 sets[4] = { 1, 1, 2, 2 }; - - return EDRAM_NUM_BANKS(cap) * - ways[EDRAM_WAYS_IDX(cap)] * - sets[EDRAM_SETS_IDX(cap)]; -} + goto mask_err; -static void edram_detect(struct drm_i915_private *dev_priv) -{ - u32 edram_cap = 0; - - if (!(IS_HASWELL(dev_priv) || - IS_BROADWELL(dev_priv) || - INTEL_GEN(dev_priv) >= 9)) - return; - - edram_cap = __raw_uncore_read32(&dev_priv->uncore, HSW_EDRAM_CAP); - - /* NB: We can't write IDICR yet because we don't have gt funcs set up */ - - if (!(edram_cap & EDRAM_ENABLED)) - return; - - /* - * The needed capability bits for size calculation are not there with - * pre gen9 so return 128MB always. - */ - if (INTEL_GEN(dev_priv) < 9) - dev_priv->edram_size_mb = 128; - else - dev_priv->edram_size_mb = - gen9_edram_size_mb(dev_priv, edram_cap); + return 0; - dev_info(dev_priv->drm.dev, - "Found %uMB of eDRAM\n", dev_priv->edram_size_mb); +mask_err: + drm_err(&i915->drm, "Can't set DMA mask/consistent mask (%d)\n", ret); + return ret; } /** @@ -1084,7 +667,11 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) intel_sanitize_options(dev_priv); /* needs to be done before ggtt probe */ - edram_detect(dev_priv); + intel_dram_edram_detect(dev_priv); + + ret = i915_set_dma_info(dev_priv); + if (ret) + return ret; i915_perf_init(dev_priv); @@ -1114,42 +701,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) pci_set_master(pdev); - /* - * We don't have a max segment size, so set it to the max so sg's - * debugging layer doesn't complain - */ - dma_set_max_seg_size(&pdev->dev, UINT_MAX); - - /* overlay on gen2 is broken and can't address above 1G */ - if (IS_GEN(dev_priv, 2)) { - ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(30)); - if (ret) { - drm_err(&dev_priv->drm, "failed to set DMA mask\n"); - - goto err_mem_regions; - } - } - - /* 965GM sometimes incorrectly writes to hardware status page (HWS) - * using 32bit addressing, overwriting memory if HWS is located - * above 4GB. - * - * The documentation also mentions an issue with undefined - * behaviour if any general state is accessed within a page above 4GB, - * which also needs to be handled carefully. - */ - if (IS_I965G(dev_priv) || IS_I965GM(dev_priv)) { - ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); - - if (ret) { - drm_err(&dev_priv->drm, "failed to set DMA mask\n"); - - goto err_mem_regions; - } - } - - pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, - PM_QOS_DEFAULT_VALUE); + cpu_latency_qos_add_request(&dev_priv->pm_qos, PM_QOS_DEFAULT_VALUE); intel_gt_init_workarounds(dev_priv); @@ -1186,7 +738,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) * Fill the dram structure to get the system raw bandwidth and * dram info. This will be used for memory latency calculation. */ - intel_get_dram_info(dev_priv); + intel_dram_detect(dev_priv); intel_bw_init_hw(dev_priv); @@ -1195,7 +747,7 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) err_msi: if (pdev->msi_enabled) pci_disable_msi(pdev); - pm_qos_remove_request(&dev_priv->pm_qos); + cpu_latency_qos_remove_request(&dev_priv->pm_qos); err_mem_regions: intel_memory_regions_driver_release(dev_priv); err_ggtt: @@ -1218,7 +770,7 @@ static void i915_driver_hw_remove(struct drm_i915_private *dev_priv) if (pdev->msi_enabled) pci_disable_msi(pdev); - pm_qos_remove_request(&dev_priv->pm_qos); + cpu_latency_qos_remove_request(&dev_priv->pm_qos); } /** @@ -1235,12 +787,7 @@ static void i915_driver_register(struct drm_i915_private *dev_priv) i915_gem_driver_register(dev_priv); i915_pmu_register(dev_priv); - /* - * Notify a valid surface after modesetting, - * when running inside a VM. - */ - if (intel_vgpu_active(dev_priv)) - I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY); + intel_vgpu_register(dev_priv); /* Reveal our presence to userspace */ if (drm_dev_register(dev, 0) == 0) { @@ -1370,8 +917,6 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent) return ERR_PTR(err); } - i915->drm.dev_private = i915; - i915->drm.pdev = pdev; pci_set_drvdata(pdev, i915); @@ -1449,7 +994,7 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) disable_rpm_wakeref_asserts(&i915->runtime_pm); - i915_detect_vgpu(i915); + intel_vgpu_detect(i915); ret = i915_driver_mmio_probe(i915); if (ret < 0) @@ -1459,10 +1004,18 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret < 0) goto out_cleanup_mmio; - ret = i915_driver_modeset_probe(i915); + ret = i915_driver_modeset_probe_noirq(i915); if (ret < 0) goto out_cleanup_hw; + ret = intel_irq_install(i915); + if (ret) + goto out_cleanup_modeset; + + ret = i915_driver_modeset_probe(i915); + if (ret < 0) + goto out_cleanup_irq; + i915_driver_register(i915); enable_rpm_wakeref_asserts(&i915->runtime_pm); @@ -1471,6 +1024,10 @@ int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; +out_cleanup_irq: + intel_irq_uninstall(i915); +out_cleanup_modeset: + i915_driver_modeset_remove_noirq(i915); out_cleanup_hw: i915_driver_hw_remove(i915); intel_memory_regions_driver_release(i915); @@ -1494,13 +1051,6 @@ void i915_driver_remove(struct drm_i915_private *i915) i915_driver_unregister(i915); - /* - * After unregistering the device to prevent any new users, cancel - * all in-flight requests so that we can quickly unbind the active - * resources. - */ - intel_gt_set_wedged(&i915->gt); - /* Flush any external code that still may be under the RCU lock */ synchronize_rcu(); @@ -1514,12 +1064,12 @@ void i915_driver_remove(struct drm_i915_private *i915) intel_irq_uninstall(i915); - i915_driver_modeset_remove_noirq(i915); + intel_modeset_driver_remove_noirq(i915); i915_reset_error_state(i915); i915_gem_driver_remove(i915); - intel_power_domains_driver_remove(i915); + i915_driver_modeset_remove_noirq(i915); i915_driver_hw_remove(i915); @@ -1765,7 +1315,6 @@ static int i915_drm_resume(struct drm_device *dev) drm_err(&dev_priv->drm, "failed to re-enable GGTT\n"); i915_ggtt_resume(&dev_priv->ggtt); - i915_gem_restore_fences(&dev_priv->ggtt); intel_csr_ucode_resume(dev_priv); @@ -2083,8 +1632,6 @@ static int intel_runtime_suspend(struct device *kdev) intel_gt_runtime_resume(&dev_priv->gt); - i915_gem_restore_fences(&dev_priv->ggtt); - enable_rpm_wakeref_asserts(rpm); return ret; @@ -2164,7 +1711,6 @@ static int intel_runtime_resume(struct device *kdev) * we can do is to hope that things will still work (and disable RPM). */ intel_gt_runtime_resume(&dev_priv->gt); - i915_gem_restore_fences(&dev_priv->ggtt); /* * On VLV/CHV display interrupts are part of the display @@ -2230,12 +1776,12 @@ const struct dev_pm_ops i915_pm_ops = { static const struct file_operations i915_driver_fops = { .owner = THIS_MODULE, .open = drm_open, - .release = drm_release, + .release = drm_release_noglobal, .unlocked_ioctl = drm_ioctl, .mmap = i915_gem_mmap, .poll = drm_poll, .read = drm_read, - .compat_ioctl = i915_compat_ioctl, + .compat_ioctl = i915_ioc32_compat_ioctl, .llseek = noop_llseek, }; @@ -2327,9 +1873,6 @@ static struct drm_driver driver = { .gem_prime_export = i915_gem_prime_export, .gem_prime_import = i915_gem_prime_import, - .get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos, - .get_scanout_position = i915_get_crtc_scanoutpos, - .dumb_create = i915_gem_dumb_create, .dumb_map_offset = i915_gem_dumb_mmap_offset, diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 9928d00ea0b1..ad373b57699a 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -59,7 +59,6 @@ #include <drm/drm_connector.h> #include <drm/i915_mei_hdcp_interface.h> -#include "i915_fixed.h" #include "i915_params.h" #include "i915_reg.h" #include "i915_utils.h" @@ -93,7 +92,6 @@ #include "intel_wopcm.h" #include "i915_gem.h" -#include "i915_gem_fence_reg.h" #include "i915_gem_gtt.h" #include "i915_gpu_error.h" #include "i915_perf_types.h" @@ -105,18 +103,23 @@ #include "intel_region_lmem.h" -#include "intel_gvt.h" - /* General customization: */ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20200114" -#define DRIVER_TIMESTAMP 1579001978 +#define DRIVER_DATE "20200430" +#define DRIVER_TIMESTAMP 1588234401 struct drm_i915_gem_object; +/* + * The code assumes that the hpd_pins below have consecutive values and + * starting with HPD_PORT_A, the HPD pin associated with any port can be + * retrieved by adding the corresponding port (or phy) enum value to + * HPD_PORT_A in most cases. For example: + * HPD_PORT_C = HPD_PORT_A + PHY_C - PHY_A + */ enum hpd_pin { HPD_NONE = 0, HPD_TV = HPD_NONE, /* TV is known to be unreliable */ @@ -145,6 +148,8 @@ enum hpd_pin { struct i915_hotplug { struct delayed_work hotplug_work; + const u32 *hpd, *pch_hpd; + struct { unsigned long last_jiffies; int count; @@ -413,6 +418,7 @@ struct intel_fbc { struct { const struct drm_format_info *format; unsigned int stride; + u64 modifier; } fb; u16 gen9_wa_cfb_stride; s8 fence_id; @@ -505,7 +511,8 @@ struct i915_psr { bool dc3co_enabled; u32 dc3co_exit_delay; struct delayed_work dc3co_work; - bool initially_probed; + bool force_mode_changed; + struct drm_dp_vsc_sdp vsc; }; #define QUIRK_LVDS_SSC_DISABLE (1<<1) @@ -536,7 +543,6 @@ struct i915_suspend_saved_registers { u32 saveSWF0[16]; u32 saveSWF1[16]; u32 saveSWF3[3]; - u64 saveFENCE[I915_MAX_NUM_FENCES]; u32 savePCH_PORT_HOTPLUG; u16 saveGCDGMBUS; }; @@ -611,13 +617,14 @@ struct i915_gem_mm { #define I915_IDLE_ENGINES_TIMEOUT (200) /* in ms */ -#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ -#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ - -#define I915_ENGINE_DEAD_TIMEOUT (4 * HZ) /* Seqno, head and subunits dead */ -#define I915_SEQNO_DEAD_TIMEOUT (12 * HZ) /* Seqno dead with active head */ +unsigned long i915_fence_context_timeout(const struct drm_i915_private *i915, + u64 context); -#define I915_ENGINE_WEDGED_TIMEOUT (60 * HZ) /* Reset but no recovery? */ +static inline unsigned long +i915_fence_timeout(const struct drm_i915_private *i915) +{ + return i915_fence_context_timeout(i915, U64_MAX); +} /* Amount of SAGV/QGV points, BSpec precisely defines this */ #define I915_NUM_QGV_POINTS 8 @@ -732,14 +739,6 @@ enum intel_ddb_partitioning { INTEL_DDB_PART_5_6, /* IVB+ */ }; -struct intel_wm_level { - bool enable; - u32 pri_val; - u32 spr_val; - u32 cur_val; - u32 fbc_val; -}; - struct ilk_wm_values { u32 wm_pipe[3]; u32 wm_lp[3]; @@ -798,56 +797,6 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, return false; } -struct skl_wm_level { - u16 min_ddb_alloc; - u16 plane_res_b; - u8 plane_res_l; - bool plane_en; - bool ignore_lines; -}; - -/* Stores plane specific WM parameters */ -struct skl_wm_params { - bool x_tiled, y_tiled; - bool rc_surface; - bool is_planar; - u32 width; - u8 cpp; - u32 plane_pixel_rate; - u32 y_min_scanlines; - u32 plane_bytes_per_line; - uint_fixed_16_16_t plane_blocks_per_line; - uint_fixed_16_16_t y_tile_minimum; - u32 linetime_us; - u32 dbuf_block_size; -}; - -enum intel_pipe_crc_source { - INTEL_PIPE_CRC_SOURCE_NONE, - INTEL_PIPE_CRC_SOURCE_PLANE1, - INTEL_PIPE_CRC_SOURCE_PLANE2, - INTEL_PIPE_CRC_SOURCE_PLANE3, - INTEL_PIPE_CRC_SOURCE_PLANE4, - INTEL_PIPE_CRC_SOURCE_PLANE5, - INTEL_PIPE_CRC_SOURCE_PLANE6, - INTEL_PIPE_CRC_SOURCE_PLANE7, - INTEL_PIPE_CRC_SOURCE_PIPE, - /* TV/DP on pre-gen5/vlv can't use the pipe source. */ - INTEL_PIPE_CRC_SOURCE_TV, - INTEL_PIPE_CRC_SOURCE_DP_B, - INTEL_PIPE_CRC_SOURCE_DP_C, - INTEL_PIPE_CRC_SOURCE_DP_D, - INTEL_PIPE_CRC_SOURCE_AUTO, - INTEL_PIPE_CRC_SOURCE_MAX, -}; - -#define INTEL_PIPE_CRC_ENTRIES_NR 128 -struct intel_pipe_crc { - spinlock_t lock; - int skipped; - enum intel_pipe_crc_source source; -}; - struct i915_frontbuffer_tracking { spinlock_t lock; @@ -865,13 +814,6 @@ struct i915_virtual_gpu { u32 caps; }; -/* used in computing the new watermarks state */ -struct intel_wm_config { - unsigned int num_pipes_active; - bool sprites_enabled; - bool sprites_scaled; -}; - struct intel_cdclk_config { unsigned int cdclk, vco, ref, bypass; u8 voltage_level; @@ -946,7 +888,6 @@ struct drm_i915_private { struct pci_dev *bridge_dev; - struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct rb_root uabi_engines; struct resource mch_res; @@ -1043,21 +984,24 @@ struct drm_i915_private { struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; -#ifdef CONFIG_DEBUG_FS - struct intel_pipe_crc pipe_crc[I915_MAX_PIPES]; -#endif + /** + * dpll and cdclk state is protected by connection_mutex + * dpll.lock serializes intel_{prepare,enable,disable}_shared_dpll. + * Must be global rather than per dpll, because on some platforms plls + * share registers. + */ + struct { + struct mutex lock; - /* dpll and cdclk state is protected by connection_mutex */ - int num_shared_dpll; - struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; - const struct intel_dpll_mgr *dpll_mgr; + int num_shared_dpll; + struct intel_shared_dpll shared_dplls[I915_NUM_PLLS]; + const struct intel_dpll_mgr *mgr; - /* - * dpll_lock serializes intel_{prepare,enable,disable}_shared_dpll. - * Must be global rather than per dpll, because on some platforms - * plls share registers. - */ - struct mutex dpll_lock; + struct { + int nssc; + int ssc; + } ref_clks; + } dpll; struct list_head global_obj_list; @@ -1078,8 +1022,6 @@ struct drm_i915_private { struct work_struct free_work; } atomic_helper; - u16 orig_clock; - bool mchbar_need_disable; struct intel_l3_parity l3_parity; @@ -1274,16 +1216,6 @@ struct drm_i915_private { */ }; -struct dram_dimm_info { - u8 size, width, ranks; -}; - -struct dram_channel_info { - struct dram_dimm_info dimm_l, dimm_s; - u8 ranks; - bool is_16gb_dimm; -}; - static inline struct drm_i915_private *to_i915(const struct drm_device *dev) { return container_of(dev, struct drm_i915_private, drm); @@ -1554,6 +1486,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define GLK_REVID_A0 0x0 #define GLK_REVID_A1 0x1 +#define GLK_REVID_A2 0x2 +#define GLK_REVID_B0 0x3 #define IS_GLK_REVID(dev_priv, since, until) \ (IS_GEMINILAKE(dev_priv) && IS_REVID(dev_priv, since, until)) @@ -1574,7 +1508,14 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define IS_ICL_REVID(p, since, until) \ (IS_ICELAKE(p) && IS_REVID(p, since, until)) +#define EHL_REVID_A0 0x0 + +#define IS_EHL_REVID(p, since, until) \ + (IS_ELKHARTLAKE(p) && IS_REVID(p, since, until)) + #define TGL_REVID_A0 0x0 +#define TGL_REVID_B0 0x1 +#define TGL_REVID_C0 0x2 #define IS_TGL_REVID(p, since, until) \ (IS_TIGERLAKE(p) && IS_REVID(p, since, until)) @@ -1672,7 +1613,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_DDI(dev_priv) (INTEL_INFO(dev_priv)->display.has_ddi) #define HAS_FPGA_DBG_UNCLAIMED(dev_priv) (INTEL_INFO(dev_priv)->has_fpga_dbg) #define HAS_PSR(dev_priv) (INTEL_INFO(dev_priv)->display.has_psr) -#define HAS_TRANSCODER_EDP(dev_priv) (INTEL_INFO(dev_priv)->trans_offsets[TRANSCODER_EDP] != 0) +#define HAS_TRANSCODER(dev_priv, trans) ((INTEL_INFO(dev_priv)->cpu_transcoder_mask & BIT(trans)) != 0) #define HAS_RC6(dev_priv) (INTEL_INFO(dev_priv)->has_rc6) #define HAS_RC6p(dev_priv) (INTEL_INFO(dev_priv)->has_rc6p) @@ -1692,10 +1633,6 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915, #define HAS_GT_UC(dev_priv) (INTEL_INFO(dev_priv)->has_gt_uc) -/* Having GuC is not the same as using GuC */ -#define USES_GUC(dev_priv) intel_uc_uses_guc(&(dev_priv)->gt.uc) -#define USES_GUC_SUBMISSION(dev_priv) intel_uc_uses_guc_submission(&(dev_priv)->gt.uc) - #define HAS_POOLED_EU(dev_priv) (INTEL_INFO(dev_priv)->has_pooled_eu) #define HAS_GLOBAL_MOCS_REGISTERS(dev_priv) (INTEL_INFO(dev_priv)->has_global_mocs) @@ -1741,11 +1678,6 @@ intel_ggtt_update_needs_vtd_wa(struct drm_i915_private *dev_priv) } /* i915_drv.c */ -#ifdef CONFIG_COMPAT -long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); -#else -#define i915_compat_ioctl NULL -#endif extern const struct dev_pm_ops i915_pm_ops; int i915_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent); @@ -1754,16 +1686,6 @@ void i915_driver_remove(struct drm_i915_private *i915); int i915_resume_switcheroo(struct drm_i915_private *i915); int i915_suspend_switcheroo(struct drm_i915_private *i915, pm_message_t state); -static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) -{ - return dev_priv->gvt; -} - -static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv) -{ - return dev_priv->vgpu.active; -} - int i915_getparam_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); @@ -1825,15 +1747,10 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, unsigned long flags); #define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0) #define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1) +#define I915_GEM_OBJECT_UNBIND_TEST BIT(2) void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); -static inline int __must_check -i915_mutex_lock_interruptible(struct drm_device *dev) -{ - return mutex_lock_interruptible(&dev->struct_mutex); -} - int i915_gem_dumb_create(struct drm_file *file_priv, struct drm_device *dev, struct drm_mode_create_dumb *args); @@ -2004,10 +1921,4 @@ i915_coherent_map_type(struct drm_i915_private *i915) return HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC; } -static inline bool intel_guc_submission_is_enabled(struct intel_guc *guc) -{ - return intel_guc_is_submission_supported(guc) && - intel_guc_is_ready(guc); -} - #endif diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index d92cf966fa3f..0cbcb9f54e7d 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -26,7 +26,6 @@ */ #include <drm/drm_vma_manager.h> -#include <drm/i915_drm.h> #include <linux/dma-fence-array.h> #include <linux/kthread.h> #include <linux/dma-resv.h> @@ -119,7 +118,7 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj, struct i915_vma *vma; int ret; - if (!atomic_read(&obj->bind_count)) + if (list_empty(&obj->vma.list)) return 0; /* @@ -142,6 +141,11 @@ try_again: if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) continue; + if (flags & I915_GEM_OBJECT_UNBIND_TEST) { + ret = -EBUSY; + break; + } + ret = -EAGAIN; if (!i915_vm_tryopen(vm)) break; @@ -994,18 +998,16 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, return ERR_PTR(ret); } + ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); + if (ret) + return ERR_PTR(ret); + if (vma->fence && !i915_gem_object_is_tiled(obj)) { mutex_lock(&ggtt->vm.mutex); - ret = i915_vma_revoke_fence(vma); + i915_vma_revoke_fence(vma); mutex_unlock(&ggtt->vm.mutex); - if (ret) - return ERR_PTR(ret); } - ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL); - if (ret) - return ERR_PTR(ret); - ret = i915_vma_wait_for_bind(vma); if (ret) { i915_vma_unpin(vma); @@ -1157,7 +1159,6 @@ err_unlock: /* Minimal basic recovery for KMS */ ret = i915_ggtt_enable_hw(dev_priv); i915_ggtt_resume(&dev_priv->ggtt); - i915_gem_restore_fences(&dev_priv->ggtt); intel_init_clock_gating(dev_priv); } diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index 0697bedebeef..6501939929d5 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -26,8 +26,6 @@ * */ -#include <drm/i915_drm.h> - #include "gem/i915_gem_context.h" #include "gt/intel_gt_requests.h" @@ -130,6 +128,13 @@ search_again: active = NULL; INIT_LIST_HEAD(&eviction_list); list_for_each_entry_safe(vma, next, &vm->bound_list, vm_link) { + if (vma == active) { /* now seen this vma twice */ + if (flags & PIN_NONBLOCK) + break; + + active = ERR_PTR(-EAGAIN); + } + /* * We keep this list in a rough least-recently scanned order * of active elements (inactive elements are cheap to reap). @@ -145,21 +150,12 @@ search_again: * To notice when we complete one full cycle, we record the * first active element seen, before moving it to the tail. */ - if (i915_vma_is_active(vma)) { - if (vma == active) { - if (flags & PIN_NONBLOCK) - break; + if (active != ERR_PTR(-EAGAIN) && i915_vma_is_active(vma)) { + if (!active) + active = vma; - active = ERR_PTR(-EAGAIN); - } - - if (active != ERR_PTR(-EAGAIN)) { - if (!active) - active = vma; - - list_move_tail(&vma->vm_link, &vm->bound_list); - continue; - } + list_move_tail(&vma->vm_link, &vm->bound_list); + continue; } if (mark_free(&scan, vma, flags, &eviction_list)) @@ -230,7 +226,12 @@ found: while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) { vma = container_of(node, struct i915_vma, node); - ret = __i915_vma_unbind(vma); + + /* If we find any non-objects (!vma), we cannot evict them */ + if (vma->node.color != I915_COLOR_UNEVICTABLE) + ret = __i915_vma_unbind(vma); + else + ret = -ENOSPC; /* XXX search failed, try again? */ } return ret; @@ -292,7 +293,8 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, GEM_BUG_ON(!drm_mm_node_allocated(node)); vma = container_of(node, typeof(*vma), node); - /* If we are using coloring to insert guard pages between + /* + * If we are using coloring to insert guard pages between * different cache domains within the address space, we have * to check whether the objects on either side of our range * abutt and conflict. If they are in conflict, then we evict @@ -309,22 +311,18 @@ int i915_gem_evict_for_node(struct i915_address_space *vm, } } - if (flags & PIN_NONBLOCK && - (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))) { + if (i915_vma_is_pinned(vma)) { ret = -ENOSPC; break; } - /* Overlap of objects in the same batch? */ - if (i915_vma_is_pinned(vma)) { + if (flags & PIN_NONBLOCK && i915_vma_is_active(vma)) { ret = -ENOSPC; - if (vma->exec_flags && - *vma->exec_flags & EXEC_OBJECT_PINNED) - ret = -EINVAL; break; } - /* Never show fear in the face of dragons! + /* + * Never show fear in the face of dragons! * * We cannot directly remove this node from within this * iterator and as with i915_gem_evict_something() we employ diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index e7834fa1e0ac..cb43381b0d37 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -15,8 +15,6 @@ #include <asm/set_memory.h> #include <asm/smp.h> -#include <drm/i915_drm.h> - #include "display/intel_frontbuffer.h" #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c index 2a4cd0ba5464..eec292d06f11 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.c +++ b/drivers/gpu/drm/i915/i915_gpu_error.c @@ -467,14 +467,14 @@ static void error_print_request(struct drm_i915_error_state_buf *m, if (!erq->seqno) return; - err_printf(m, "%s pid %d, seqno %8x:%08x%s%s, prio %d, start %08x, head %08x, tail %08x\n", + err_printf(m, "%s pid %d, seqno %8x:%08x%s%s, prio %d, head %08x, tail %08x\n", prefix, erq->pid, erq->context, erq->seqno, test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &erq->flags) ? "!" : "", test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &erq->flags) ? "+" : "", erq->sched_attr.priority, - erq->start, erq->head, erq->tail); + erq->head, erq->tail); } static void error_print_context(struct drm_i915_error_state_buf *m, @@ -1207,21 +1207,22 @@ static void engine_record_registers(struct intel_engine_coredump *ee) static void record_request(const struct i915_request *request, struct i915_request_coredump *erq) { - const struct i915_gem_context *ctx; - erq->flags = request->fence.flags; erq->context = request->fence.context; erq->seqno = request->fence.seqno; erq->sched_attr = request->sched.attr; - erq->start = i915_ggtt_offset(request->ring->vma); erq->head = request->head; erq->tail = request->tail; erq->pid = 0; rcu_read_lock(); - ctx = rcu_dereference(request->context->gem_context); - if (ctx) - erq->pid = pid_nr(ctx->pid); + if (!intel_context_is_closed(request->context)) { + const struct i915_gem_context *ctx; + + ctx = rcu_dereference(request->context->gem_context); + if (ctx) + erq->pid = pid_nr(ctx->pid); + } rcu_read_unlock(); } @@ -1319,26 +1320,6 @@ capture_user(struct intel_engine_capture_vma *capture, return capture; } -static struct i915_vma_coredump * -capture_object(const struct intel_gt *gt, - struct drm_i915_gem_object *obj, - const char *name, - struct i915_vma_compress *compress) -{ - if (obj && i915_gem_object_has_pages(obj)) { - struct i915_vma fake = { - .node = { .start = U64_MAX, .size = obj->base.size }, - .size = obj->base.size, - .pages = obj->mm.pages, - .obj = obj, - }; - - return i915_vma_coredump_create(gt, &fake, name, compress); - } else { - return NULL; - } -} - static void add_vma(struct intel_engine_coredump *ee, struct i915_vma_coredump *vma) { @@ -1427,12 +1408,6 @@ intel_engine_coredump_add_vma(struct intel_engine_coredump *ee, engine->wa_ctx.vma, "WA context", compress)); - - add_vma(ee, - capture_object(engine->gt, - engine->default_state, - "NULL context", - compress)); } static struct intel_engine_coredump * @@ -1858,7 +1833,7 @@ void i915_error_state_store(struct i915_gpu_coredump *error) return; i915 = error->i915; - dev_info(i915->drm.dev, "%s\n", error_msg(error)); + drm_info(&i915->drm, "%s\n", error_msg(error)); if (error->simulated || cmpxchg(&i915->gpu_error.first_error, NULL, error)) diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h index 0d1f6c8ff355..76b80fbfb7e9 100644 --- a/drivers/gpu/drm/i915/i915_gpu_error.h +++ b/drivers/gpu/drm/i915/i915_gpu_error.h @@ -42,7 +42,7 @@ struct i915_vma_coredump { int num_pages; int page_count; int unused; - u32 *pages[0]; + u32 *pages[]; }; struct i915_request_coredump { @@ -50,7 +50,6 @@ struct i915_request_coredump { pid_t pid; u32 context; u32 seqno; - u32 start; u32 head; u32 tail; struct i915_sched_attr sched_attr; diff --git a/drivers/gpu/drm/i915/i915_ioc32.c b/drivers/gpu/drm/i915/i915_ioc32.c index c1007245f46d..8e45ca3d2ede 100644 --- a/drivers/gpu/drm/i915/i915_ioc32.c +++ b/drivers/gpu/drm/i915/i915_ioc32.c @@ -28,9 +28,10 @@ */ #include <linux/compat.h> -#include <drm/i915_drm.h> #include <drm/drm_ioctl.h> + #include "i915_drv.h" +#include "i915_ioc32.h" struct drm_i915_getparam32 { s32 param; @@ -67,7 +68,7 @@ static drm_ioctl_compat_t *i915_compat_ioctls[] = { }; /** - * i915_compat_ioctl - handle the mistakes of the past + * i915_ioc32_compat_ioctl - handle the mistakes of the past * @filp: the file pointer * @cmd: the ioctl command (and encoded flags) * @arg: the ioctl argument (from userspace) @@ -75,7 +76,7 @@ static drm_ioctl_compat_t *i915_compat_ioctls[] = { * Called whenever a 32-bit process running under a 64-bit kernel * performs an ioctl on /dev/dri/card<n>. */ -long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +long i915_ioc32_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) { unsigned int nr = DRM_IOCTL_NR(cmd); drm_ioctl_compat_t *fn = NULL; diff --git a/drivers/gpu/drm/i915/i915_ioc32.h b/drivers/gpu/drm/i915/i915_ioc32.h new file mode 100644 index 000000000000..40dcd55ca213 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_ioc32.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __I915_IOC32_H__ +#define __I915_IOC32_H__ + +#ifdef CONFIG_COMPAT +struct file; +long i915_ioc32_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); +#else +#define i915_ioc32_compat_ioctl NULL +#endif + +#endif /* __I915_IOC32_H__ */ diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 3d0cd0960bd2..4dc601dffc08 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -34,7 +34,6 @@ #include <drm/drm_drv.h> #include <drm/drm_irq.h> -#include <drm/i915_drm.h> #include "display/intel_display_types.h" #include "display/intel_fifo_underrun.h" @@ -125,7 +124,6 @@ static const u32 hpd_status_i915[HPD_NUM_PINS] = { [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS, }; -/* BXT hpd list */ static const u32 hpd_bxt[HPD_NUM_PINS] = { [HPD_PORT_A] = BXT_DE_PORT_HP_DDIA, [HPD_PORT_B] = BXT_DE_PORT_HP_DDIB, @@ -169,6 +167,57 @@ static const u32 hpd_tgp[HPD_NUM_PINS] = { [HPD_PORT_I] = SDE_TC_HOTPLUG_ICP(PORT_TC6), }; +static void intel_hpd_init_pins(struct drm_i915_private *dev_priv) +{ + struct i915_hotplug *hpd = &dev_priv->hotplug; + + if (HAS_GMCH(dev_priv)) { + if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || + IS_CHERRYVIEW(dev_priv)) + hpd->hpd = hpd_status_g4x; + else + hpd->hpd = hpd_status_i915; + return; + } + + if (INTEL_GEN(dev_priv) >= 12) + hpd->hpd = hpd_gen12; + else if (INTEL_GEN(dev_priv) >= 11) + hpd->hpd = hpd_gen11; + else if (IS_GEN9_LP(dev_priv)) + hpd->hpd = hpd_bxt; + else if (INTEL_GEN(dev_priv) >= 8) + hpd->hpd = hpd_bdw; + else if (INTEL_GEN(dev_priv) >= 7) + hpd->hpd = hpd_ivb; + else + hpd->hpd = hpd_ilk; + + if (!HAS_PCH_SPLIT(dev_priv) || HAS_PCH_NOP(dev_priv)) + return; + + if (HAS_PCH_TGP(dev_priv) || HAS_PCH_JSP(dev_priv)) + hpd->pch_hpd = hpd_tgp; + else if (HAS_PCH_ICP(dev_priv) || HAS_PCH_MCC(dev_priv)) + hpd->pch_hpd = hpd_icp; + else if (HAS_PCH_CNP(dev_priv) || HAS_PCH_SPT(dev_priv)) + hpd->pch_hpd = hpd_spt; + else if (HAS_PCH_LPT(dev_priv) || HAS_PCH_CPT(dev_priv)) + hpd->pch_hpd = hpd_cpt; + else if (HAS_PCH_IBX(dev_priv)) + hpd->pch_hpd = hpd_ibx; + else + MISSING_CASE(INTEL_PCH_TYPE(dev_priv)); +} + +static void +intel_handle_vblank(struct drm_i915_private *dev_priv, enum pipe pipe) +{ + struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + + drm_crtc_handle_vblank(&crtc->base); +} + void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr, i915_reg_t iir, i915_reg_t ier) { @@ -772,13 +821,15 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc) return (position + crtc->scanline_offset) % vtotal; } -bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int index, - bool in_vblank_irq, int *vpos, int *hpos, - ktime_t *stime, ktime_t *etime, - const struct drm_display_mode *mode) +static bool i915_get_crtc_scanoutpos(struct drm_crtc *_crtc, + bool in_vblank_irq, + int *vpos, int *hpos, + ktime_t *stime, ktime_t *etime, + const struct drm_display_mode *mode) { + struct drm_device *dev = _crtc->dev; struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *crtc = to_intel_crtc(drm_crtc_from_index(dev, index)); + struct intel_crtc *crtc = to_intel_crtc(_crtc); enum pipe pipe = crtc->pipe; int position; int vbl_start, vbl_end, hsync_start, htotal, vtotal; @@ -890,6 +941,14 @@ bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int index, return true; } +bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error, + ktime_t *vblank_time, bool in_vblank_irq) +{ + return drm_crtc_vblank_helper_get_vblank_timestamp_internal( + crtc, max_error, vblank_time, in_vblank_irq, + i915_get_crtc_scanoutpos); +} + int intel_get_crtc_scanline(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); @@ -1200,8 +1259,8 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv, u32 crc2, u32 crc3, u32 crc4) { - struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe]; struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe); + struct intel_pipe_crc *pipe_crc = &crtc->pipe_crc; u32 crcs[5] = { crc0, crc1, crc2, crc3, crc4 }; trace_intel_pipe_crc(crtc, crcs); @@ -1364,7 +1423,7 @@ static void i8xx_pipestat_irq_handler(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) i9xx_pipe_crc_irq_handler(dev_priv, pipe); @@ -1382,7 +1441,7 @@ static void i915_pipestat_irq_handler(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) blc_event = true; @@ -1406,7 +1465,7 @@ static void i965_pipestat_irq_handler(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS) blc_event = true; @@ -1432,7 +1491,7 @@ static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS) i9xx_pipe_crc_irq_handler(dev_priv, pipe); @@ -1487,33 +1546,27 @@ static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 hotplug_status) { u32 pin_mask = 0, long_mask = 0; + u32 hotplug_trigger; - if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) || - IS_CHERRYVIEW(dev_priv)) { - u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; - - if (hotplug_trigger) { - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, - hotplug_trigger, hotplug_trigger, - hpd_status_g4x, - i9xx_port_hotplug_long_detect); + if (IS_G4X(dev_priv) || + IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) + hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X; + else + hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; - intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); - } + if (hotplug_trigger) { + intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, + hotplug_trigger, hotplug_trigger, + dev_priv->hotplug.hpd, + i9xx_port_hotplug_long_detect); - if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) - dp_aux_irq_handler(dev_priv); - } else { - u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915; - - if (hotplug_trigger) { - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, - hotplug_trigger, hotplug_trigger, - hpd_status_i915, - i9xx_port_hotplug_long_detect); - intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); - } + intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); } + + if ((IS_G4X(dev_priv) || + IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) && + hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X) + dp_aux_irq_handler(dev_priv); } static irqreturn_t valleyview_irq_handler(int irq, void *arg) @@ -1679,8 +1732,7 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg) } static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv, - u32 hotplug_trigger, - const u32 hpd[HPD_NUM_PINS]) + u32 hotplug_trigger) { u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; @@ -1703,8 +1755,9 @@ static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv, if (!hotplug_trigger) return; - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, - dig_hotplug_reg, hpd, + intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, + hotplug_trigger, dig_hotplug_reg, + dev_priv->hotplug.pch_hpd, pch_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -1715,7 +1768,7 @@ static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) enum pipe pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK; - ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx); + ibx_hpd_irq_handler(dev_priv, hotplug_trigger); if (pch_iir & SDE_AUDIO_POWER_MASK) { int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >> @@ -1739,11 +1792,12 @@ static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) if (pch_iir & SDE_POISON) drm_err(&dev_priv->drm, "PCH poison interrupt\n"); - if (pch_iir & SDE_FDI_MASK) + if (pch_iir & SDE_FDI_MASK) { for_each_pipe(dev_priv, pipe) drm_dbg(&dev_priv->drm, " pipe %c FDI IIR: 0x%08x\n", pipe_name(pipe), I915_READ(FDI_RX_IIR(pipe))); + } if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) drm_dbg(&dev_priv->drm, "PCH transcoder CRC done interrupt\n"); @@ -1802,7 +1856,7 @@ static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) enum pipe pipe; u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT; - ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt); + ibx_hpd_irq_handler(dev_priv, hotplug_trigger); if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) { int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >> @@ -1823,11 +1877,12 @@ static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) if (pch_iir & SDE_AUDIO_CP_CHG_CPT) drm_dbg(&dev_priv->drm, "Audio CP change interrupt\n"); - if (pch_iir & SDE_FDI_MASK_CPT) + if (pch_iir & SDE_FDI_MASK_CPT) { for_each_pipe(dev_priv, pipe) drm_dbg(&dev_priv->drm, " pipe %c FDI IIR: 0x%08x\n", pipe_name(pipe), I915_READ(FDI_RX_IIR(pipe))); + } if (pch_iir & SDE_ERROR_CPT) cpt_serr_int_handler(dev_priv); @@ -1838,22 +1893,18 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) u32 ddi_hotplug_trigger, tc_hotplug_trigger; u32 pin_mask = 0, long_mask = 0; bool (*tc_port_hotplug_long_detect)(enum hpd_pin pin, u32 val); - const u32 *pins; if (HAS_PCH_TGP(dev_priv)) { ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP; tc_hotplug_trigger = pch_iir & SDE_TC_MASK_TGP; tc_port_hotplug_long_detect = tgp_tc_port_hotplug_long_detect; - pins = hpd_tgp; } else if (HAS_PCH_JSP(dev_priv)) { ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_TGP; tc_hotplug_trigger = 0; - pins = hpd_tgp; } else if (HAS_PCH_MCC(dev_priv)) { ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP; tc_hotplug_trigger = pch_iir & SDE_TC_HOTPLUG_ICP(PORT_TC1); tc_port_hotplug_long_detect = icp_tc_port_hotplug_long_detect; - pins = hpd_icp; } else { drm_WARN(&dev_priv->drm, !HAS_PCH_ICP(dev_priv), "Unrecognized PCH type 0x%x\n", @@ -1862,7 +1913,6 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP; tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP; tc_port_hotplug_long_detect = icp_tc_port_hotplug_long_detect; - pins = hpd_icp; } if (ddi_hotplug_trigger) { @@ -1872,8 +1922,8 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg); intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, - ddi_hotplug_trigger, - dig_hotplug_reg, pins, + ddi_hotplug_trigger, dig_hotplug_reg, + dev_priv->hotplug.pch_hpd, icp_ddi_port_hotplug_long_detect); } @@ -1884,8 +1934,8 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg); intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, - tc_hotplug_trigger, - dig_hotplug_reg, pins, + tc_hotplug_trigger, dig_hotplug_reg, + dev_priv->hotplug.pch_hpd, tc_port_hotplug_long_detect); } @@ -1910,7 +1960,8 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, - hotplug_trigger, dig_hotplug_reg, hpd_spt, + hotplug_trigger, dig_hotplug_reg, + dev_priv->hotplug.pch_hpd, spt_port_hotplug_long_detect); } @@ -1921,7 +1972,8 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) I915_WRITE(PCH_PORT_HOTPLUG2, dig_hotplug_reg); intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, - hotplug2_trigger, dig_hotplug_reg, hpd_spt, + hotplug2_trigger, dig_hotplug_reg, + dev_priv->hotplug.pch_hpd, spt_port_hotplug2_long_detect); } @@ -1933,16 +1985,16 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir) } static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv, - u32 hotplug_trigger, - const u32 hpd[HPD_NUM_PINS]) + u32 hotplug_trigger) { u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL); I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, dig_hotplug_reg); - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, - dig_hotplug_reg, hpd, + intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, + hotplug_trigger, dig_hotplug_reg, + dev_priv->hotplug.hpd, ilk_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -1955,7 +2007,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG; if (hotplug_trigger) - ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk); + ilk_hpd_irq_handler(dev_priv, hotplug_trigger); if (de_iir & DE_AUX_CHANNEL_A) dp_aux_irq_handler(dev_priv); @@ -1968,7 +2020,7 @@ static void ilk_display_irq_handler(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { if (de_iir & DE_PIPE_VBLANK(pipe)) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe)) intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe); @@ -2001,7 +2053,7 @@ static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB; if (hotplug_trigger) - ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb); + ilk_hpd_irq_handler(dev_priv, hotplug_trigger); if (de_iir & DE_ERR_INT_IVB) ivb_err_int_handler(dev_priv); @@ -2021,7 +2073,7 @@ static void ivb_display_irq_handler(struct drm_i915_private *dev_priv, for_each_pipe(dev_priv, pipe) { if (de_iir & (DE_PIPE_VBLANK_IVB(pipe))) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); } /* check event from PCH */ @@ -2111,16 +2163,16 @@ static irqreturn_t ilk_irq_handler(int irq, void *arg) } static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv, - u32 hotplug_trigger, - const u32 hpd[HPD_NUM_PINS]) + u32 hotplug_trigger) { u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0; dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG); I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg); - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, hotplug_trigger, - dig_hotplug_reg, hpd, + intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, + hotplug_trigger, dig_hotplug_reg, + dev_priv->hotplug.hpd, bxt_port_hotplug_long_detect); intel_hpd_irq_handler(dev_priv, pin_mask, long_mask); @@ -2132,15 +2184,11 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir) u32 trigger_tc = iir & GEN11_DE_TC_HOTPLUG_MASK; u32 trigger_tbt = iir & GEN11_DE_TBT_HOTPLUG_MASK; long_pulse_detect_func long_pulse_detect; - const u32 *hpd; - if (INTEL_GEN(dev_priv) >= 12) { + if (INTEL_GEN(dev_priv) >= 12) long_pulse_detect = gen12_port_hotplug_long_detect; - hpd = hpd_gen12; - } else { + else long_pulse_detect = gen11_port_hotplug_long_detect; - hpd = hpd_gen11; - } if (trigger_tc) { u32 dig_hotplug_reg; @@ -2148,8 +2196,10 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir) dig_hotplug_reg = I915_READ(GEN11_TC_HOTPLUG_CTL); I915_WRITE(GEN11_TC_HOTPLUG_CTL, dig_hotplug_reg); - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tc, - dig_hotplug_reg, hpd, long_pulse_detect); + intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, + trigger_tc, dig_hotplug_reg, + dev_priv->hotplug.hpd, + long_pulse_detect); } if (trigger_tbt) { @@ -2158,8 +2208,10 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir) dig_hotplug_reg = I915_READ(GEN11_TBT_HOTPLUG_CTL); I915_WRITE(GEN11_TBT_HOTPLUG_CTL, dig_hotplug_reg); - intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, trigger_tbt, - dig_hotplug_reg, hpd, long_pulse_detect); + intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask, + trigger_tbt, dig_hotplug_reg, + dev_priv->hotplug.hpd, + long_pulse_detect); } if (pin_mask) @@ -2290,15 +2342,13 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) if (IS_GEN9_LP(dev_priv)) { tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK; if (tmp_mask) { - bxt_hpd_irq_handler(dev_priv, tmp_mask, - hpd_bxt); + bxt_hpd_irq_handler(dev_priv, tmp_mask); found = true; } } else if (IS_BROADWELL(dev_priv)) { tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG; if (tmp_mask) { - ilk_hpd_irq_handler(dev_priv, - tmp_mask, hpd_bdw); + ilk_hpd_irq_handler(dev_priv, tmp_mask); found = true; } } @@ -2334,7 +2384,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl) I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir); if (iir & GEN8_PIPE_VBLANK) - drm_handle_vblank(&dev_priv->drm, pipe); + intel_handle_vblank(dev_priv, pipe); if (iir & GEN8_PIPE_CDCLK_CRC_DONE) hsw_pipe_crc_irq_handler(dev_priv, pipe); @@ -2851,6 +2901,14 @@ static void gen11_display_irq_reset(struct drm_i915_private *dev_priv) if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) GEN3_IRQ_RESET(uncore, SDE); + + /* Wa_14010685332:icl */ + if (INTEL_PCH_TYPE(dev_priv) == PCH_ICP) { + intel_uncore_rmw(uncore, SOUTH_CHICKEN1, + SBCLK_RUN_REFCLK_DIS, SBCLK_RUN_REFCLK_DIS); + intel_uncore_rmw(uncore, SOUTH_CHICKEN1, + SBCLK_RUN_REFCLK_DIS, 0); + } } static void gen11_irq_reset(struct drm_i915_private *dev_priv) @@ -2970,13 +3028,12 @@ static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - if (HAS_PCH_IBX(dev_priv)) { + if (HAS_PCH_IBX(dev_priv)) hotplug_irqs = SDE_HOTPLUG_MASK; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx); - } else { + else hotplug_irqs = SDE_HOTPLUG_MASK_CPT; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt); - } + + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd); ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); @@ -3002,13 +3059,12 @@ static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv, static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv, u32 sde_ddi_mask, u32 sde_tc_mask, - u32 ddi_enable_mask, u32 tc_enable_mask, - const u32 *pins) + u32 ddi_enable_mask, u32 tc_enable_mask) { u32 hotplug_irqs, enabled_irqs; hotplug_irqs = sde_ddi_mask | sde_tc_mask; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, pins); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd); I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ); @@ -3025,8 +3081,7 @@ static void mcc_hpd_irq_setup(struct drm_i915_private *dev_priv) { icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_ICP, SDE_TC_HOTPLUG_ICP(PORT_TC1), - ICP_DDI_HPD_ENABLE_MASK, ICP_TC_HPD_ENABLE(PORT_TC1), - hpd_icp); + ICP_DDI_HPD_ENABLE_MASK, ICP_TC_HPD_ENABLE(PORT_TC1)); } /* @@ -3038,8 +3093,7 @@ static void jsp_hpd_irq_setup(struct drm_i915_private *dev_priv) { icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_TGP, 0, - TGP_DDI_HPD_ENABLE_MASK, 0, - hpd_tgp); + TGP_DDI_HPD_ENABLE_MASK, 0); } static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv) @@ -3064,11 +3118,9 @@ static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv) static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - const u32 *hpd; u32 val; - hpd = INTEL_GEN(dev_priv) >= 12 ? hpd_gen12 : hpd_gen11; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); hotplug_irqs = GEN11_DE_TC_HOTPLUG_MASK | GEN11_DE_TBT_HOTPLUG_MASK; val = I915_READ(GEN11_DE_HPD_IMR); @@ -3080,12 +3132,10 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv) if (INTEL_PCH_TYPE(dev_priv) >= PCH_TGP) icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_TGP, SDE_TC_MASK_TGP, - TGP_DDI_HPD_ENABLE_MASK, - TGP_TC_HPD_ENABLE_MASK, hpd_tgp); + TGP_DDI_HPD_ENABLE_MASK, TGP_TC_HPD_ENABLE_MASK); else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP) icp_hpd_irq_setup(dev_priv, SDE_DDI_MASK_ICP, SDE_TC_MASK_ICP, - ICP_DDI_HPD_ENABLE_MASK, - ICP_TC_HPD_ENABLE_MASK, hpd_icp); + ICP_DDI_HPD_ENABLE_MASK, ICP_TC_HPD_ENABLE_MASK); } static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv) @@ -3121,7 +3171,7 @@ static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv) I915_WRITE(SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ); hotplug_irqs = SDE_HOTPLUG_MASK_SPT; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd); ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs); @@ -3150,17 +3200,17 @@ static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv) if (INTEL_GEN(dev_priv) >= 8) { hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); } else if (INTEL_GEN(dev_priv) >= 7) { hotplug_irqs = DE_DP_A_HOTPLUG_IVB; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); } else { hotplug_irqs = DE_DP_A_HOTPLUG; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs); } @@ -3211,7 +3261,7 @@ static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv) { u32 hotplug_irqs, enabled_irqs; - enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt); + enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd); hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK; bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs); @@ -3339,9 +3389,10 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) { struct intel_uncore *uncore = &dev_priv->uncore; - u32 de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE; + u32 de_pipe_masked = gen8_de_pipe_fault_mask(dev_priv) | + GEN8_PIPE_CDCLK_CRC_DONE; u32 de_pipe_enables; - u32 de_port_masked = GEN8_AUX_CHANNEL_A; + u32 de_port_masked = gen8_de_port_aux_mask(dev_priv); u32 de_port_enables; u32 de_misc_masked = GEN8_DE_EDP_PSR; enum pipe pipe; @@ -3349,21 +3400,8 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv) if (INTEL_GEN(dev_priv) <= 10) de_misc_masked |= GEN8_DE_MISC_GSE; - if (INTEL_GEN(dev_priv) >= 9) { - de_pipe_masked |= GEN9_DE_PIPE_IRQ_FAULT_ERRORS; - de_port_masked |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C | - GEN9_AUX_CHANNEL_D; - if (IS_GEN9_LP(dev_priv)) - de_port_masked |= BXT_DE_PORT_GMBUS; - } else { - de_pipe_masked |= GEN8_DE_PIPE_IRQ_FAULT_ERRORS; - } - - if (INTEL_GEN(dev_priv) >= 11) - de_port_masked |= ICL_AUX_CHANNEL_E; - - if (IS_CNL_WITH_PORT_F(dev_priv) || INTEL_GEN(dev_priv) >= 11) - de_port_masked |= CNL_AUX_CHANNEL_F; + if (IS_GEN9_LP(dev_priv)) + de_port_masked |= BXT_DE_PORT_GMBUS; de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN; @@ -3639,7 +3677,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg) intel_uncore_write16(&dev_priv->uncore, GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]); + intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); if (iir & I915_MASTER_ERROR_INTERRUPT) i8xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -3744,7 +3782,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg) I915_WRITE(GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]); + intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -3886,10 +3924,10 @@ static irqreturn_t i965_irq_handler(int irq, void *arg) I915_WRITE(GEN2_IIR, iir); if (iir & I915_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->engine[RCS0]); + intel_engine_signal_breadcrumbs(dev_priv->gt.engine[RCS0]); if (iir & I915_BSD_USER_INTERRUPT) - intel_engine_signal_breadcrumbs(dev_priv->engine[VCS0]); + intel_engine_signal_breadcrumbs(dev_priv->gt.engine[VCS0]); if (iir & I915_MASTER_ERROR_INTERRUPT) i9xx_error_irq_handler(dev_priv, eir, eir_stuck); @@ -3917,6 +3955,8 @@ void intel_irq_init(struct drm_i915_private *dev_priv) struct drm_device *dev = &dev_priv->drm; int i; + intel_hpd_init_pins(dev_priv); + intel_hpd_init_work(dev_priv); INIT_WORK(&dev_priv->l3_parity.error_work, ivb_parity_work); diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h index 812c47a9c2d6..25f25cd95818 100644 --- a/drivers/gpu/drm/i915/i915_irq.h +++ b/drivers/gpu/drm/i915/i915_irq.h @@ -101,10 +101,8 @@ void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv, void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv, u8 pipe_mask); -bool i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe, - bool in_vblank_irq, int *vpos, int *hpos, - ktime_t *stime, ktime_t *etime, - const struct drm_display_mode *mode); +bool intel_crtc_get_vblank_timestamp(struct drm_crtc *crtc, int *max_error, + ktime_t *vblank_time, bool in_vblank_irq); u32 i915_get_vblank_counter(struct drm_crtc *crtc); u32 g4x_get_vblank_counter(struct drm_crtc *crtc); diff --git a/drivers/gpu/drm/i915/i915_memcpy.c b/drivers/gpu/drm/i915/i915_memcpy.c index fdd550405fd3..7b3b83bd5ab8 100644 --- a/drivers/gpu/drm/i915/i915_memcpy.c +++ b/drivers/gpu/drm/i915/i915_memcpy.c @@ -35,7 +35,6 @@ static DEFINE_STATIC_KEY_FALSE(has_movntdqa); -#ifdef CONFIG_AS_MOVNTDQA static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) { kernel_fpu_begin(); @@ -93,10 +92,6 @@ static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len) kernel_fpu_end(); } -#else -static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) {} -static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len) {} -#endif /** * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index 24b1f0ce8743..193048ce3c3a 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -26,6 +26,7 @@ #include <linux/vga_switcheroo.h> #include <drm/drm_drv.h> +#include <drm/i915_pciids.h> #include "display/intel_fbdev.h" @@ -159,6 +160,7 @@ GEN(2), \ .is_mobile = 1, \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_overlay = 1, \ .display.cursor_needs_physical = 1, \ .display.overlay_needs_physical = 1, \ @@ -169,6 +171,7 @@ .engine_mask = BIT(RCS0), \ .has_snoop = true, \ .has_coherent_ggtt = false, \ + .dma_mask_size = 32, \ I9XX_PIPE_OFFSETS, \ I9XX_CURSOR_OFFSETS, \ I9XX_COLORS, \ @@ -178,6 +181,7 @@ #define I845_FEATURES \ GEN(2), \ .pipe_mask = BIT(PIPE_A), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A), \ .display.has_overlay = 1, \ .display.overlay_needs_physical = 1, \ .display.has_gmch = 1, \ @@ -187,6 +191,7 @@ .engine_mask = BIT(RCS0), \ .has_snoop = true, \ .has_coherent_ggtt = false, \ + .dma_mask_size = 32, \ I845_PIPE_OFFSETS, \ I845_CURSOR_OFFSETS, \ I9XX_COLORS, \ @@ -217,11 +222,13 @@ static const struct intel_device_info i865g_info = { #define GEN3_FEATURES \ GEN(3), \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_gmch = 1, \ .gpu_reset_clobbers_display = true, \ .engine_mask = BIT(RCS0), \ .has_snoop = true, \ .has_coherent_ggtt = true, \ + .dma_mask_size = 32, \ I9XX_PIPE_OFFSETS, \ I9XX_CURSOR_OFFSETS, \ I9XX_COLORS, \ @@ -282,6 +289,7 @@ static const struct intel_device_info g33_info = { PLATFORM(INTEL_G33), .display.has_hotplug = 1, .display.has_overlay = 1, + .dma_mask_size = 36, }; static const struct intel_device_info pnv_g_info = { @@ -289,6 +297,7 @@ static const struct intel_device_info pnv_g_info = { PLATFORM(INTEL_PINEVIEW), .display.has_hotplug = 1, .display.has_overlay = 1, + .dma_mask_size = 36, }; static const struct intel_device_info pnv_m_info = { @@ -297,17 +306,20 @@ static const struct intel_device_info pnv_m_info = { .is_mobile = 1, .display.has_hotplug = 1, .display.has_overlay = 1, + .dma_mask_size = 36, }; #define GEN4_FEATURES \ GEN(4), \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ .display.has_gmch = 1, \ .gpu_reset_clobbers_display = true, \ .engine_mask = BIT(RCS0), \ .has_snoop = true, \ .has_coherent_ggtt = true, \ + .dma_mask_size = 36, \ I9XX_PIPE_OFFSETS, \ I9XX_CURSOR_OFFSETS, \ I965_COLORS, \ @@ -353,12 +365,14 @@ static const struct intel_device_info gm45_info = { #define GEN5_FEATURES \ GEN(5), \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ .engine_mask = BIT(RCS0) | BIT(VCS0), \ .has_snoop = true, \ .has_coherent_ggtt = true, \ /* ilk does support rc6, but we do not implement [power] contexts */ \ .has_rc6 = 0, \ + .dma_mask_size = 36, \ I9XX_PIPE_OFFSETS, \ I9XX_CURSOR_OFFSETS, \ ILK_COLORS, \ @@ -380,6 +394,7 @@ static const struct intel_device_info ilk_m_info = { #define GEN6_FEATURES \ GEN(6), \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \ .display.has_hotplug = 1, \ .display.has_fbc = 1, \ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ @@ -388,6 +403,7 @@ static const struct intel_device_info ilk_m_info = { .has_rc6 = 1, \ .has_rc6p = 1, \ .has_rps = true, \ + .dma_mask_size = 40, \ .ppgtt_type = INTEL_PPGTT_ALIASING, \ .ppgtt_size = 31, \ I9XX_PIPE_OFFSETS, \ @@ -429,6 +445,7 @@ static const struct intel_device_info snb_m_gt2_info = { #define GEN7_FEATURES \ GEN(7), \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), \ .display.has_hotplug = 1, \ .display.has_fbc = 1, \ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \ @@ -437,7 +454,8 @@ static const struct intel_device_info snb_m_gt2_info = { .has_rc6 = 1, \ .has_rc6p = 1, \ .has_rps = true, \ - .ppgtt_type = INTEL_PPGTT_FULL, \ + .dma_mask_size = 40, \ + .ppgtt_type = INTEL_PPGTT_ALIASING, \ .ppgtt_size = 31, \ IVB_PIPE_OFFSETS, \ IVB_CURSOR_OFFSETS, \ @@ -481,6 +499,7 @@ static const struct intel_device_info ivb_q_info = { PLATFORM(INTEL_IVYBRIDGE), .gt = 2, .pipe_mask = 0, /* legal, last one wins */ + .cpu_transcoder_mask = 0, .has_l3_dpf = 1, }; @@ -489,12 +508,14 @@ static const struct intel_device_info vlv_info = { GEN(7), .is_lp = 1, .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), .has_runtime_pm = 1, .has_rc6 = 1, .has_rps = true, .display.has_gmch = 1, .display.has_hotplug = 1, - .ppgtt_type = INTEL_PPGTT_FULL, + .dma_mask_size = 40, + .ppgtt_type = INTEL_PPGTT_ALIASING, .ppgtt_size = 31, .has_snoop = true, .has_coherent_ggtt = false, @@ -510,6 +531,8 @@ static const struct intel_device_info vlv_info = { #define G75_FEATURES \ GEN7_FEATURES, \ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP), \ .display.has_ddi = 1, \ .has_fpga_dbg = 1, \ .display.has_psr = 1, \ @@ -542,6 +565,7 @@ static const struct intel_device_info hsw_gt3_info = { G75_FEATURES, \ GEN(8), \ .has_logical_ring_contexts = 1, \ + .dma_mask_size = 39, \ .ppgtt_type = INTEL_PPGTT_FULL, \ .ppgtt_size = 48, \ .has_64bit_reloc = 1, \ @@ -580,6 +604,7 @@ static const struct intel_device_info chv_info = { PLATFORM(INTEL_CHERRYVIEW), GEN(8), .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), .display.has_hotplug = 1, .is_lp = 1, .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), @@ -589,7 +614,8 @@ static const struct intel_device_info chv_info = { .has_rps = true, .has_logical_ring_contexts = 1, .display.has_gmch = 1, - .ppgtt_type = INTEL_PPGTT_ALIASING, + .dma_mask_size = 39, + .ppgtt_type = INTEL_PPGTT_FULL, .ppgtt_size = 32, .has_reset_engine = 1, .has_snoop = true, @@ -655,6 +681,9 @@ static const struct intel_device_info skl_gt4_info = { .display.has_hotplug = 1, \ .engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \ .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP) | \ + BIT(TRANSCODER_DSI_A) | BIT(TRANSCODER_DSI_C), \ .has_64bit_reloc = 1, \ .display.has_ddi = 1, \ .has_fpga_dbg = 1, \ @@ -669,6 +698,7 @@ static const struct intel_device_info skl_gt4_info = { .has_logical_ring_contexts = 1, \ .has_logical_ring_preemption = 1, \ .has_gt_uc = 1, \ + .dma_mask_size = 39, \ .ppgtt_type = INTEL_PPGTT_FULL, \ .ppgtt_size = 48, \ .has_reset_engine = 1, \ @@ -758,6 +788,9 @@ static const struct intel_device_info cnl_info = { #define GEN11_FEATURES \ GEN10_FEATURES, \ GEN11_DEFAULT_PAGE_SIZES, \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP) | \ + BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), \ .pipe_offsets = { \ [TRANSCODER_A] = PIPE_A_OFFSET, \ [TRANSCODER_B] = PIPE_B_OFFSET, \ @@ -798,6 +831,10 @@ static const struct intel_device_info ehl_info = { #define GEN12_FEATURES \ GEN11_FEATURES, \ GEN(12), \ + .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \ + .cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \ + BIT(TRANSCODER_C) | BIT(TRANSCODER_D) | \ + BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), \ .pipe_offsets = { \ [TRANSCODER_A] = PIPE_A_OFFSET, \ [TRANSCODER_B] = PIPE_B_OFFSET, \ @@ -821,8 +858,6 @@ static const struct intel_device_info ehl_info = { static const struct intel_device_info tgl_info = { GEN12_FEATURES, PLATFORM(INTEL_TIGERLAKE), - .pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), - .require_force_probe = 1, .display.has_modular_fia = 1, .engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2), diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index b5249ee5bda6..c533f569dd42 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -204,21 +204,6 @@ #include "i915_drv.h" #include "i915_perf.h" -#include "oa/i915_oa_hsw.h" -#include "oa/i915_oa_bdw.h" -#include "oa/i915_oa_chv.h" -#include "oa/i915_oa_sklgt2.h" -#include "oa/i915_oa_sklgt3.h" -#include "oa/i915_oa_sklgt4.h" -#include "oa/i915_oa_bxt.h" -#include "oa/i915_oa_kblgt2.h" -#include "oa/i915_oa_kblgt3.h" -#include "oa/i915_oa_glk.h" -#include "oa/i915_oa_cflgt2.h" -#include "oa/i915_oa_cflgt3.h" -#include "oa/i915_oa_cnl.h" -#include "oa/i915_oa_icl.h" -#include "oa/i915_oa_tgl.h" /* HW requires this to be a power of two, between 128k and 16M, though driver * is currently generally designed assuming the largest 16M size is used such @@ -238,26 +223,17 @@ * * Although this can be observed explicitly while copying reports to userspace * by checking for a zeroed report-id field in tail reports, we want to account - * for this earlier, as part of the oa_buffer_check to avoid lots of redundant - * read() attempts. - * - * In effect we define a tail pointer for reading that lags the real tail - * pointer by at least %OA_TAIL_MARGIN_NSEC nanoseconds, which gives enough - * time for the corresponding reports to become visible to the CPU. - * - * To manage this we actually track two tail pointers: - * 1) An 'aging' tail with an associated timestamp that is tracked until we - * can trust the corresponding data is visible to the CPU; at which point - * it is considered 'aged'. - * 2) An 'aged' tail that can be used for read()ing. - * - * The two separate pointers let us decouple read()s from tail pointer aging. - * - * The tail pointers are checked and updated at a limited rate within a hrtimer - * callback (the same callback that is used for delivering EPOLLIN events) - * - * Initially the tails are marked invalid with %INVALID_TAIL_PTR which - * indicates that an updated tail pointer is needed. + * for this earlier, as part of the oa_buffer_check_unlocked to avoid lots of + * redundant read() attempts. + * + * We workaround this issue in oa_buffer_check_unlocked() by reading the reports + * in the OA buffer, starting from the tail reported by the HW until we find a + * report with its first 2 dwords not 0 meaning its previous report is + * completely in memory and ready to be read. Those dwords are also set to 0 + * once read and the whole buffer is cleared upon OA buffer initialization. The + * first dword is the reason for this report while the second is the timestamp, + * making the chances of having those 2 fields at 0 fairly unlikely. A more + * detailed explanation is available in oa_buffer_check_unlocked(). * * Most of the implementation details for this workaround are in * oa_buffer_check_unlocked() and _append_oa_reports() @@ -272,11 +248,11 @@ #define OA_TAIL_MARGIN_NSEC 100000ULL #define INVALID_TAIL_PTR 0xffffffff -/* frequency for checking whether the OA unit has written new reports to the - * circular OA buffer... +/* The default frequency for checking whether the OA unit has written new + * reports to the circular OA buffer... */ -#define POLL_FREQUENCY 200 -#define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) +#define DEFAULT_POLL_FREQUENCY_HZ 200 +#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ) /* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */ static u32 i915_perf_stream_paranoid = true; @@ -359,6 +335,12 @@ static const struct i915_oa_format gen12_oa_formats[I915_OA_FORMAT_MAX] = { * @oa_periodic: Whether to enable periodic OA unit sampling * @oa_period_exponent: The OA unit sampling period is derived from this * @engine: The engine (typically rcs0) being monitored by the OA unit + * @has_sseu: Whether @sseu was specified by userspace + * @sseu: internal SSEU configuration computed either from the userspace + * specified configuration in the opening parameters or a default value + * (see get_default_sseu_config()) + * @poll_oa_period: The period in nanoseconds at which the CPU will check for OA + * data availability * * As read_properties_unlocked() enumerates and validates the properties given * to open a stream of metrics the configuration is built up in the structure @@ -378,6 +360,11 @@ struct perf_open_properties { int oa_period_exponent; struct intel_engine_cs *engine; + + bool has_sseu; + struct intel_sseu sseu; + + u64 poll_oa_period; }; struct i915_oa_config_bo { @@ -409,10 +396,7 @@ i915_perf_get_oa_config(struct i915_perf *perf, int metrics_set) struct i915_oa_config *oa_config; rcu_read_lock(); - if (metrics_set == 1) - oa_config = &perf->test_config; - else - oa_config = idr_find(&perf->metrics_idr, metrics_set); + oa_config = idr_find(&perf->metrics_idr, metrics_set); if (oa_config) oa_config = i915_oa_config_get(oa_config); rcu_read_unlock(); @@ -465,8 +449,8 @@ static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) * (See description of OA_TAIL_MARGIN_NSEC above for further details.) * * Besides returning true when there is data available to read() this function - * also has the side effect of updating the oa_buffer.tails[], .aging_timestamp - * and .aged_tail_idx state used for reading. + * also updates the tail, aging_tail and aging_timestamp in the oa_buffer + * object. * * Note: It's safe to read OA config state here unlocked, assuming that this is * only called while the stream is enabled, while the global OA configuration @@ -476,28 +460,19 @@ static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) */ static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) { + u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); int report_size = stream->oa_buffer.format_size; unsigned long flags; - unsigned int aged_idx; - u32 head, hw_tail, aged_tail, aging_tail; + bool pollin; + u32 hw_tail; u64 now; /* We have to consider the (unlikely) possibility that read() errors - * could result in an OA buffer reset which might reset the head, - * tails[] and aged_tail state. + * could result in an OA buffer reset which might reset the head and + * tail state. */ spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); - /* NB: The head we observe here might effectively be a little out of - * date (between head and tails[aged_idx].offset if there is currently - * a read() in progress. - */ - head = stream->oa_buffer.head; - - aged_idx = stream->oa_buffer.aged_tail_idx; - aged_tail = stream->oa_buffer.tails[aged_idx].offset; - aging_tail = stream->oa_buffer.tails[!aged_idx].offset; - hw_tail = stream->perf->ops.oa_hw_tail_read(stream); /* The tail pointer increases in 64 byte increments, @@ -507,63 +482,63 @@ static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) now = ktime_get_mono_fast_ns(); - /* Update the aged tail - * - * Flip the tail pointer available for read()s once the aging tail is - * old enough to trust that the corresponding data will be visible to - * the CPU... - * - * Do this before updating the aging pointer in case we may be able to - * immediately start aging a new pointer too (if new data has become - * available) without needing to wait for a later hrtimer callback. - */ - if (aging_tail != INVALID_TAIL_PTR && - ((now - stream->oa_buffer.aging_timestamp) > - OA_TAIL_MARGIN_NSEC)) { - - aged_idx ^= 1; - stream->oa_buffer.aged_tail_idx = aged_idx; + if (hw_tail == stream->oa_buffer.aging_tail && + (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) { + /* If the HW tail hasn't move since the last check and the HW + * tail has been aging for long enough, declare it the new + * tail. + */ + stream->oa_buffer.tail = stream->oa_buffer.aging_tail; + } else { + u32 head, tail, aged_tail; - aged_tail = aging_tail; + /* NB: The head we observe here might effectively be a little + * out of date. If a read() is in progress, the head could be + * anywhere between this head and stream->oa_buffer.tail. + */ + head = stream->oa_buffer.head - gtt_offset; + aged_tail = stream->oa_buffer.tail - gtt_offset; + + hw_tail -= gtt_offset; + tail = hw_tail; + + /* Walk the stream backward until we find a report with dword 0 + * & 1 not at 0. Since the circular buffer pointers progress by + * increments of 64 bytes and that reports can be up to 256 + * bytes long, we can't tell whether a report has fully landed + * in memory before the first 2 dwords of the following report + * have effectively landed. + * + * This is assuming that the writes of the OA unit land in + * memory in the order they were written to. + * If not : (╯°□°）╯︵ ┻━┻ + */ + while (OA_TAKEN(tail, aged_tail) >= report_size) { + u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail); - /* Mark that we need a new pointer to start aging... */ - stream->oa_buffer.tails[!aged_idx].offset = INVALID_TAIL_PTR; - aging_tail = INVALID_TAIL_PTR; - } + if (report32[0] != 0 || report32[1] != 0) + break; - /* Update the aging tail - * - * We throttle aging tail updates until we have a new tail that - * represents >= one report more data than is already available for - * reading. This ensures there will be enough data for a successful - * read once this new pointer has aged and ensures we will give the new - * pointer time to age. - */ - if (aging_tail == INVALID_TAIL_PTR && - (aged_tail == INVALID_TAIL_PTR || - OA_TAKEN(hw_tail, aged_tail) >= report_size)) { - struct i915_vma *vma = stream->oa_buffer.vma; - u32 gtt_offset = i915_ggtt_offset(vma); - - /* Be paranoid and do a bounds check on the pointer read back - * from hardware, just in case some spurious hardware condition - * could put the tail out of bounds... - */ - if (hw_tail >= gtt_offset && - hw_tail < (gtt_offset + OA_BUFFER_SIZE)) { - stream->oa_buffer.tails[!aged_idx].offset = - aging_tail = hw_tail; - stream->oa_buffer.aging_timestamp = now; - } else { - DRM_ERROR("Ignoring spurious out of range OA buffer tail pointer = %x\n", - hw_tail); + tail = (tail - report_size) & (OA_BUFFER_SIZE - 1); } + + if (OA_TAKEN(hw_tail, tail) > report_size && + __ratelimit(&stream->perf->tail_pointer_race)) + DRM_NOTE("unlanded report(s) head=0x%x " + "tail=0x%x hw_tail=0x%x\n", + head, tail, hw_tail); + + stream->oa_buffer.tail = gtt_offset + tail; + stream->oa_buffer.aging_tail = gtt_offset + hw_tail; + stream->oa_buffer.aging_timestamp = now; } + pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset, + stream->oa_buffer.head - gtt_offset) >= report_size; + spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); - return aged_tail == INVALID_TAIL_PTR ? - false : OA_TAKEN(aged_tail, head) >= report_size; + return pollin; } /** @@ -681,7 +656,6 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, u32 mask = (OA_BUFFER_SIZE - 1); size_t start_offset = *offset; unsigned long flags; - unsigned int aged_tail_idx; u32 head, tail; u32 taken; int ret = 0; @@ -692,19 +666,11 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); head = stream->oa_buffer.head; - aged_tail_idx = stream->oa_buffer.aged_tail_idx; - tail = stream->oa_buffer.tails[aged_tail_idx].offset; + tail = stream->oa_buffer.tail; spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); /* - * An invalid tail pointer here means we're still waiting for the poll - * hrtimer callback to give us a pointer - */ - if (tail == INVALID_TAIL_PTR) - return -EAGAIN; - - /* * NB: oa_buffer.head/tail include the gtt_offset which we don't want * while indexing relative to oa_buf_base. */ @@ -745,7 +711,8 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, */ if (drm_WARN_ON(&uncore->i915->drm, (OA_BUFFER_SIZE - head) < report_size)) { - DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); + drm_err(&uncore->i915->drm, + "Spurious OA head ptr: non-integral report offset\n"); break; } @@ -836,13 +803,11 @@ static int gen8_append_oa_reports(struct i915_perf_stream *stream, } /* - * The above reason field sanity check is based on - * the assumption that the OA buffer is initially - * zeroed and we reset the field after copying so the - * check is still meaningful once old reports start - * being overwritten. + * Clear out the first 2 dword as a mean to detect unlanded + * reports. */ report32[0] = 0; + report32[1] = 0; } if (start_offset != *offset) { @@ -983,7 +948,6 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream, u32 mask = (OA_BUFFER_SIZE - 1); size_t start_offset = *offset; unsigned long flags; - unsigned int aged_tail_idx; u32 head, tail; u32 taken; int ret = 0; @@ -994,17 +958,10 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream, spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); head = stream->oa_buffer.head; - aged_tail_idx = stream->oa_buffer.aged_tail_idx; - tail = stream->oa_buffer.tails[aged_tail_idx].offset; + tail = stream->oa_buffer.tail; spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); - /* An invalid tail pointer here means we're still waiting for the poll - * hrtimer callback to give us a pointer - */ - if (tail == INVALID_TAIL_PTR) - return -EAGAIN; - /* NB: oa_buffer.head/tail include the gtt_offset which we don't want * while indexing relative to oa_buf_base. */ @@ -1041,7 +998,8 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream, */ if (drm_WARN_ON(&uncore->i915->drm, (OA_BUFFER_SIZE - head) < report_size)) { - DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); + drm_err(&uncore->i915->drm, + "Spurious OA head ptr: non-integral report offset\n"); break; } @@ -1061,13 +1019,11 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream, if (ret) break; - /* The above report-id field sanity check is based on - * the assumption that the OA buffer is initially - * zeroed and we reset the field after copying so the - * check is still meaningful once old reports start - * being overwritten. + /* Clear out the first 2 dwords as a mean to detect unlanded + * reports. */ report32[0] = 0; + report32[1] = 0; } if (start_offset != *offset) { @@ -1307,8 +1263,7 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) * dropped by GuC. They won't be part of the context * ID in the OA reports, so squash those lower bits. */ - stream->specific_ctx_id = - lower_32_bits(ce->lrc_desc) >> 12; + stream->specific_ctx_id = ce->lrc.lrca >> 12; /* * GuC uses the top bit to signal proxy submission, so @@ -1325,11 +1280,10 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); /* * Pick an unused context id - * 0 - (NUM_CONTEXT_TAG - 1) are used by other contexts + * 0 - BITS_PER_LONG are used by other contexts * GEN12_MAX_CONTEXT_HW_ID (0x7ff) is used by idle context */ stream->specific_ctx_id = (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); - BUILD_BUG_ON((GEN12_MAX_CONTEXT_HW_ID - 1) < NUM_CONTEXT_TAG); break; } @@ -1339,9 +1293,10 @@ static int oa_get_render_ctx_id(struct i915_perf_stream *stream) ce->tag = stream->specific_ctx_id; - DRM_DEBUG_DRIVER("filtering on ctx_id=0x%x ctx_id_mask=0x%x\n", - stream->specific_ctx_id, - stream->specific_ctx_id_mask); + drm_dbg(&stream->perf->i915->drm, + "filtering on ctx_id=0x%x ctx_id_mask=0x%x\n", + stream->specific_ctx_id, + stream->specific_ctx_id_mask); return 0; } @@ -1401,8 +1356,10 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream) /* * Unset exclusive_stream first, it will be checked while disabling * the metric set on gen8+. + * + * See i915_oa_init_reg_state() and lrc_configure_all_contexts() */ - perf->exclusive_stream = NULL; + WRITE_ONCE(perf->exclusive_stream, NULL); perf->ops.disable_metric_set(stream); free_oa_buffer(stream); @@ -1443,8 +1400,8 @@ static void gen7_init_oa_buffer(struct i915_perf_stream *stream) gtt_offset | OABUFFER_SIZE_16M); /* Mark that we need updated tail pointers to read from... */ - stream->oa_buffer.tails[0].offset = INVALID_TAIL_PTR; - stream->oa_buffer.tails[1].offset = INVALID_TAIL_PTR; + stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; + stream->oa_buffer.tail = gtt_offset; spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); @@ -1466,8 +1423,6 @@ static void gen7_init_oa_buffer(struct i915_perf_stream *stream) * memory... */ memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); - - stream->pollin = false; } static void gen8_init_oa_buffer(struct i915_perf_stream *stream) @@ -1497,8 +1452,8 @@ static void gen8_init_oa_buffer(struct i915_perf_stream *stream) intel_uncore_write(uncore, GEN8_OATAILPTR, gtt_offset & GEN8_OATAILPTR_MASK); /* Mark that we need updated tail pointers to read from... */ - stream->oa_buffer.tails[0].offset = INVALID_TAIL_PTR; - stream->oa_buffer.tails[1].offset = INVALID_TAIL_PTR; + stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; + stream->oa_buffer.tail = gtt_offset; /* * Reset state used to recognise context switches, affecting which @@ -1522,8 +1477,6 @@ static void gen8_init_oa_buffer(struct i915_perf_stream *stream) * memory... */ memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); - - stream->pollin = false; } static void gen12_init_oa_buffer(struct i915_perf_stream *stream) @@ -1553,8 +1506,8 @@ static void gen12_init_oa_buffer(struct i915_perf_stream *stream) gtt_offset & GEN12_OAG_OATAILPTR_MASK); /* Mark that we need updated tail pointers to read from... */ - stream->oa_buffer.tails[0].offset = INVALID_TAIL_PTR; - stream->oa_buffer.tails[1].offset = INVALID_TAIL_PTR; + stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; + stream->oa_buffer.tail = gtt_offset; /* * Reset state used to recognise context switches, affecting which @@ -1579,8 +1532,6 @@ static void gen12_init_oa_buffer(struct i915_perf_stream *stream) */ memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.vma->size); - - stream->pollin = false; } static int alloc_oa_buffer(struct i915_perf_stream *stream) @@ -1966,9 +1917,11 @@ out: return i915_vma_get(oa_bo->vma); } -static int emit_oa_config(struct i915_perf_stream *stream, - struct i915_oa_config *oa_config, - struct intel_context *ce) +static int +emit_oa_config(struct i915_perf_stream *stream, + struct i915_oa_config *oa_config, + struct intel_context *ce, + struct i915_active *active) { struct i915_request *rq; struct i915_vma *vma; @@ -1990,6 +1943,18 @@ static int emit_oa_config(struct i915_perf_stream *stream, goto err_vma_unpin; } + if (!IS_ERR_OR_NULL(active)) { + /* After all individual context modifications */ + err = i915_request_await_active(rq, active, + I915_ACTIVE_AWAIT_ACTIVE); + if (err) + goto err_add_request; + + err = i915_active_add_request(active, rq); + if (err) + goto err_add_request; + } + i915_vma_lock(vma); err = i915_request_await_object(rq, vma->obj, 0); if (!err) @@ -2001,6 +1966,9 @@ static int emit_oa_config(struct i915_perf_stream *stream, err = rq->engine->emit_bb_start(rq, vma->node.start, 0, I915_DISPATCH_SECURE); + if (err) + goto err_add_request; + err_add_request: i915_request_add(rq); err_vma_unpin: @@ -2015,7 +1983,9 @@ static struct intel_context *oa_context(struct i915_perf_stream *stream) return stream->pinned_ctx ?: stream->engine->kernel_context; } -static int hsw_enable_metric_set(struct i915_perf_stream *stream) +static int +hsw_enable_metric_set(struct i915_perf_stream *stream, + struct i915_active *active) { struct intel_uncore *uncore = stream->uncore; @@ -2034,7 +2004,9 @@ static int hsw_enable_metric_set(struct i915_perf_stream *stream) intel_uncore_rmw(uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE); - return emit_oa_config(stream, stream->oa_config, oa_context(stream)); + return emit_oa_config(stream, + stream->oa_config, oa_context(stream), + active); } static void hsw_disable_metric_set(struct i915_perf_stream *stream) @@ -2104,9 +2076,6 @@ gen8_update_reg_state_unlocked(const struct intel_context *ce, for (i = 0; i < ARRAY_SIZE(flex_regs); i++) reg_state[ctx_flexeu0 + i * 2 + 1] = oa_config_flex_reg(stream->oa_config, flex_regs[i]); - - reg_state[CTX_R_PWR_CLK_STATE] = - intel_sseu_make_rpcs(ce->engine->i915, &ce->sseu); } struct flex { @@ -2127,7 +2096,7 @@ gen8_store_flex(struct i915_request *rq, if (IS_ERR(cs)) return PTR_ERR(cs); - offset = i915_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE; + offset = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET; do { *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *cs++ = offset + flex->offset * sizeof(u32); @@ -2184,18 +2153,31 @@ static int gen8_modify_context(struct intel_context *ce, return err; } -static int gen8_modify_self(struct intel_context *ce, - const struct flex *flex, unsigned int count) +static int +gen8_modify_self(struct intel_context *ce, + const struct flex *flex, unsigned int count, + struct i915_active *active) { struct i915_request *rq; int err; + intel_engine_pm_get(ce->engine); rq = i915_request_create(ce); + intel_engine_pm_put(ce->engine); if (IS_ERR(rq)) return PTR_ERR(rq); + if (!IS_ERR_OR_NULL(active)) { + err = i915_active_add_request(active, rq); + if (err) + goto err_add_request; + } + err = gen8_load_flex(rq, ce, flex, count); + if (err) + goto err_add_request; +err_add_request: i915_request_add(rq); return err; } @@ -2229,7 +2211,8 @@ static int gen8_configure_context(struct i915_gem_context *ctx, return err; } -static int gen12_configure_oar_context(struct i915_perf_stream *stream, bool enable) +static int gen12_configure_oar_context(struct i915_perf_stream *stream, + struct i915_active *active) { int err; struct intel_context *ce = stream->pinned_ctx; @@ -2238,7 +2221,7 @@ static int gen12_configure_oar_context(struct i915_perf_stream *stream, bool ena { GEN8_OACTXCONTROL, stream->perf->ctx_oactxctrl_offset + 1, - enable ? GEN8_OA_COUNTER_RESUME : 0, + active ? GEN8_OA_COUNTER_RESUME : 0, }, }; /* Offsets in regs_lri are not used since this configuration is only @@ -2250,13 +2233,13 @@ static int gen12_configure_oar_context(struct i915_perf_stream *stream, bool ena GEN12_OAR_OACONTROL, GEN12_OAR_OACONTROL_OFFSET + 1, (format << GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT) | - (enable ? GEN12_OAR_OACONTROL_COUNTER_ENABLE : 0) + (active ? GEN12_OAR_OACONTROL_COUNTER_ENABLE : 0) }, { RING_CONTEXT_CONTROL(ce->engine->mmio_base), CTX_CONTEXT_CONTROL, _MASKED_FIELD(GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE, - enable ? + active ? GEN12_CTX_CTRL_OAR_CONTEXT_ENABLE : 0) }, @@ -2273,7 +2256,7 @@ static int gen12_configure_oar_context(struct i915_perf_stream *stream, bool ena return err; /* Apply regs_lri using LRI with pinned context */ - return gen8_modify_self(ce, regs_lri, ARRAY_SIZE(regs_lri)); + return gen8_modify_self(ce, regs_lri, ARRAY_SIZE(regs_lri), active); } /* @@ -2301,9 +2284,11 @@ static int gen12_configure_oar_context(struct i915_perf_stream *stream, bool ena * Note: it's only the RCS/Render context that has any OA state. * Note: the first flex register passed must always be R_PWR_CLK_STATE */ -static int oa_configure_all_contexts(struct i915_perf_stream *stream, - struct flex *regs, - size_t num_regs) +static int +oa_configure_all_contexts(struct i915_perf_stream *stream, + struct flex *regs, + size_t num_regs, + struct i915_active *active) { struct drm_i915_private *i915 = stream->perf->i915; struct intel_engine_cs *engine; @@ -2360,7 +2345,7 @@ static int oa_configure_all_contexts(struct i915_perf_stream *stream, regs[0].value = intel_sseu_make_rpcs(i915, &ce->sseu); - err = gen8_modify_self(ce, regs, num_regs); + err = gen8_modify_self(ce, regs, num_regs, active); if (err) return err; } @@ -2368,8 +2353,10 @@ static int oa_configure_all_contexts(struct i915_perf_stream *stream, return 0; } -static int gen12_configure_all_contexts(struct i915_perf_stream *stream, - const struct i915_oa_config *oa_config) +static int +gen12_configure_all_contexts(struct i915_perf_stream *stream, + const struct i915_oa_config *oa_config, + struct i915_active *active) { struct flex regs[] = { { @@ -2378,11 +2365,15 @@ static int gen12_configure_all_contexts(struct i915_perf_stream *stream, }, }; - return oa_configure_all_contexts(stream, regs, ARRAY_SIZE(regs)); + return oa_configure_all_contexts(stream, + regs, ARRAY_SIZE(regs), + active); } -static int lrc_configure_all_contexts(struct i915_perf_stream *stream, - const struct i915_oa_config *oa_config) +static int +lrc_configure_all_contexts(struct i915_perf_stream *stream, + const struct i915_oa_config *oa_config, + struct i915_active *active) { /* The MMIO offsets for Flex EU registers aren't contiguous */ const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; @@ -2415,10 +2406,14 @@ static int lrc_configure_all_contexts(struct i915_perf_stream *stream, for (i = 2; i < ARRAY_SIZE(regs); i++) regs[i].value = oa_config_flex_reg(oa_config, regs[i].reg); - return oa_configure_all_contexts(stream, regs, ARRAY_SIZE(regs)); + return oa_configure_all_contexts(stream, + regs, ARRAY_SIZE(regs), + active); } -static int gen8_enable_metric_set(struct i915_perf_stream *stream) +static int +gen8_enable_metric_set(struct i915_perf_stream *stream, + struct i915_active *active) { struct intel_uncore *uncore = stream->uncore; struct i915_oa_config *oa_config = stream->oa_config; @@ -2458,11 +2453,13 @@ static int gen8_enable_metric_set(struct i915_perf_stream *stream) * to make sure all slices/subslices are ON before writing to NOA * registers. */ - ret = lrc_configure_all_contexts(stream, oa_config); + ret = lrc_configure_all_contexts(stream, oa_config, active); if (ret) return ret; - return emit_oa_config(stream, oa_config, oa_context(stream)); + return emit_oa_config(stream, + stream->oa_config, oa_context(stream), + active); } static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) @@ -2472,7 +2469,9 @@ static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) 0 : GEN12_OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS); } -static int gen12_enable_metric_set(struct i915_perf_stream *stream) +static int +gen12_enable_metric_set(struct i915_perf_stream *stream, + struct i915_active *active) { struct intel_uncore *uncore = stream->uncore; struct i915_oa_config *oa_config = stream->oa_config; @@ -2501,7 +2500,7 @@ static int gen12_enable_metric_set(struct i915_perf_stream *stream) * to make sure all slices/subslices are ON before writing to NOA * registers. */ - ret = gen12_configure_all_contexts(stream, oa_config); + ret = gen12_configure_all_contexts(stream, oa_config, active); if (ret) return ret; @@ -2511,12 +2510,14 @@ static int gen12_enable_metric_set(struct i915_perf_stream *stream) * requested this. */ if (stream->ctx) { - ret = gen12_configure_oar_context(stream, true); + ret = gen12_configure_oar_context(stream, active); if (ret) return ret; } - return emit_oa_config(stream, oa_config, oa_context(stream)); + return emit_oa_config(stream, + stream->oa_config, oa_context(stream), + active); } static void gen8_disable_metric_set(struct i915_perf_stream *stream) @@ -2524,7 +2525,7 @@ static void gen8_disable_metric_set(struct i915_perf_stream *stream) struct intel_uncore *uncore = stream->uncore; /* Reset all contexts' slices/subslices configurations. */ - lrc_configure_all_contexts(stream, NULL); + lrc_configure_all_contexts(stream, NULL, NULL); intel_uncore_rmw(uncore, GDT_CHICKEN_BITS, GT_NOA_ENABLE, 0); } @@ -2534,7 +2535,7 @@ static void gen10_disable_metric_set(struct i915_perf_stream *stream) struct intel_uncore *uncore = stream->uncore; /* Reset all contexts' slices/subslices configurations. */ - lrc_configure_all_contexts(stream, NULL); + lrc_configure_all_contexts(stream, NULL, NULL); /* Make sure we disable noa to save power. */ intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, 0); @@ -2545,11 +2546,11 @@ static void gen12_disable_metric_set(struct i915_perf_stream *stream) struct intel_uncore *uncore = stream->uncore; /* Reset all contexts' slices/subslices configurations. */ - gen12_configure_all_contexts(stream, NULL); + gen12_configure_all_contexts(stream, NULL, NULL); /* disable the context save/restore or OAR counters */ if (stream->ctx) - gen12_configure_oar_context(stream, false); + gen12_configure_oar_context(stream, NULL); /* Make sure we disable noa to save power. */ intel_uncore_rmw(uncore, RPM_CONFIG1, GEN10_GT_NOA_ENABLE, 0); @@ -2641,11 +2642,13 @@ static void gen12_oa_enable(struct i915_perf_stream *stream) */ static void i915_oa_stream_enable(struct i915_perf_stream *stream) { + stream->pollin = false; + stream->perf->ops.oa_enable(stream); if (stream->periodic) hrtimer_start(&stream->poll_check_timer, - ns_to_ktime(POLL_PERIOD), + ns_to_ktime(stream->poll_oa_period), HRTIMER_MODE_REL_PINNED); } @@ -2657,7 +2660,8 @@ static void gen7_oa_disable(struct i915_perf_stream *stream) if (intel_wait_for_register(uncore, GEN7_OACONTROL, GEN7_OACONTROL_ENABLE, 0, 50)) - DRM_ERROR("wait for OA to be disabled timed out\n"); + drm_err(&stream->perf->i915->drm, + "wait for OA to be disabled timed out\n"); } static void gen8_oa_disable(struct i915_perf_stream *stream) @@ -2668,7 +2672,8 @@ static void gen8_oa_disable(struct i915_perf_stream *stream) if (intel_wait_for_register(uncore, GEN8_OACONTROL, GEN8_OA_COUNTER_ENABLE, 0, 50)) - DRM_ERROR("wait for OA to be disabled timed out\n"); + drm_err(&stream->perf->i915->drm, + "wait for OA to be disabled timed out\n"); } static void gen12_oa_disable(struct i915_perf_stream *stream) @@ -2680,7 +2685,16 @@ static void gen12_oa_disable(struct i915_perf_stream *stream) GEN12_OAG_OACONTROL, GEN12_OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50)) - DRM_ERROR("wait for OA to be disabled timed out\n"); + drm_err(&stream->perf->i915->drm, + "wait for OA to be disabled timed out\n"); + + intel_uncore_write(uncore, GEN12_OA_TLB_INV_CR, 1); + if (intel_wait_for_register(uncore, + GEN12_OA_TLB_INV_CR, + 1, 0, + 50)) + drm_err(&stream->perf->i915->drm, + "wait for OA tlb invalidate timed out\n"); } /** @@ -2708,6 +2722,56 @@ static const struct i915_perf_stream_ops i915_oa_stream_ops = { .read = i915_oa_read, }; +static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream) +{ + struct i915_active *active; + int err; + + active = i915_active_create(); + if (!active) + return -ENOMEM; + + err = stream->perf->ops.enable_metric_set(stream, active); + if (err == 0) + __i915_active_wait(active, TASK_UNINTERRUPTIBLE); + + i915_active_put(active); + return err; +} + +static void +get_default_sseu_config(struct intel_sseu *out_sseu, + struct intel_engine_cs *engine) +{ + const struct sseu_dev_info *devinfo_sseu = + &RUNTIME_INFO(engine->i915)->sseu; + + *out_sseu = intel_sseu_from_device_info(devinfo_sseu); + + if (IS_GEN(engine->i915, 11)) { + /* + * We only need subslice count so it doesn't matter which ones + * we select - just turn off low bits in the amount of half of + * all available subslices per slice. + */ + out_sseu->subslice_mask = + ~(~0 << (hweight8(out_sseu->subslice_mask) / 2)); + out_sseu->slice_mask = 0x1; + } +} + +static int +get_sseu_config(struct intel_sseu *out_sseu, + struct intel_engine_cs *engine, + const struct drm_i915_gem_context_param_sseu *drm_sseu) +{ + if (drm_sseu->engine.engine_class != engine->uabi_class || + drm_sseu->engine.engine_instance != engine->uabi_instance) + return -EINVAL; + + return i915_gem_user_to_context_sseu(engine->i915, drm_sseu, out_sseu); +} + /** * i915_oa_stream_init - validate combined props for OA stream and init * @stream: An i915 perf stream @@ -2840,9 +2904,11 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream, goto err_oa_buf_alloc; stream->ops = &i915_oa_stream_ops; - perf->exclusive_stream = stream; - ret = perf->ops.enable_metric_set(stream); + perf->sseu = props->sseu; + WRITE_ONCE(perf->exclusive_stream, stream); + + ret = i915_perf_stream_enable_sync(stream); if (ret) { DRM_DEBUG("Unable to enable metric set\n"); goto err_enable; @@ -2860,7 +2926,7 @@ static int i915_oa_stream_init(struct i915_perf_stream *stream, return 0; err_enable: - perf->exclusive_stream = NULL; + WRITE_ONCE(perf->exclusive_stream, NULL); perf->ops.disable_metric_set(stream); free_oa_buffer(stream); @@ -2886,64 +2952,16 @@ void i915_oa_init_reg_state(const struct intel_context *ce, { struct i915_perf_stream *stream; - /* perf.exclusive_stream serialised by lrc_configure_all_contexts() */ - if (engine->class != RENDER_CLASS) return; - stream = engine->i915->perf.exclusive_stream; - /* - * For gen12, only CTX_R_PWR_CLK_STATE needs update, but the caller - * is already doing that, so nothing to be done for gen12 here. - */ + /* perf.exclusive_stream serialised by lrc_configure_all_contexts() */ + stream = READ_ONCE(engine->i915->perf.exclusive_stream); if (stream && INTEL_GEN(stream->perf->i915) < 12) gen8_update_reg_state_unlocked(ce, stream); } /** - * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation - * @stream: An i915 perf stream - * @file: An i915 perf stream file - * @buf: destination buffer given by userspace - * @count: the number of bytes userspace wants to read - * @ppos: (inout) file seek position (unused) - * - * Besides wrapping &i915_perf_stream_ops->read this provides a common place to - * ensure that if we've successfully copied any data then reporting that takes - * precedence over any internal error status, so the data isn't lost. - * - * For example ret will be -ENOSPC whenever there is more buffered data than - * can be copied to userspace, but that's only interesting if we weren't able - * to copy some data because it implies the userspace buffer is too small to - * receive a single record (and we never split records). - * - * Another case with ret == -EFAULT is more of a grey area since it would seem - * like bad form for userspace to ask us to overrun its buffer, but the user - * knows best: - * - * http://yarchive.net/comp/linux/partial_reads_writes.html - * - * Returns: The number of bytes copied or a negative error code on failure. - */ -static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, - struct file *file, - char __user *buf, - size_t count, - loff_t *ppos) -{ - /* Note we keep the offset (aka bytes read) separate from any - * error status so that the final check for whether we return - * the bytes read with a higher precedence than any error (see - * comment below) doesn't need to be handled/duplicated in - * stream->ops->read() implementations. - */ - size_t offset = 0; - int ret = stream->ops->read(stream, buf, count, &offset); - - return offset ?: (ret ?: -EAGAIN); -} - -/** * i915_perf_read - handles read() FOP for i915 perf stream FDs * @file: An i915 perf stream file * @buf: destination buffer given by userspace @@ -2968,7 +2986,8 @@ static ssize_t i915_perf_read(struct file *file, { struct i915_perf_stream *stream = file->private_data; struct i915_perf *perf = stream->perf; - ssize_t ret; + size_t offset = 0; + int ret; /* To ensure it's handled consistently we simply treat all reads of a * disabled stream as an error. In particular it might otherwise lead @@ -2991,13 +3010,12 @@ static ssize_t i915_perf_read(struct file *file, return ret; mutex_lock(&perf->lock); - ret = i915_perf_read_locked(stream, file, - buf, count, ppos); + ret = stream->ops->read(stream, buf, count, &offset); mutex_unlock(&perf->lock); - } while (ret == -EAGAIN); + } while (!offset && !ret); } else { mutex_lock(&perf->lock); - ret = i915_perf_read_locked(stream, file, buf, count, ppos); + ret = stream->ops->read(stream, buf, count, &offset); mutex_unlock(&perf->lock); } @@ -3008,15 +3026,15 @@ static ssize_t i915_perf_read(struct file *file, * and read() returning -EAGAIN. Clearing the oa.pollin state here * effectively ensures we back off until the next hrtimer callback * before reporting another EPOLLIN event. + * The exception to this is if ops->read() returned -ENOSPC which means + * that more OA data is available than could fit in the user provided + * buffer. In this case we want the next poll() call to not block. */ - if (ret >= 0 || ret == -EAGAIN) { - /* Maybe make ->pollin per-stream state if we support multiple - * concurrent streams in the future. - */ + if (ret != -ENOSPC) stream->pollin = false; - } - return ret; + /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, ... */ + return offset ?: (ret ?: -EAGAIN); } static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) @@ -3029,7 +3047,8 @@ static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) wake_up(&stream->poll_wq); } - hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); + hrtimer_forward_now(hrtimer, + ns_to_ktime(stream->poll_oa_period)); return HRTIMER_RESTART; } @@ -3171,8 +3190,8 @@ static long i915_perf_config_locked(struct i915_perf_stream *stream, * When set globally, we use a low priority kernel context, * so it will effectively take effect when idle. */ - err = emit_oa_config(stream, config, oa_context(stream)); - if (err == 0) + err = emit_oa_config(stream, config, oa_context(stream), NULL); + if (!err) config = xchg(&stream->oa_config, config); else ret = err; @@ -3388,6 +3407,14 @@ i915_perf_open_ioctl_locked(struct i915_perf *perf, privileged_op = true; } + /* + * Asking for SSEU configuration is a priviliged operation. + */ + if (props->has_sseu) + privileged_op = true; + else + get_default_sseu_config(&props->sseu, props->engine); + /* Similar to perf's kernel.perf_paranoid_cpu sysctl option * we check a dev.i915.perf_stream_paranoid sysctl option * to determine if it's ok to access system wide OA counters @@ -3408,6 +3435,7 @@ i915_perf_open_ioctl_locked(struct i915_perf *perf, stream->perf = perf; stream->ctx = specific_ctx; + stream->poll_oa_period = props->poll_oa_period; ret = i915_oa_stream_init(stream, param, props); if (ret) @@ -3483,8 +3511,10 @@ static int read_properties_unlocked(struct i915_perf *perf, { u64 __user *uprop = uprops; u32 i; + int ret; memset(props, 0, sizeof(struct perf_open_properties)); + props->poll_oa_period = DEFAULT_POLL_PERIOD_NS; if (!n_props) { DRM_DEBUG("No i915 perf properties given\n"); @@ -3514,7 +3544,6 @@ static int read_properties_unlocked(struct i915_perf *perf, for (i = 0; i < n_props; i++) { u64 oa_period, oa_freq_hz; u64 id, value; - int ret; ret = get_user(id, uprop); if (ret) @@ -3600,6 +3629,32 @@ static int read_properties_unlocked(struct i915_perf *perf, case DRM_I915_PERF_PROP_HOLD_PREEMPTION: props->hold_preemption = !!value; break; + case DRM_I915_PERF_PROP_GLOBAL_SSEU: { + struct drm_i915_gem_context_param_sseu user_sseu; + + if (copy_from_user(&user_sseu, + u64_to_user_ptr(value), + sizeof(user_sseu))) { + DRM_DEBUG("Unable to copy global sseu parameter\n"); + return -EFAULT; + } + + ret = get_sseu_config(&props->sseu, props->engine, &user_sseu); + if (ret) { + DRM_DEBUG("Invalid SSEU configuration\n"); + return ret; + } + props->has_sseu = true; + break; + } + case DRM_I915_PERF_PROP_POLL_OA_PERIOD: + if (value < 100000 /* 100us */) { + DRM_DEBUG("OA availability timer too small (%lluns < 100us)\n", + value); + return -EINVAL; + } + props->poll_oa_period = value; + break; case DRM_I915_PERF_PROP_MAX: MISSING_CASE(id); return -EINVAL; @@ -3682,7 +3737,6 @@ int i915_perf_open_ioctl(struct drm_device *dev, void *data, void i915_perf_register(struct drm_i915_private *i915) { struct i915_perf *perf = &i915->perf; - int ret; if (!perf->i915) return; @@ -3696,64 +3750,7 @@ void i915_perf_register(struct drm_i915_private *i915) perf->metrics_kobj = kobject_create_and_add("metrics", &i915->drm.primary->kdev->kobj); - if (!perf->metrics_kobj) - goto exit; - - sysfs_attr_init(&perf->test_config.sysfs_metric_id.attr); - - if (IS_TIGERLAKE(i915)) { - i915_perf_load_test_config_tgl(i915); - } else if (INTEL_GEN(i915) >= 11) { - i915_perf_load_test_config_icl(i915); - } else if (IS_CANNONLAKE(i915)) { - i915_perf_load_test_config_cnl(i915); - } else if (IS_COFFEELAKE(i915)) { - if (IS_CFL_GT2(i915)) - i915_perf_load_test_config_cflgt2(i915); - if (IS_CFL_GT3(i915)) - i915_perf_load_test_config_cflgt3(i915); - } else if (IS_GEMINILAKE(i915)) { - i915_perf_load_test_config_glk(i915); - } else if (IS_KABYLAKE(i915)) { - if (IS_KBL_GT2(i915)) - i915_perf_load_test_config_kblgt2(i915); - else if (IS_KBL_GT3(i915)) - i915_perf_load_test_config_kblgt3(i915); - } else if (IS_BROXTON(i915)) { - i915_perf_load_test_config_bxt(i915); - } else if (IS_SKYLAKE(i915)) { - if (IS_SKL_GT2(i915)) - i915_perf_load_test_config_sklgt2(i915); - else if (IS_SKL_GT3(i915)) - i915_perf_load_test_config_sklgt3(i915); - else if (IS_SKL_GT4(i915)) - i915_perf_load_test_config_sklgt4(i915); - } else if (IS_CHERRYVIEW(i915)) { - i915_perf_load_test_config_chv(i915); - } else if (IS_BROADWELL(i915)) { - i915_perf_load_test_config_bdw(i915); - } else if (IS_HASWELL(i915)) { - i915_perf_load_test_config_hsw(i915); - } - - if (perf->test_config.id == 0) - goto sysfs_error; - - ret = sysfs_create_group(perf->metrics_kobj, - &perf->test_config.sysfs_metric); - if (ret) - goto sysfs_error; - perf->test_config.perf = perf; - kref_init(&perf->test_config.ref); - - goto exit; - -sysfs_error: - kobject_put(perf->metrics_kobj); - perf->metrics_kobj = NULL; - -exit: mutex_unlock(&perf->lock); } @@ -3773,9 +3770,6 @@ void i915_perf_unregister(struct drm_i915_private *i915) if (!perf->metrics_kobj) return; - sysfs_remove_group(perf->metrics_kobj, - &perf->test_config.sysfs_metric); - kobject_put(perf->metrics_kobj); perf->metrics_kobj = NULL; } @@ -4374,6 +4368,11 @@ void i915_perf_init(struct drm_i915_private *i915) ratelimit_set_flags(&perf->spurious_report_rs, RATELIMIT_MSG_ON_RELEASE); + ratelimit_state_init(&perf->tail_pointer_race, + 5 * HZ, 10); + ratelimit_set_flags(&perf->tail_pointer_race, + RATELIMIT_MSG_ON_RELEASE); + atomic64_set(&perf->noa_programming_delay, 500 * 1000 /* 500us */); @@ -4434,8 +4433,15 @@ int i915_perf_ioctl_version(void) * preemption on a particular context so that performance data is * accessible from a delta of MI_RPC reports without looking at the * OA buffer. + * + * 4: Add DRM_I915_PERF_PROP_ALLOWED_SSEU to limit what contexts can + * be run for the duration of the performance recording based on + * their SSEU configuration. + * + * 5: Add DRM_I915_PERF_PROP_POLL_OA_PERIOD parameter that controls the + * interval for the hrtimer used to check for OA data. */ - return 3; + return 5; } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) diff --git a/drivers/gpu/drm/i915/i915_perf_types.h b/drivers/gpu/drm/i915/i915_perf_types.h index 45e581455f5d..a36a455ae336 100644 --- a/drivers/gpu/drm/i915/i915_perf_types.h +++ b/drivers/gpu/drm/i915/i915_perf_types.h @@ -16,11 +16,13 @@ #include <linux/uuid.h> #include <linux/wait.h> +#include "gt/intel_sseu.h" #include "i915_reg.h" #include "intel_wakeref.h" struct drm_i915_private; struct file; +struct i915_active; struct i915_gem_context; struct i915_perf; struct i915_vma; @@ -272,21 +274,10 @@ struct i915_perf_stream { spinlock_t ptr_lock; /** - * @tails: One 'aging' tail pointer and one 'aged' tail pointer ready to - * used for reading. - * - * Initial values of 0xffffffff are invalid and imply that an - * update is required (and should be ignored by an attempted - * read) - */ - struct { - u32 offset; - } tails[2]; - - /** - * @aged_tail_idx: Index for the aged tail ready to read() data up to. + * @aging_tail: The last HW tail reported by HW. The data + * might not have made it to memory yet though. */ - unsigned int aged_tail_idx; + u32 aging_tail; /** * @aging_timestamp: A monotonic timestamp for when the current aging tail pointer @@ -302,6 +293,11 @@ struct i915_perf_stream { * OA buffer data to userspace. */ u32 head; + + /** + * @tail: The last verified tail that can be read by userspace. + */ + u32 tail; } oa_buffer; /** @@ -309,6 +305,12 @@ struct i915_perf_stream { * reprogrammed. */ struct i915_vma *noa_wait; + + /** + * @poll_oa_period: The period in nanoseconds at which the OA + * buffer should be checked for available data. + */ + u64 poll_oa_period; }; /** @@ -339,7 +341,8 @@ struct i915_oa_ops { * counter reports being sampled. May apply system constraints such as * disabling EU clock gating as required. */ - int (*enable_metric_set)(struct i915_perf_stream *stream); + int (*enable_metric_set)(struct i915_perf_stream *stream, + struct i915_active *active); /** * @disable_metric_set: Remove system constraints associated with using @@ -407,12 +410,22 @@ struct i915_perf { struct i915_perf_stream *exclusive_stream; /** + * @sseu: sseu configuration selected to run while perf is active, + * applies to all contexts. + */ + struct intel_sseu sseu; + + /** * For rate limiting any notifications of spurious * invalid OA reports */ struct ratelimit_state spurious_report_rs; - struct i915_oa_config test_config; + /** + * For rate limiting any notifications of tail pointer + * race. + */ + struct ratelimit_state tail_pointer_race; u32 gen7_latched_oastatus1; u32 ctx_oactxctrl_offset; diff --git a/drivers/gpu/drm/i915/i915_pmu.c b/drivers/gpu/drm/i915/i915_pmu.c index a3b61fb96226..e991a707bdb7 100644 --- a/drivers/gpu/drm/i915/i915_pmu.c +++ b/drivers/gpu/drm/i915/i915_pmu.c @@ -439,29 +439,10 @@ static u64 count_interrupts(struct drm_i915_private *i915) return sum; } -static void engine_event_destroy(struct perf_event *event) -{ - struct drm_i915_private *i915 = - container_of(event->pmu, typeof(*i915), pmu.base); - struct intel_engine_cs *engine; - - engine = intel_engine_lookup_user(i915, - engine_event_class(event), - engine_event_instance(event)); - if (drm_WARN_ON_ONCE(&i915->drm, !engine)) - return; - - if (engine_event_sample(event) == I915_SAMPLE_BUSY && - intel_engine_supports_stats(engine)) - intel_disable_engine_stats(engine); -} - static void i915_pmu_event_destroy(struct perf_event *event) { WARN_ON(event->parent); - - if (is_engine_event(event)) - engine_event_destroy(event); + module_put(THIS_MODULE); } static int @@ -514,23 +495,13 @@ static int engine_event_init(struct perf_event *event) struct drm_i915_private *i915 = container_of(event->pmu, typeof(*i915), pmu.base); struct intel_engine_cs *engine; - u8 sample; - int ret; engine = intel_engine_lookup_user(i915, engine_event_class(event), engine_event_instance(event)); if (!engine) return -ENODEV; - sample = engine_event_sample(event); - ret = engine_event_status(engine, sample); - if (ret) - return ret; - - if (sample == I915_SAMPLE_BUSY && intel_engine_supports_stats(engine)) - ret = intel_enable_engine_stats(engine); - - return ret; + return engine_event_status(engine, engine_event_sample(event)); } static int i915_pmu_event_init(struct perf_event *event) @@ -563,8 +534,10 @@ static int i915_pmu_event_init(struct perf_event *event) if (ret) return ret; - if (!event->parent) + if (!event->parent) { + __module_get(THIS_MODULE); event->destroy = i915_pmu_event_destroy; + } return 0; } @@ -822,11 +795,6 @@ static ssize_t i915_pmu_event_show(struct device *dev, return sprintf(buf, "config=0x%lx\n", eattr->val); } -static struct attribute_group i915_pmu_events_attr_group = { - .name = "events", - /* Patch in attrs at runtime. */ -}; - static ssize_t i915_pmu_get_attr_cpumask(struct device *dev, struct device_attribute *attr, @@ -846,13 +814,6 @@ static const struct attribute_group i915_pmu_cpumask_attr_group = { .attrs = i915_cpumask_attrs, }; -static const struct attribute_group *i915_pmu_attr_groups[] = { - &i915_pmu_format_attr_group, - &i915_pmu_events_attr_group, - &i915_pmu_cpumask_attr_group, - NULL -}; - #define __event(__config, __name, __unit) \ { \ .config = (__config), \ @@ -1026,23 +987,23 @@ err_alloc: static void free_event_attributes(struct i915_pmu *pmu) { - struct attribute **attr_iter = i915_pmu_events_attr_group.attrs; + struct attribute **attr_iter = pmu->events_attr_group.attrs; for (; *attr_iter; attr_iter++) kfree((*attr_iter)->name); - kfree(i915_pmu_events_attr_group.attrs); + kfree(pmu->events_attr_group.attrs); kfree(pmu->i915_attr); kfree(pmu->pmu_attr); - i915_pmu_events_attr_group.attrs = NULL; + pmu->events_attr_group.attrs = NULL; pmu->i915_attr = NULL; pmu->pmu_attr = NULL; } static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) { - struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node); + struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); GEM_BUG_ON(!pmu->base.event_init); @@ -1055,7 +1016,7 @@ static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) { - struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node); + struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); unsigned int target; GEM_BUG_ON(!pmu->base.event_init); @@ -1072,8 +1033,6 @@ static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) return 0; } -static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; - static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) { enum cpuhp_state slot; @@ -1087,21 +1046,22 @@ static int i915_pmu_register_cpuhp_state(struct i915_pmu *pmu) return ret; slot = ret; - ret = cpuhp_state_add_instance(slot, &pmu->node); + ret = cpuhp_state_add_instance(slot, &pmu->cpuhp.node); if (ret) { cpuhp_remove_multi_state(slot); return ret; } - cpuhp_slot = slot; + pmu->cpuhp.slot = slot; return 0; } static void i915_pmu_unregister_cpuhp_state(struct i915_pmu *pmu) { - WARN_ON(cpuhp_slot == CPUHP_INVALID); - WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &pmu->node)); - cpuhp_remove_multi_state(cpuhp_slot); + WARN_ON(pmu->cpuhp.slot == CPUHP_INVALID); + WARN_ON(cpuhp_state_remove_instance(pmu->cpuhp.slot, &pmu->cpuhp.node)); + cpuhp_remove_multi_state(pmu->cpuhp.slot); + pmu->cpuhp.slot = CPUHP_INVALID; } static bool is_igp(struct drm_i915_private *i915) @@ -1118,16 +1078,24 @@ static bool is_igp(struct drm_i915_private *i915) void i915_pmu_register(struct drm_i915_private *i915) { struct i915_pmu *pmu = &i915->pmu; + const struct attribute_group *attr_groups[] = { + &i915_pmu_format_attr_group, + &pmu->events_attr_group, + &i915_pmu_cpumask_attr_group, + NULL + }; + int ret = -ENOMEM; if (INTEL_GEN(i915) <= 2) { - dev_info(i915->drm.dev, "PMU not supported for this GPU."); + drm_info(&i915->drm, "PMU not supported for this GPU."); return; } spin_lock_init(&pmu->lock); hrtimer_init(&pmu->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); pmu->timer.function = i915_sample; + pmu->cpuhp.slot = CPUHP_INVALID; if (!is_igp(i915)) { pmu->name = kasprintf(GFP_KERNEL, @@ -1143,11 +1111,16 @@ void i915_pmu_register(struct drm_i915_private *i915) if (!pmu->name) goto err; - i915_pmu_events_attr_group.attrs = create_event_attributes(pmu); - if (!i915_pmu_events_attr_group.attrs) + pmu->events_attr_group.name = "events"; + pmu->events_attr_group.attrs = create_event_attributes(pmu); + if (!pmu->events_attr_group.attrs) goto err_name; - pmu->base.attr_groups = i915_pmu_attr_groups; + pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), + GFP_KERNEL); + if (!pmu->base.attr_groups) + goto err_attr; + pmu->base.task_ctx_nr = perf_invalid_context; pmu->base.event_init = i915_pmu_event_init; pmu->base.add = i915_pmu_event_add; @@ -1159,7 +1132,7 @@ void i915_pmu_register(struct drm_i915_private *i915) ret = perf_pmu_register(&pmu->base, pmu->name, -1); if (ret) - goto err_attr; + goto err_groups; ret = i915_pmu_register_cpuhp_state(pmu); if (ret) @@ -1169,6 +1142,8 @@ void i915_pmu_register(struct drm_i915_private *i915) err_unreg: perf_pmu_unregister(&pmu->base); +err_groups: + kfree(pmu->base.attr_groups); err_attr: pmu->base.event_init = NULL; free_event_attributes(pmu); @@ -1176,7 +1151,7 @@ err_name: if (!is_igp(i915)) kfree(pmu->name); err: - dev_notice(i915->drm.dev, "Failed to register PMU!\n"); + drm_notice(&i915->drm, "Failed to register PMU!\n"); } void i915_pmu_unregister(struct drm_i915_private *i915) @@ -1194,6 +1169,7 @@ void i915_pmu_unregister(struct drm_i915_private *i915) perf_pmu_unregister(&pmu->base); pmu->base.event_init = NULL; + kfree(pmu->base.attr_groups); if (!is_igp(i915)) kfree(pmu->name); free_event_attributes(pmu); diff --git a/drivers/gpu/drm/i915/i915_pmu.h b/drivers/gpu/drm/i915/i915_pmu.h index 6c1647c5daf2..941f0c14037c 100644 --- a/drivers/gpu/drm/i915/i915_pmu.h +++ b/drivers/gpu/drm/i915/i915_pmu.h @@ -10,7 +10,7 @@ #include <linux/hrtimer.h> #include <linux/perf_event.h> #include <linux/spinlock_types.h> -#include <drm/i915_drm.h> +#include <uapi/drm/i915_drm.h> struct drm_i915_private; @@ -39,9 +39,12 @@ struct i915_pmu_sample { struct i915_pmu { /** - * @node: List node for CPU hotplug handling. + * @cpuhp: Struct used for CPU hotplug handling. */ - struct hlist_node node; + struct { + struct hlist_node node; + enum cpuhp_state slot; + } cpuhp; /** * @base: PMU base. */ @@ -105,6 +108,10 @@ struct i915_pmu { */ ktime_t sleep_last; /** + * @events_attr_group: Device events attribute group. + */ + struct attribute_group events_attr_group; + /** * @i915_attr: Memory block holding device attributes. */ void *i915_attr; diff --git a/drivers/gpu/drm/i915/i915_priolist_types.h b/drivers/gpu/drm/i915/i915_priolist_types.h index 732aad148881..5003a71113cb 100644 --- a/drivers/gpu/drm/i915/i915_priolist_types.h +++ b/drivers/gpu/drm/i915/i915_priolist_types.h @@ -24,15 +24,12 @@ enum { I915_PRIORITY_DISPLAY, }; -#define I915_USER_PRIORITY_SHIFT 2 +#define I915_USER_PRIORITY_SHIFT 0 #define I915_USER_PRIORITY(x) ((x) << I915_USER_PRIORITY_SHIFT) #define I915_PRIORITY_COUNT BIT(I915_USER_PRIORITY_SHIFT) #define I915_PRIORITY_MASK (I915_PRIORITY_COUNT - 1) -#define I915_PRIORITY_WAIT ((u8)BIT(0)) -#define I915_PRIORITY_NOSEMAPHORE ((u8)BIT(1)) - /* Smallest priority value that cannot be bumped. */ #define I915_PRIORITY_INVALID (INT_MIN | (u8)I915_PRIORITY_MASK) @@ -47,8 +44,6 @@ enum { #define I915_PRIORITY_UNPREEMPTABLE INT_MAX #define I915_PRIORITY_BARRIER INT_MAX -#define __NO_PREEMPTION (I915_PRIORITY_WAIT) - struct i915_priolist { struct list_head requests[I915_PRIORITY_COUNT]; struct rb_node node; diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index b09c1d6dc0aa..6c076a24eb82 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -34,8 +34,8 @@ * Follow the style described here for new macros, and while changing existing * macros. Do **not** mass change existing definitions just to update the style. * - * Layout - * ~~~~~~ + * File Layout + * ~~~~~~~~~~~ * * Keep helper macros near the top. For example, _PIPE() and friends. * @@ -561,6 +561,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) * Registers used only by the command parser */ #define BCS_SWCTRL _MMIO(0x22200) +#define BCS_SRC_Y REG_BIT(0) +#define BCS_DST_Y REG_BIT(1) /* There are 16 GPR registers */ #define BCS_GPR(n) _MMIO(0x22600 + (n) * 8) @@ -693,6 +695,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define OABUFFER_SIZE_8M (6 << 3) #define OABUFFER_SIZE_16M (7 << 3) +#define GEN12_OA_TLB_INV_CR _MMIO(0xceec) + /* Gen12 OAR unit */ #define GEN12_OAR_OACONTROL _MMIO(0x2960) #define GEN12_OAR_OACONTROL_COUNTER_FORMAT_SHIFT 1 @@ -2553,6 +2557,14 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN10_PAT_INDEX(index) _MMIO(0x40e0 + (index) * 4) #define GEN12_PAT_INDEX(index) _MMIO(0x4800 + (index) * 4) #define BSD_HWS_PGA_GEN7 _MMIO(0x04180) +#define GEN12_GFX_CCS_AUX_NV _MMIO(0x4208) +#define GEN12_VD0_AUX_NV _MMIO(0x4218) +#define GEN12_VD1_AUX_NV _MMIO(0x4228) +#define GEN12_VD2_AUX_NV _MMIO(0x4298) +#define GEN12_VD3_AUX_NV _MMIO(0x42A8) +#define GEN12_VE0_AUX_NV _MMIO(0x4238) +#define GEN12_VE1_AUX_NV _MMIO(0x42B8) +#define AUX_INV REG_BIT(0) #define BLT_HWS_PGA_GEN7 _MMIO(0x04280) #define VEBOX_HWS_PGA_GEN7 _MMIO(0x04380) #define RING_ACTHD(base) _MMIO((base) + 0x74) @@ -2655,6 +2667,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define RING_DMA_FADD_UDW(base) _MMIO((base) + 0x60) /* gen8+ */ #define RING_INSTPM(base) _MMIO((base) + 0xc0) #define RING_MI_MODE(base) _MMIO((base) + 0x9c) +#define RING_CMD_BUF_CCTL(base) _MMIO((base) + 0x84) #define INSTPS _MMIO(0x2070) /* 965+ only */ #define GEN4_INSTDONE1 _MMIO(0x207c) /* 965+ only, aka INSTDONE_2 on SNB */ #define ACTHD_I965 _MMIO(0x2074) @@ -2865,6 +2878,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define MI_ARB_STATE _MMIO(0x20e4) /* 915+ only */ #define MBUS_ABOX_CTL _MMIO(0x45038) +#define MBUS_ABOX1_CTL _MMIO(0x45048) +#define MBUS_ABOX2_CTL _MMIO(0x4504C) #define MBUS_ABOX_BW_CREDIT_MASK (3 << 20) #define MBUS_ABOX_BW_CREDIT(x) ((x) << 20) #define MBUS_ABOX_B_CREDIT_MASK (0xF << 16) @@ -3090,6 +3105,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GT_BSD_CS_ERROR_INTERRUPT (1 << 15) #define GT_BSD_USER_INTERRUPT (1 << 12) #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1 (1 << 11) /* hsw+; rsvd on snb, ivb, vlv */ +#define GT_WAIT_SEMAPHORE_INTERRUPT REG_BIT(11) /* bdw+ */ #define GT_CONTEXT_SWITCH_INTERRUPT (1 << 8) #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT (1 << 5) /* !snb */ #define GT_RENDER_PIPECTL_NOTIFY_INTERRUPT (1 << 4) @@ -3283,6 +3299,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) /* Framebuffer compression for Ivybridge */ #define IVB_FBC_RT_BASE _MMIO(0x7020) +#define IVB_FBC_RT_BASE_UPPER _MMIO(0x7024) #define IPS_CTL _MMIO(0x43408) #define IPS_ENABLE (1 << 31) @@ -4007,31 +4024,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) #define GEN6_RP_STATE_LIMITS _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994) #define GEN6_RP_STATE_CAP _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998) #define BXT_RP_STATE_CAP _MMIO(0x138170) - -/* - * Make these a multiple of magic 25 to avoid SNB (eg. Dell XPS - * 8300) freezing up around GPU hangs. Looks as if even - * scheduling/timer interrupts start misbehaving if the RPS - * EI/thresholds are "bad", leading to a very sluggish or even - * frozen machine. - */ -#define INTERVAL_1_28_US(us) roundup(((us) * 100) >> 7, 25) -#define INTERVAL_1_33_US(us) (((us) * 3) >> 2) -#define INTERVAL_0_833_US(us) (((us) * 6) / 5) -#define GT_INTERVAL_FROM_US(dev_priv, us) (INTEL_GEN(dev_priv) >= 9 ? \ - (IS_GEN9_LP(dev_priv) ? \ - INTERVAL_0_833_US(us) : \ - INTERVAL_1_33_US(us)) : \ - INTERVAL_1_28_US(us)) - -#define INTERVAL_1_28_TO_US(interval) (((interval) << 7) / 100) -#define INTERVAL_1_33_TO_US(interval) (((interval) << 2) / 3) -#define INTERVAL_0_833_TO_US(interval) (((interval) * 5) / 6) -#define GT_PM_INTERVAL_TO_US(dev_priv, interval) (INTEL_GEN(dev_priv) >= 9 ? \ - (IS_GEN9_LP(dev_priv) ? \ - INTERVAL_0_833_TO_US(interval) : \ - INTERVAL_1_33_TO_US(interval)) : \ - INTERVAL_1_28_TO_US(interval)) +#define GEN9_RP_STATE_LIMITS _MMIO(0x138148) /* * Logical Context regs @@ -4319,6 +4312,96 @@ enum { #define EXITLINE_MASK REG_GENMASK(12, 0) #define EXITLINE_SHIFT 0 +/* VRR registers */ +#define _TRANS_VRR_CTL_A 0x60420 +#define _TRANS_VRR_CTL_B 0x61420 +#define _TRANS_VRR_CTL_C 0x62420 +#define _TRANS_VRR_CTL_D 0x63420 +#define TRANS_VRR_CTL(trans) _MMIO_TRANS2(trans, _TRANS_VRR_CTL_A) +#define VRR_CTL_VRR_ENABLE REG_BIT(31) +#define VRR_CTL_IGN_MAX_SHIFT REG_BIT(30) +#define VRR_CTL_FLIP_LINE_EN REG_BIT(29) +#define VRR_CTL_LINE_COUNT_MASK REG_GENMASK(10, 3) +#define VRR_CTL_SW_FULLLINE_COUNT REG_BIT(0) + +#define _TRANS_VRR_VMAX_A 0x60424 +#define _TRANS_VRR_VMAX_B 0x61424 +#define _TRANS_VRR_VMAX_C 0x62424 +#define _TRANS_VRR_VMAX_D 0x63424 +#define TRANS_VRR_VMAX(trans) _MMIO_TRANS2(trans, _TRANS_VRR_VMAX_A) +#define VRR_VMAX_MASK REG_GENMASK(19, 0) + +#define _TRANS_VRR_VMIN_A 0x60434 +#define _TRANS_VRR_VMIN_B 0x61434 +#define _TRANS_VRR_VMIN_C 0x62434 +#define _TRANS_VRR_VMIN_D 0x63434 +#define TRANS_VRR_VMIN(trans) _MMIO_TRANS2(trans, _TRANS_VRR_VMIN_A) +#define VRR_VMIN_MASK REG_GENMASK(15, 0) + +#define _TRANS_VRR_VMAXSHIFT_A 0x60428 +#define _TRANS_VRR_VMAXSHIFT_B 0x61428 +#define _TRANS_VRR_VMAXSHIFT_C 0x62428 +#define _TRANS_VRR_VMAXSHIFT_D 0x63428 +#define TRANS_VRR_VMAXSHIFT(trans) _MMIO_TRANS2(trans, \ + _TRANS_VRR_VMAXSHIFT_A) +#define VRR_VMAXSHIFT_DEC_MASK REG_GENMASK(29, 16) +#define VRR_VMAXSHIFT_DEC REG_BIT(16) +#define VRR_VMAXSHIFT_INC_MASK REG_GENMASK(12, 0) + +#define _TRANS_VRR_STATUS_A 0x6042C +#define _TRANS_VRR_STATUS_B 0x6142C +#define _TRANS_VRR_STATUS_C 0x6242C +#define _TRANS_VRR_STATUS_D 0x6342C +#define TRANS_VRR_STATUS(trans) _MMIO_TRANS2(trans, _TRANS_VRR_STATUS_A) +#define VRR_STATUS_VMAX_REACHED REG_BIT(31) +#define VRR_STATUS_NOFLIP_TILL_BNDR REG_BIT(30) +#define VRR_STATUS_FLIP_BEF_BNDR REG_BIT(29) +#define VRR_STATUS_NO_FLIP_FRAME REG_BIT(28) +#define VRR_STATUS_VRR_EN_LIVE REG_BIT(27) +#define VRR_STATUS_FLIPS_SERVICED REG_BIT(26) +#define VRR_STATUS_VBLANK_MASK REG_GENMASK(22, 20) +#define STATUS_FSM_IDLE REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 0) +#define STATUS_FSM_WAIT_TILL_FDB REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 1) +#define STATUS_FSM_WAIT_TILL_FS REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 2) +#define STATUS_FSM_WAIT_TILL_FLIP REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 3) +#define STATUS_FSM_PIPELINE_FILL REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 4) +#define STATUS_FSM_ACTIVE REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 5) +#define STATUS_FSM_LEGACY_VBLANK REG_FIELD_PREP(VRR_STATUS_VBLANK_MASK, 6) + +#define _TRANS_VRR_VTOTAL_PREV_A 0x60480 +#define _TRANS_VRR_VTOTAL_PREV_B 0x61480 +#define _TRANS_VRR_VTOTAL_PREV_C 0x62480 +#define _TRANS_VRR_VTOTAL_PREV_D 0x63480 +#define TRANS_VRR_VTOTAL_PREV(trans) _MMIO_TRANS2(trans, \ + _TRANS_VRR_VTOTAL_PREV_A) +#define VRR_VTOTAL_FLIP_BEFR_BNDR REG_BIT(31) +#define VRR_VTOTAL_FLIP_AFTER_BNDR REG_BIT(30) +#define VRR_VTOTAL_FLIP_AFTER_DBLBUF REG_BIT(29) +#define VRR_VTOTAL_PREV_FRAME_MASK REG_GENMASK(19, 0) + +#define _TRANS_VRR_FLIPLINE_A 0x60438 +#define _TRANS_VRR_FLIPLINE_B 0x61438 +#define _TRANS_VRR_FLIPLINE_C 0x62438 +#define _TRANS_VRR_FLIPLINE_D 0x63438 +#define TRANS_VRR_FLIPLINE(trans) _MMIO_TRANS2(trans, \ + _TRANS_VRR_FLIPLINE_A) +#define VRR_FLIPLINE_MASK REG_GENMASK(19, 0) + +#define _TRANS_VRR_STATUS2_A 0x6043C +#define _TRANS_VRR_STATUS2_B 0x6143C +#define _TRANS_VRR_STATUS2_C 0x6243C +#define _TRANS_VRR_STATUS2_D 0x6343C +#define TRANS_VRR_STATUS2(trans) _MMIO_TRANS2(trans, _TRANS_VRR_STATUS2_A) +#define VRR_STATUS2_VERT_LN_CNT_MASK REG_GENMASK(19, 0) + +#define _TRANS_PUSH_A 0x60A70 +#define _TRANS_PUSH_B 0x61A70 +#define _TRANS_PUSH_C 0x62A70 +#define _TRANS_PUSH_D 0x63A70 +#define TRANS_PUSH(trans) _MMIO_TRANS2(trans, _TRANS_PUSH_A) +#define TRANS_PUSH_EN REG_BIT(31) +#define TRANS_PUSH_SEND REG_BIT(30) + /* * HSW+ eDP PSR registers * @@ -4858,16 +4941,6 @@ enum { #define _PP_STATUS 0x61200 #define PP_STATUS(pps_idx) _MMIO_PPS(pps_idx, _PP_STATUS) #define PP_ON REG_BIT(31) - -#define _PP_CONTROL_1 0xc7204 -#define _PP_CONTROL_2 0xc7304 -#define ICP_PP_CONTROL(x) _MMIO(((x) == 1) ? _PP_CONTROL_1 : \ - _PP_CONTROL_2) -#define POWER_CYCLE_DELAY_MASK REG_GENMASK(8, 4) -#define VDD_OVERRIDE_FORCE REG_BIT(3) -#define BACKLIGHT_ENABLE REG_BIT(2) -#define PWR_DOWN_ON_RESET REG_BIT(1) -#define PWR_STATE_TARGET REG_BIT(0) /* * Indicates that all dependencies of the panel are on: * @@ -4928,6 +5001,7 @@ enum { #define PFIT_ENABLE (1 << 31) #define PFIT_PIPE_MASK (3 << 29) #define PFIT_PIPE_SHIFT 29 +#define PFIT_PIPE(pipe) ((pipe) << 29) #define VERT_INTERP_DISABLE (0 << 10) #define VERT_INTERP_BILINEAR (1 << 10) #define VERT_INTERP_MASK (3 << 10) @@ -5877,7 +5951,6 @@ enum { #define _PIPEAGCMAX 0x70010 #define _PIPEBGCMAX 0x71010 -#define PIPEGCMAX_RGB_MASK REG_GENMASK(15, 0) #define PIPEGCMAX(pipe, i) _MMIO_PIPE2(pipe, _PIPEAGCMAX + (i) * 4) #define _PIPE_MISC_A 0x70030 @@ -5886,6 +5959,7 @@ enum { #define PIPEMISC_YUV420_MODE_FULL_BLEND (1 << 26) /* glk+ */ #define PIPEMISC_HDR_MODE_PRECISION (1 << 23) /* icl+ */ #define PIPEMISC_OUTPUT_COLORSPACE_YUV (1 << 11) +#define PIPEMISC_PIXEL_ROUNDING_TRUNC REG_BIT(8) /* tgl+ */ #define PIPEMISC_DITHER_BPC_MASK (7 << 5) #define PIPEMISC_DITHER_8_BPC (0 << 5) #define PIPEMISC_DITHER_10_BPC (1 << 5) @@ -6768,7 +6842,7 @@ enum { #define PLANE_CTL_FORMAT_P012 (5 << 24) #define PLANE_CTL_FORMAT_XRGB_16161616F (6 << 24) #define PLANE_CTL_FORMAT_P016 (7 << 24) -#define PLANE_CTL_FORMAT_AYUV (8 << 24) +#define PLANE_CTL_FORMAT_XYUV (8 << 24) #define PLANE_CTL_FORMAT_INDEXED (12 << 24) #define PLANE_CTL_FORMAT_RGB_565 (14 << 24) #define ICL_PLANE_CTL_FORMAT_MASK (0x1f << 23) @@ -7764,6 +7838,7 @@ enum { #define BW_BUDDY1_CTL _MMIO(0x45140) #define BW_BUDDY2_CTL _MMIO(0x45150) #define BW_BUDDY_DISABLE REG_BIT(31) +#define BW_BUDDY_TLB_REQ_TIMER_MASK REG_GENMASK(21, 16) #define BW_BUDDY1_PAGE_MASK _MMIO(0x45144) #define BW_BUDDY2_PAGE_MASK _MMIO(0x45154) @@ -8506,6 +8581,7 @@ enum { #define FDI_BC_BIFURCATION_SELECT (1 << 12) #define CHASSIS_CLK_REQ_DURATION_MASK (0xf << 8) #define CHASSIS_CLK_REQ_DURATION(x) ((x) << 8) +#define SBCLK_RUN_REFCLK_DIS (1 << 7) #define SPT_PWM_GRANULARITY (1 << 0) #define SOUTH_CHICKEN2 _MMIO(0xc2004) #define FDI_MPHY_IOSFSB_RESET_STATUS (1 << 13) @@ -8997,6 +9073,7 @@ enum { #define GEN7_PCODE_ILLEGAL_DATA 0x3 #define GEN11_PCODE_ILLEGAL_SUBCOMMAND 0x4 #define GEN11_PCODE_LOCKED 0x6 +#define GEN11_PCODE_REJECTED 0x11 #define GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE 0x10 #define GEN6_PCODE_WRITE_RC6VIDS 0x4 #define GEN6_PCODE_READ_RC6VIDS 0x5 @@ -9018,10 +9095,18 @@ enum { #define ICL_PCODE_MEM_SUBSYSYSTEM_INFO 0xd #define ICL_PCODE_MEM_SS_READ_GLOBAL_INFO (0x0 << 8) #define ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point) (((point) << 16) | (0x1 << 8)) +#define ICL_PCODE_SAGV_DE_MEM_SS_CONFIG 0xe +#define ICL_PCODE_POINTS_RESTRICTED 0x0 +#define ICL_PCODE_POINTS_RESTRICTED_MASK 0x1 #define GEN6_PCODE_READ_D_COMP 0x10 #define GEN6_PCODE_WRITE_D_COMP 0x11 +#define ICL_PCODE_EXIT_TCCOLD 0x12 #define HSW_PCODE_DE_WRITE_FREQ_REQ 0x17 #define DISPLAY_IPS_CONTROL 0x19 +#define TGL_PCODE_TCCOLD 0x26 +#define TGL_PCODE_EXIT_TCCOLD_DATA_L_EXIT_FAILED REG_BIT(0) +#define TGL_PCODE_EXIT_TCCOLD_DATA_H_BLOCK_REQ 0 +#define TGL_PCODE_EXIT_TCCOLD_DATA_H_UNBLOCK_REQ REG_BIT(0) /* See also IPS_CTL */ #define IPS_PCODE_CONTROL (1 << 30) #define HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL 0x1A @@ -9145,14 +9230,19 @@ enum { #define THROTTLE_12_5 (7 << 2) #define DISABLE_EARLY_EOT (1 << 1) -#define GEN7_ROW_CHICKEN2 _MMIO(0xe4f4) -#define GEN12_DISABLE_EARLY_READ BIT(14) +#define GEN7_ROW_CHICKEN2 _MMIO(0xe4f4) +#define GEN12_DISABLE_EARLY_READ REG_BIT(14) +#define GEN12_PUSH_CONST_DEREF_HOLD_DIS REG_BIT(8) #define GEN7_ROW_CHICKEN2_GT2 _MMIO(0xf4f4) #define DOP_CLOCK_GATING_DISABLE (1 << 0) #define PUSH_CONSTANT_DEREF_DISABLE (1 << 8) #define GEN11_TDL_CLOCK_GATING_FIX_DISABLE (1 << 1) +#define GEN9_ROW_CHICKEN4 _MMIO(0xe48c) +#define GEN12_DISABLE_TDL_PUSH REG_BIT(9) +#define GEN11_DIS_PICK_2ND_EU REG_BIT(7) + #define HSW_ROW_CHICKEN3 _MMIO(0xe49c) #define HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE (1 << 6) @@ -9253,6 +9343,10 @@ enum { #define AUD_CONFIG_PIXEL_CLOCK_HDMI_74250 (7 << 16) #define AUD_CONFIG_PIXEL_CLOCK_HDMI_148352 (8 << 16) #define AUD_CONFIG_PIXEL_CLOCK_HDMI_148500 (9 << 16) +#define AUD_CONFIG_PIXEL_CLOCK_HDMI_296703 (10 << 16) +#define AUD_CONFIG_PIXEL_CLOCK_HDMI_297000 (11 << 16) +#define AUD_CONFIG_PIXEL_CLOCK_HDMI_593407 (12 << 16) +#define AUD_CONFIG_PIXEL_CLOCK_HDMI_594000 (13 << 16) #define AUD_CONFIG_DISABLE_NCTS (1 << 3) /* HSW Audio */ @@ -9299,6 +9393,22 @@ enum { #define AUD_PIN_BUF_CTL _MMIO(0x48414) #define AUD_PIN_BUF_ENABLE REG_BIT(31) +/* Display Audio Config Reg */ +#define AUD_CONFIG_BE _MMIO(0x65ef0) +#define HBLANK_EARLY_ENABLE_ICL(pipe) (0x1 << (20 - (pipe))) +#define HBLANK_EARLY_ENABLE_TGL(pipe) (0x1 << (24 + (pipe))) +#define HBLANK_START_COUNT_MASK(pipe) (0x7 << (3 + ((pipe) * 6))) +#define HBLANK_START_COUNT(pipe, val) (((val) & 0x7) << (3 + ((pipe)) * 6)) +#define NUMBER_SAMPLES_PER_LINE_MASK(pipe) (0x3 << ((pipe) * 6)) +#define NUMBER_SAMPLES_PER_LINE(pipe, val) (((val) & 0x3) << ((pipe) * 6)) + +#define HBLANK_START_COUNT_8 0 +#define HBLANK_START_COUNT_16 1 +#define HBLANK_START_COUNT_32 2 +#define HBLANK_START_COUNT_64 3 +#define HBLANK_START_COUNT_96 4 +#define HBLANK_START_COUNT_128 5 + /* * HSW - ICL power wells * @@ -9694,8 +9804,11 @@ enum skl_power_gate { #define TRANS_DDI_BPC_10 (1 << 20) #define TRANS_DDI_BPC_6 (2 << 20) #define TRANS_DDI_BPC_12 (3 << 20) +#define TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK REG_GENMASK(19, 18) /* bdw-cnl */ +#define TRANS_DDI_PORT_SYNC_MASTER_SELECT(x) REG_FIELD_PREP(TRANS_DDI_PORT_SYNC_MASTER_SELECT_MASK, (x)) #define TRANS_DDI_PVSYNC (1 << 17) #define TRANS_DDI_PHSYNC (1 << 16) +#define TRANS_DDI_PORT_SYNC_ENABLE REG_BIT(15) /* bdw-cnl */ #define TRANS_DDI_EDP_INPUT_MASK (7 << 12) #define TRANS_DDI_EDP_INPUT_A_ON (0 << 12) #define TRANS_DDI_EDP_INPUT_A_ONOFF (4 << 12) @@ -9722,12 +9835,10 @@ enum skl_power_gate { #define _TRANS_DDI_FUNC_CTL2_EDP 0x6f404 #define _TRANS_DDI_FUNC_CTL2_DSI0 0x6b404 #define _TRANS_DDI_FUNC_CTL2_DSI1 0x6bc04 -#define TRANS_DDI_FUNC_CTL2(tran) _MMIO_TRANS2(tran, \ - _TRANS_DDI_FUNC_CTL2_A) -#define PORT_SYNC_MODE_ENABLE (1 << 4) -#define PORT_SYNC_MODE_MASTER_SELECT(x) ((x) << 0) -#define PORT_SYNC_MODE_MASTER_SELECT_MASK (0x7 << 0) -#define PORT_SYNC_MODE_MASTER_SELECT_SHIFT 0 +#define TRANS_DDI_FUNC_CTL2(tran) _MMIO_TRANS2(tran, _TRANS_DDI_FUNC_CTL2_A) +#define PORT_SYNC_MODE_ENABLE REG_BIT(4) +#define PORT_SYNC_MODE_MASTER_SELECT_MASK REG_GENMASK(2, 0) +#define PORT_SYNC_MODE_MASTER_SELECT(x) REG_FIELD_PREP(PORT_SYNC_MODE_MASTER_SELECT_MASK, (x)) /* DisplayPort Transport Control */ #define _DP_TP_CTL_A 0x64040 @@ -9788,6 +9899,24 @@ enum skl_power_gate { #define DDI_BUF_BALANCE_LEG_ENABLE (1 << 31) #define DDI_BUF_TRANS_HI(port, i) _MMIO(_PORT(port, _DDI_BUF_TRANS_A, _DDI_BUF_TRANS_B) + (i) * 8 + 4) +/* DDI DP Compliance Control */ +#define _DDI_DP_COMP_CTL_A 0x605F0 +#define _DDI_DP_COMP_CTL_B 0x615F0 +#define DDI_DP_COMP_CTL(pipe) _MMIO_PIPE(pipe, _DDI_DP_COMP_CTL_A, _DDI_DP_COMP_CTL_B) +#define DDI_DP_COMP_CTL_ENABLE (1 << 31) +#define DDI_DP_COMP_CTL_D10_2 (0 << 28) +#define DDI_DP_COMP_CTL_SCRAMBLED_0 (1 << 28) +#define DDI_DP_COMP_CTL_PRBS7 (2 << 28) +#define DDI_DP_COMP_CTL_CUSTOM80 (3 << 28) +#define DDI_DP_COMP_CTL_HBR2 (4 << 28) +#define DDI_DP_COMP_CTL_SCRAMBLED_1 (5 << 28) +#define DDI_DP_COMP_CTL_HBR2_RESET (0xFC << 0) + +/* DDI DP Compliance Pattern */ +#define _DDI_DP_COMP_PAT_A 0x605F4 +#define _DDI_DP_COMP_PAT_B 0x615F4 +#define DDI_DP_COMP_PAT(pipe, i) _MMIO(_PIPE(pipe, _DDI_DP_COMP_PAT_A, _DDI_DP_COMP_PAT_B) + (i) * 4) + /* Sideband Interface (SBI) is programmed indirectly, via * SBI_ADDR, which contains the register offset; and SBI_DATA, * which contains the payload */ @@ -10735,6 +10864,12 @@ enum skl_power_gate { #define _PAL_PREC_MULTI_SEG_DATA_A 0x4A40C #define _PAL_PREC_MULTI_SEG_DATA_B 0x4AC0C +#define PAL_PREC_MULTI_SEG_RED_LDW_MASK REG_GENMASK(29, 24) +#define PAL_PREC_MULTI_SEG_RED_UDW_MASK REG_GENMASK(29, 20) +#define PAL_PREC_MULTI_SEG_GREEN_LDW_MASK REG_GENMASK(19, 14) +#define PAL_PREC_MULTI_SEG_GREEN_UDW_MASK REG_GENMASK(19, 10) +#define PAL_PREC_MULTI_SEG_BLUE_LDW_MASK REG_GENMASK(9, 4) +#define PAL_PREC_MULTI_SEG_BLUE_UDW_MASK REG_GENMASK(9, 0) #define PREC_PAL_MULTI_SEG_INDEX(pipe) _MMIO_PIPE(pipe, \ _PAL_PREC_MULTI_SEG_INDEX_A, \ diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c index 6daf18dbb3d4..526c1e9acbd5 100644 --- a/drivers/gpu/drm/i915/i915_request.c +++ b/drivers/gpu/drm/i915/i915_request.c @@ -23,6 +23,7 @@ */ #include <linux/dma-fence-array.h> +#include <linux/dma-fence-chain.h> #include <linux/irq_work.h> #include <linux/prefetch.h> #include <linux/sched.h> @@ -51,7 +52,6 @@ struct execute_cb { static struct i915_global_request { struct i915_global base; struct kmem_cache *slab_requests; - struct kmem_cache *slab_dependencies; struct kmem_cache *slab_execute_cbs; } global; @@ -102,6 +102,11 @@ static signed long i915_fence_wait(struct dma_fence *fence, timeout); } +struct kmem_cache *i915_request_slab_cache(void) +{ + return global.slab_requests; +} + static void i915_fence_release(struct dma_fence *fence) { struct i915_request *rq = to_request(fence); @@ -116,6 +121,10 @@ static void i915_fence_release(struct dma_fence *fence) i915_sw_fence_fini(&rq->submit); i915_sw_fence_fini(&rq->semaphore); + /* Keep one request on each engine for reserved use under mempressure */ + if (!cmpxchg(&rq->engine->request_pool, NULL, rq)) + return; + kmem_cache_free(global.slab_requests, rq); } @@ -291,7 +300,7 @@ bool i915_request_retire(struct i915_request *rq) spin_unlock_irq(&rq->lock); remove_from_client(rq); - list_del(&rq->link); + __list_del_entry(&rq->link); /* poison neither prev/next (RCU walks) */ intel_context_exit(rq->context); intel_context_unpin(rq->context); @@ -359,11 +368,53 @@ __await_execution(struct i915_request *rq, } spin_unlock_irq(&signal->lock); - /* Copy across semaphore status as we need the same behaviour */ - rq->sched.flags |= signal->sched.flags; return 0; } +static bool fatal_error(int error) +{ + switch (error) { + case 0: /* not an error! */ + case -EAGAIN: /* innocent victim of a GT reset (__i915_request_reset) */ + case -ETIMEDOUT: /* waiting for Godot (timer_i915_sw_fence_wake) */ + return false; + default: + return true; + } +} + +void __i915_request_skip(struct i915_request *rq) +{ + GEM_BUG_ON(!fatal_error(rq->fence.error)); + + if (rq->infix == rq->postfix) + return; + + /* + * As this request likely depends on state from the lost + * context, clear out all the user operations leaving the + * breadcrumb at the end (so we get the fence notifications). + */ + __i915_request_fill(rq, 0); + rq->infix = rq->postfix; +} + +void i915_request_set_error_once(struct i915_request *rq, int error) +{ + int old; + + GEM_BUG_ON(!IS_ERR_VALUE((long)error)); + + if (i915_request_signaled(rq)) + return; + + old = READ_ONCE(rq->fence.error); + do { + if (fatal_error(old)) + return; + } while (!try_cmpxchg(&rq->fence.error, &old, error)); +} + bool __i915_request_submit(struct i915_request *request) { struct intel_engine_cs *engine = request->engine; @@ -393,8 +444,10 @@ bool __i915_request_submit(struct i915_request *request) if (i915_request_completed(request)) goto xfer; - if (intel_context_is_banned(request->context)) - i915_request_skip(request, -EIO); + if (unlikely(intel_context_is_banned(request->context))) + i915_request_set_error_once(request, -EIO); + if (unlikely(fatal_error(request->fence.error))) + __i915_request_skip(request); /* * Are we using semaphores when the gpu is already saturated? @@ -482,10 +535,8 @@ void __i915_request_unsubmit(struct i915_request *request) spin_unlock(&request->lock); /* We've already spun, don't charge on resubmitting. */ - if (request->sched.semaphores && i915_request_started(request)) { - request->sched.attr.priority |= I915_PRIORITY_NOSEMAPHORE; + if (request->sched.semaphores && i915_request_started(request)) request->sched.semaphores = 0; - } /* * We don't need to wake_up any waiters on request->execute, they @@ -520,7 +571,7 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) trace_i915_request_submit(request); if (unlikely(fence->error)) - i915_request_skip(request, fence->error); + i915_request_set_error_once(request, fence->error); /* * We need to serialize use of the submit_request() callback @@ -546,16 +597,14 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) static int __i915_sw_fence_call semaphore_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) { - struct i915_request *request = - container_of(fence, typeof(*request), semaphore); + struct i915_request *rq = container_of(fence, typeof(*rq), semaphore); switch (state) { case FENCE_COMPLETE: - i915_schedule_bump_priority(request, I915_PRIORITY_NOSEMAPHORE); break; case FENCE_FREE: - i915_request_put(request); + i915_request_put(rq); break; } @@ -572,14 +621,22 @@ static void retire_requests(struct intel_timeline *tl) } static noinline struct i915_request * -request_alloc_slow(struct intel_timeline *tl, gfp_t gfp) +request_alloc_slow(struct intel_timeline *tl, + struct i915_request **rsvd, + gfp_t gfp) { struct i915_request *rq; - if (list_empty(&tl->requests)) - goto out; + /* If we cannot wait, dip into our reserves */ + if (!gfpflags_allow_blocking(gfp)) { + rq = xchg(rsvd, NULL); + if (!rq) /* Use the normal failure path for one final WARN */ + goto out; - if (!gfpflags_allow_blocking(gfp)) + return rq; + } + + if (list_empty(&tl->requests)) goto out; /* Move our oldest request to the slab-cache (if not in use!) */ @@ -664,7 +721,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) rq = kmem_cache_alloc(global.slab_requests, gfp | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); if (unlikely(!rq)) { - rq = request_alloc_slow(tl, gfp); + rq = request_alloc_slow(tl, &ce->engine->request_pool, gfp); if (!rq) { ret = -ENOMEM; goto err_unreserve; @@ -692,6 +749,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) RCU_INIT_POINTER(rq->timeline, tl); RCU_INIT_POINTER(rq->hwsp_cacheline, tl->hwsp_cacheline); rq->hwsp_seqno = tl->hwsp_seqno; + GEM_BUG_ON(i915_request_completed(rq)); rq->rcustate = get_state_synchronize_rcu(); /* acts as smp_mb() */ @@ -737,6 +795,8 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) rq->infix = rq->ring->emit; /* end of header; start of user payload */ intel_context_mark_active(ce); + list_add_tail_rcu(&rq->link, &tl->requests); + return rq; err_unwind: @@ -790,16 +850,26 @@ i915_request_await_start(struct i915_request *rq, struct i915_request *signal) struct dma_fence *fence; int err; - GEM_BUG_ON(i915_request_timeline(rq) == - rcu_access_pointer(signal->timeline)); + if (i915_request_timeline(rq) == rcu_access_pointer(signal->timeline)) + return 0; + + if (i915_request_started(signal)) + return 0; fence = NULL; rcu_read_lock(); spin_lock_irq(&signal->lock); - if (!i915_request_started(signal) && - !list_is_first(&signal->link, - &rcu_dereference(signal->timeline)->requests)) { - struct i915_request *prev = list_prev_entry(signal, link); + do { + struct list_head *pos = READ_ONCE(signal->link.prev); + struct i915_request *prev; + + /* Confirm signal has not been retired, the link is valid */ + if (unlikely(i915_request_started(signal))) + break; + + /* Is signal the earliest request on its timeline? */ + if (pos == &rcu_dereference(signal->timeline)->requests) + break; /* * Peek at the request before us in the timeline. That @@ -807,20 +877,25 @@ i915_request_await_start(struct i915_request *rq, struct i915_request *signal) * after acquiring a reference to it, confirm that it is * still part of the signaler's timeline. */ - if (i915_request_get_rcu(prev)) { - if (list_next_entry(prev, link) == signal) - fence = &prev->fence; - else - i915_request_put(prev); + prev = list_entry(pos, typeof(*prev), link); + if (!i915_request_get_rcu(prev)) + break; + + /* After the strong barrier, confirm prev is still attached */ + if (unlikely(READ_ONCE(prev->link.next) != &signal->link)) { + i915_request_put(prev); + break; } - } + + fence = &prev->fence; + } while (0); spin_unlock_irq(&signal->lock); rcu_read_unlock(); if (!fence) return 0; err = 0; - if (intel_timeline_sync_is_later(i915_request_timeline(rq), fence)) + if (!intel_timeline_sync_is_later(i915_request_timeline(rq), fence)) err = i915_sw_fence_await_dma_fence(&rq->submit, fence, 0, I915_FENCE_GFP); @@ -844,7 +919,7 @@ already_busywaiting(struct i915_request *rq) * * See the are-we-too-late? check in __i915_request_submit(). */ - return rq->sched.semaphores | rq->engine->saturated; + return rq->sched.semaphores | READ_ONCE(rq->engine->saturated); } static int @@ -858,6 +933,7 @@ __emit_semaphore_wait(struct i915_request *to, u32 *cs; GEM_BUG_ON(INTEL_GEN(to->i915) < 8); + GEM_BUG_ON(i915_request_has_initial_breadcrumb(to)); /* We need to pin the signaler's HWSP until we are finished reading. */ err = intel_timeline_read_hwsp(from, to, &hwsp_offset); @@ -902,14 +978,29 @@ emit_semaphore_wait(struct i915_request *to, struct i915_request *from, gfp_t gfp) { + const intel_engine_mask_t mask = READ_ONCE(from->engine)->mask; + struct i915_sw_fence *wait = &to->submit; + if (!intel_context_use_semaphores(to->context)) goto await_fence; + if (i915_request_has_initial_breadcrumb(to)) + goto await_fence; + if (!rcu_access_pointer(from->hwsp_cacheline)) goto await_fence; + /* + * If this or its dependents are waiting on an external fence + * that may fail catastrophically, then we want to avoid using + * sempahores as they bypass the fence signaling metadata, and we + * lose the fence->error propagation. + */ + if (from->sched.flags & I915_SCHED_HAS_EXTERNAL_CHAIN) + goto await_fence; + /* Just emit the first semaphore we see as request space is limited. */ - if (already_busywaiting(to) & from->engine->mask) + if (already_busywaiting(to) & mask) goto await_fence; if (i915_request_await_start(to, from) < 0) @@ -922,12 +1013,11 @@ emit_semaphore_wait(struct i915_request *to, if (__emit_semaphore_wait(to, from, from->fence.seqno)) goto await_fence; - to->sched.semaphores |= from->engine->mask; - to->sched.flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN; - return 0; + to->sched.semaphores |= mask; + wait = &to->semaphore; await_fence: - return i915_sw_fence_await_dma_fence(&to->submit, + return i915_sw_fence_await_dma_fence(wait, &from->fence, 0, I915_FENCE_GFP); } @@ -940,11 +1030,15 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from) GEM_BUG_ON(to == from); GEM_BUG_ON(to->timeline == from->timeline); - if (i915_request_completed(from)) + if (i915_request_completed(from)) { + i915_sw_fence_set_error_once(&to->submit, from->fence.error); return 0; + } if (to->engine->schedule) { - ret = i915_sched_node_add_dependency(&to->sched, &from->sched); + ret = i915_sched_node_add_dependency(&to->sched, + &from->sched, + I915_DEPENDENCY_EXTERNAL); if (ret < 0) return ret; } @@ -958,15 +1052,56 @@ i915_request_await_request(struct i915_request *to, struct i915_request *from) if (ret < 0) return ret; - if (to->sched.flags & I915_SCHED_HAS_SEMAPHORE_CHAIN) { - ret = i915_sw_fence_await_dma_fence(&to->semaphore, - &from->fence, 0, - I915_FENCE_GFP); - if (ret < 0) - return ret; + return 0; +} + +static void mark_external(struct i915_request *rq) +{ + /* + * The downside of using semaphores is that we lose metadata passing + * along the signaling chain. This is particularly nasty when we + * need to pass along a fatal error such as EFAULT or EDEADLK. For + * fatal errors we want to scrub the request before it is executed, + * which means that we cannot preload the request onto HW and have + * it wait upon a semaphore. + */ + rq->sched.flags |= I915_SCHED_HAS_EXTERNAL_CHAIN; +} + +static int +__i915_request_await_external(struct i915_request *rq, struct dma_fence *fence) +{ + mark_external(rq); + return i915_sw_fence_await_dma_fence(&rq->submit, fence, + i915_fence_context_timeout(rq->i915, + fence->context), + I915_FENCE_GFP); +} + +static int +i915_request_await_external(struct i915_request *rq, struct dma_fence *fence) +{ + struct dma_fence *iter; + int err = 0; + + if (!to_dma_fence_chain(fence)) + return __i915_request_await_external(rq, fence); + + dma_fence_chain_for_each(iter, fence) { + struct dma_fence_chain *chain = to_dma_fence_chain(iter); + + if (!dma_fence_is_i915(chain->fence)) { + err = __i915_request_await_external(rq, iter); + break; + } + + err = i915_request_await_dma_fence(rq, chain->fence); + if (err < 0) + break; } - return 0; + dma_fence_put(iter); + return err; } int @@ -1016,9 +1151,7 @@ i915_request_await_dma_fence(struct i915_request *rq, struct dma_fence *fence) if (dma_fence_is_i915(fence)) ret = i915_request_await_request(rq, to_request(fence)); else - ret = i915_sw_fence_await_dma_fence(&rq->submit, fence, - fence->context ? I915_FENCE_TIMEOUT : 0, - I915_FENCE_GFP); + ret = i915_request_await_external(rq, fence); if (ret < 0) return ret; @@ -1065,17 +1198,51 @@ __i915_request_await_execution(struct i915_request *to, &from->fence)) return 0; - /* Ensure both start together [after all semaphores in signal] */ - if (intel_engine_has_semaphores(to->engine)) - err = __emit_semaphore_wait(to, from, from->fence.seqno - 1); - else - err = i915_request_await_start(to, from); + /* + * Wait until the start of this request. + * + * The execution cb fires when we submit the request to HW. But in + * many cases this may be long before the request itself is ready to + * run (consider that we submit 2 requests for the same context, where + * the request of interest is behind an indefinite spinner). So we hook + * up to both to reduce our queues and keep the execution lag minimised + * in the worst case, though we hope that the await_start is elided. + */ + err = i915_request_await_start(to, from); if (err < 0) return err; + /* + * Ensure both start together [after all semaphores in signal] + * + * Now that we are queued to the HW at roughly the same time (thanks + * to the execute cb) and are ready to run at roughly the same time + * (thanks to the await start), our signaler may still be indefinitely + * delayed by waiting on a semaphore from a remote engine. If our + * signaler depends on a semaphore, so indirectly do we, and we do not + * want to start our payload until our signaler also starts theirs. + * So we wait. + * + * However, there is also a second condition for which we need to wait + * for the precise start of the signaler. Consider that the signaler + * was submitted in a chain of requests following another context + * (with just an ordinary intra-engine fence dependency between the + * two). In this case the signaler is queued to HW, but not for + * immediate execution, and so we must wait until it reaches the + * active slot. + */ + if (intel_engine_has_semaphores(to->engine) && + !i915_request_has_initial_breadcrumb(to)) { + err = __emit_semaphore_wait(to, from, from->fence.seqno - 1); + if (err < 0) + return err; + } + /* Couple the dependency tree for PI on this exposed to->fence */ if (to->engine->schedule) { - err = i915_sched_node_add_dependency(&to->sched, &from->sched); + err = i915_sched_node_add_dependency(&to->sched, + &from->sched, + I915_DEPENDENCY_WEAK); if (err < 0) return err; } @@ -1111,6 +1278,9 @@ i915_request_await_execution(struct i915_request *rq, continue; } + if (fence->context == rq->fence.context) + continue; + /* * We don't squash repeated fence dependencies here as we * want to run our callback in all cases. @@ -1121,9 +1291,7 @@ i915_request_await_execution(struct i915_request *rq, to_request(fence), hook); else - ret = i915_sw_fence_await_dma_fence(&rq->submit, fence, - I915_FENCE_TIMEOUT, - GFP_KERNEL); + ret = i915_request_await_external(rq, fence); if (ret < 0) return ret; } while (--nchild); @@ -1193,23 +1361,6 @@ i915_request_await_object(struct i915_request *to, return ret; } -void i915_request_skip(struct i915_request *rq, int error) -{ - GEM_BUG_ON(!IS_ERR_VALUE((long)error)); - dma_fence_set_error(&rq->fence, error); - - if (rq->infix == rq->postfix) - return; - - /* - * As this request likely depends on state from the lost - * context, clear out all the user operations leaving the - * breadcrumb at the end (so we get the fence notifications). - */ - __i915_request_fill(rq, 0); - rq->infix = rq->postfix; -} - static struct i915_request * __i915_request_add_to_timeline(struct i915_request *rq) { @@ -1239,7 +1390,17 @@ __i915_request_add_to_timeline(struct i915_request *rq) prev = to_request(__i915_active_fence_set(&timeline->last_request, &rq->fence)); if (prev && !i915_request_completed(prev)) { - if (is_power_of_2(prev->engine->mask | rq->engine->mask)) + /* + * The requests are supposed to be kept in order. However, + * we need to be wary in case the timeline->last_request + * is used as a barrier for external modification to this + * context. + */ + GEM_BUG_ON(prev->context == rq->context && + i915_seqno_passed(prev->fence.seqno, + rq->fence.seqno)); + + if (is_power_of_2(READ_ONCE(prev->engine)->mask | rq->engine->mask)) i915_sw_fence_await_sw_fence(&rq->submit, &prev->submit, &rq->submitq); @@ -1254,8 +1415,6 @@ __i915_request_add_to_timeline(struct i915_request *rq) 0); } - list_add_tail(&rq->link, &timeline->requests); - /* * Make sure that no request gazumped us - if it was allocated after * our i915_request_alloc() and called __i915_request_add() before @@ -1315,9 +1474,9 @@ void __i915_request_queue(struct i915_request *rq, * decide whether to preempt the entire chain so that it is ready to * run at the earliest possible convenience. */ - i915_sw_fence_commit(&rq->semaphore); if (attr && rq->engine->schedule) rq->engine->schedule(rq, attr); + i915_sw_fence_commit(&rq->semaphore); i915_sw_fence_commit(&rq->submit); } @@ -1325,72 +1484,27 @@ void i915_request_add(struct i915_request *rq) { struct intel_timeline * const tl = i915_request_timeline(rq); struct i915_sched_attr attr = {}; - struct i915_request *prev; + struct i915_gem_context *ctx; lockdep_assert_held(&tl->mutex); lockdep_unpin_lock(&tl->mutex, rq->cookie); trace_i915_request_add(rq); + __i915_request_commit(rq); - prev = __i915_request_commit(rq); - - if (rcu_access_pointer(rq->context->gem_context)) - attr = i915_request_gem_context(rq)->sched; - - /* - * Boost actual workloads past semaphores! - * - * With semaphores we spin on one engine waiting for another, - * simply to reduce the latency of starting our work when - * the signaler completes. However, if there is any other - * work that we could be doing on this engine instead, that - * is better utilisation and will reduce the overall duration - * of the current work. To avoid PI boosting a semaphore - * far in the distance past over useful work, we keep a history - * of any semaphore use along our dependency chain. - */ - if (!(rq->sched.flags & I915_SCHED_HAS_SEMAPHORE_CHAIN)) - attr.priority |= I915_PRIORITY_NOSEMAPHORE; - - /* - * Boost priorities to new clients (new request flows). - * - * Allow interactive/synchronous clients to jump ahead of - * the bulk clients. (FQ_CODEL) - */ - if (list_empty(&rq->sched.signalers_list)) - attr.priority |= I915_PRIORITY_WAIT; + /* XXX placeholder for selftests */ + rcu_read_lock(); + ctx = rcu_dereference(rq->context->gem_context); + if (ctx) + attr = ctx->sched; + rcu_read_unlock(); - local_bh_disable(); __i915_request_queue(rq, &attr); - local_bh_enable(); /* Kick the execlists tasklet if just scheduled */ - - /* - * In typical scenarios, we do not expect the previous request on - * the timeline to be still tracked by timeline->last_request if it - * has been completed. If the completed request is still here, that - * implies that request retirement is a long way behind submission, - * suggesting that we haven't been retiring frequently enough from - * the combination of retire-before-alloc, waiters and the background - * retirement worker. So if the last request on this timeline was - * already completed, do a catch up pass, flushing the retirement queue - * up to this client. Since we have now moved the heaviest operations - * during retirement onto secondary workers, such as freeing objects - * or contexts, retiring a bunch of requests is mostly list management - * (and cache misses), and so we should not be overly penalizing this - * client by performing excess work, though we may still performing - * work on behalf of others -- but instead we should benefit from - * improved resource management. (Well, that's the theory at least.) - */ - if (prev && - i915_request_completed(prev) && - rcu_access_pointer(prev->timeline) == tl) - i915_request_retire_upto(prev); mutex_unlock(&tl->mutex); } -static unsigned long local_clock_us(unsigned int *cpu) +static unsigned long local_clock_ns(unsigned int *cpu) { unsigned long t; @@ -1407,7 +1521,7 @@ static unsigned long local_clock_us(unsigned int *cpu) * stop busywaiting, see busywait_stop(). */ *cpu = get_cpu(); - t = local_clock() >> 10; + t = local_clock(); put_cpu(); return t; @@ -1417,15 +1531,15 @@ static bool busywait_stop(unsigned long timeout, unsigned int cpu) { unsigned int this_cpu; - if (time_after(local_clock_us(&this_cpu), timeout)) + if (time_after(local_clock_ns(&this_cpu), timeout)) return true; return this_cpu != cpu; } -static bool __i915_spin_request(const struct i915_request * const rq, - int state, unsigned long timeout_us) +static bool __i915_spin_request(const struct i915_request * const rq, int state) { + unsigned long timeout_ns; unsigned int cpu; /* @@ -1453,7 +1567,8 @@ static bool __i915_spin_request(const struct i915_request * const rq, * takes to sleep on a request, on the order of a microsecond. */ - timeout_us += local_clock_us(&cpu); + timeout_ns = READ_ONCE(rq->engine->props.max_busywait_duration_ns); + timeout_ns += local_clock_ns(&cpu); do { if (i915_request_completed(rq)) return true; @@ -1461,7 +1576,7 @@ static bool __i915_spin_request(const struct i915_request * const rq, if (signal_pending_state(state, current)) break; - if (busywait_stop(timeout_us, cpu)) + if (busywait_stop(timeout_ns, cpu)) break; cpu_relax(); @@ -1547,8 +1662,8 @@ long i915_request_wait(struct i915_request *rq, * completion. That requires having a good predictor for the request * duration, which we currently lack. */ - if (IS_ACTIVE(CONFIG_DRM_I915_SPIN_REQUEST) && - __i915_spin_request(rq, state, CONFIG_DRM_I915_SPIN_REQUEST)) { + if (IS_ACTIVE(CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT) && + __i915_spin_request(rq, state)) { dma_fence_signal(&rq->fence); goto out; } @@ -1568,7 +1683,6 @@ long i915_request_wait(struct i915_request *rq, if (flags & I915_WAIT_PRIORITY) { if (!i915_request_started(rq) && INTEL_GEN(rq->i915) >= 6) intel_rps_boost(rq); - i915_schedule_bump_priority(rq, I915_PRIORITY_WAIT); } wait.tsk = current; @@ -1614,14 +1728,12 @@ out: static void i915_global_request_shrink(void) { - kmem_cache_shrink(global.slab_dependencies); kmem_cache_shrink(global.slab_execute_cbs); kmem_cache_shrink(global.slab_requests); } static void i915_global_request_exit(void) { - kmem_cache_destroy(global.slab_dependencies); kmem_cache_destroy(global.slab_execute_cbs); kmem_cache_destroy(global.slab_requests); } @@ -1651,17 +1763,9 @@ int __init i915_global_request_init(void) if (!global.slab_execute_cbs) goto err_requests; - global.slab_dependencies = KMEM_CACHE(i915_dependency, - SLAB_HWCACHE_ALIGN | - SLAB_RECLAIM_ACCOUNT); - if (!global.slab_dependencies) - goto err_execute_cbs; - i915_global_register(&global.base); return 0; -err_execute_cbs: - kmem_cache_destroy(global.slab_execute_cbs); err_requests: kmem_cache_destroy(global.slab_requests); return -ENOMEM; diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h index da8420f03232..8ec7ee4dbadc 100644 --- a/drivers/gpu/drm/i915/i915_request.h +++ b/drivers/gpu/drm/i915/i915_request.h @@ -26,6 +26,7 @@ #define I915_REQUEST_H #include <linux/dma-fence.h> +#include <linux/irq_work.h> #include <linux/lockdep.h> #include "gem/i915_gem_context_types.h" @@ -83,19 +84,26 @@ enum { I915_FENCE_FLAG_PQUEUE, /* - * I915_FENCE_FLAG_SIGNAL - this request is currently on signal_list + * I915_FENCE_FLAG_HOLD - this request is currently on hold * - * Internal bookkeeping used by the breadcrumb code to track when - * a request is on the various signal_list. + * This request has been suspended, pending an ongoing investigation. */ - I915_FENCE_FLAG_SIGNAL, + I915_FENCE_FLAG_HOLD, /* - * I915_FENCE_FLAG_HOLD - this request is currently on hold + * I915_FENCE_FLAG_INITIAL_BREADCRUMB - this request has the initial + * breadcrumb that marks the end of semaphore waits and start of the + * user payload. + */ + I915_FENCE_FLAG_INITIAL_BREADCRUMB, + + /* + * I915_FENCE_FLAG_SIGNAL - this request is currently on signal_list * - * This request has been suspended, pending an ongoing investigation. + * Internal bookkeeping used by the breadcrumb code to track when + * a request is on the various signal_list. */ - I915_FENCE_FLAG_HOLD, + I915_FENCE_FLAG_SIGNAL, /* * I915_FENCE_FLAG_NOPREEMPT - this request should not be preempted @@ -298,11 +306,16 @@ static inline bool dma_fence_is_i915(const struct dma_fence *fence) return fence->ops == &i915_fence_ops; } +struct kmem_cache *i915_request_slab_cache(void); + struct i915_request * __must_check __i915_request_create(struct intel_context *ce, gfp_t gfp); struct i915_request * __must_check i915_request_create(struct intel_context *ce); +void i915_request_set_error_once(struct i915_request *rq, int error); +void __i915_request_skip(struct i915_request *rq); + struct i915_request *__i915_request_commit(struct i915_request *request); void __i915_request_queue(struct i915_request *rq, const struct i915_sched_attr *attr); @@ -352,8 +365,6 @@ void i915_request_add(struct i915_request *rq); bool __i915_request_submit(struct i915_request *request); void i915_request_submit(struct i915_request *request); -void i915_request_skip(struct i915_request *request, int error); - void __i915_request_unsubmit(struct i915_request *request); void i915_request_unsubmit(struct i915_request *request); @@ -385,6 +396,12 @@ static inline bool i915_request_in_priority_queue(const struct i915_request *rq) return test_bit(I915_FENCE_FLAG_PQUEUE, &rq->fence.flags); } +static inline bool +i915_request_has_initial_breadcrumb(const struct i915_request *rq) +{ + return test_bit(I915_FENCE_FLAG_INITIAL_BREADCRUMB, &rq->fence.flags); +} + /** * Returns true if seq1 is later than seq2. */ @@ -395,7 +412,9 @@ static inline bool i915_seqno_passed(u32 seq1, u32 seq2) static inline u32 __hwsp_seqno(const struct i915_request *rq) { - return READ_ONCE(*rq->hwsp_seqno); + const u32 *hwsp = READ_ONCE(rq->hwsp_seqno); + + return READ_ONCE(*hwsp); } /** @@ -509,7 +528,8 @@ static inline bool i915_request_completed(const struct i915_request *rq) static inline void i915_request_mark_complete(struct i915_request *rq) { - rq->hwsp_seqno = (u32 *)&rq->fence.seqno; /* decouple from HWSP */ + WRITE_ONCE(rq->hwsp_seqno, /* decouple from HWSP */ + (u32 *)&rq->fence.seqno); } static inline bool i915_request_has_waitboost(const struct i915_request *rq) diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index e19a37a83397..f4ea318781f0 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -51,11 +51,11 @@ static void assert_priolists(struct intel_engine_execlists * const execlists) GEM_BUG_ON(rb_first_cached(&execlists->queue) != rb_first(&execlists->queue.rb_root)); - last_prio = (INT_MAX >> I915_USER_PRIORITY_SHIFT) + 1; + last_prio = INT_MAX; for (rb = rb_first_cached(&execlists->queue); rb; rb = rb_next(rb)) { const struct i915_priolist *p = to_priolist(rb); - GEM_BUG_ON(p->priority >= last_prio); + GEM_BUG_ON(p->priority > last_prio); last_prio = p->priority; GEM_BUG_ON(!p->used); @@ -174,7 +174,7 @@ sched_lock_engine(const struct i915_sched_node *node, static inline int rq_prio(const struct i915_request *rq) { - return rq->sched.attr.priority | __NO_PREEMPTION; + return rq->sched.attr.priority; } static inline bool need_preempt(int prio, int active) @@ -209,6 +209,14 @@ static void kick_submission(struct intel_engine_cs *engine, if (!inflight) goto unlock; + ENGINE_TRACE(engine, + "bumping queue-priority-hint:%d for rq:%llx:%lld, inflight:%llx:%lld prio %d\n", + prio, + rq->fence.context, rq->fence.seqno, + inflight->fence.context, inflight->fence.seqno, + inflight->sched.attr.priority); + engine->execlists.queue_priority_hint = prio; + /* * If we are already the currently executing context, don't * bother evaluating if we should preempt ourselves. @@ -216,7 +224,6 @@ static void kick_submission(struct intel_engine_cs *engine, if (inflight->context == rq->context) goto unlock; - engine->execlists.queue_priority_hint = prio; if (need_preempt(prio, rq_prio(inflight))) tasklet_hi_schedule(&engine->execlists.tasklet); @@ -227,10 +234,10 @@ unlock: static void __i915_schedule(struct i915_sched_node *node, const struct i915_sched_attr *attr) { + const int prio = max(attr->priority, node->attr.priority); struct intel_engine_cs *engine; struct i915_dependency *dep, *p; struct i915_dependency stack; - const int prio = attr->priority; struct sched_cache cache; LIST_HEAD(dfs); @@ -238,9 +245,6 @@ static void __i915_schedule(struct i915_sched_node *node, lockdep_assert_held(&schedule_lock); GEM_BUG_ON(prio == I915_PRIORITY_INVALID); - if (prio <= READ_ONCE(node->attr.priority)) - return; - if (node_signaled(node)) return; @@ -324,7 +328,7 @@ static void __i915_schedule(struct i915_sched_node *node, GEM_BUG_ON(node_to_request(node)->engine != engine); - node->attr.priority = prio; + WRITE_ONCE(node->attr.priority, prio); /* * Once the request is ready, it will be placed into the @@ -363,6 +367,9 @@ static void __bump_priority(struct i915_sched_node *node, unsigned int bump) { struct i915_sched_attr attr = node->attr; + if (attr.priority & bump) + return; + attr.priority |= bump; __i915_schedule(node, &attr); } @@ -427,25 +434,12 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node, dep->waiter = node; dep->flags = flags; - /* Keep track of whether anyone on this chain has a semaphore */ - if (signal->flags & I915_SCHED_HAS_SEMAPHORE_CHAIN && - !node_started(signal)) - node->flags |= I915_SCHED_HAS_SEMAPHORE_CHAIN; - /* All set, now publish. Beware the lockless walkers. */ list_add_rcu(&dep->signal_link, &node->signalers_list); list_add_rcu(&dep->wait_link, &signal->waiters_list); - /* - * As we do not allow WAIT to preempt inflight requests, - * once we have executed a request, along with triggering - * any execution callbacks, we must preserve its ordering - * within the non-preemptible FIFO. - */ - BUILD_BUG_ON(__NO_PREEMPTION & ~I915_PRIORITY_MASK); - if (flags & I915_DEPENDENCY_EXTERNAL) - __bump_priority(signal, __NO_PREEMPTION); - + /* Propagate the chains */ + node->flags |= signal->flags; ret = true; } @@ -455,7 +449,8 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node, } int i915_sched_node_add_dependency(struct i915_sched_node *node, - struct i915_sched_node *signal) + struct i915_sched_node *signal, + unsigned long flags) { struct i915_dependency *dep; @@ -463,11 +458,14 @@ int i915_sched_node_add_dependency(struct i915_sched_node *node, if (!dep) return -ENOMEM; + local_bh_disable(); + if (!__i915_sched_node_add_dependency(node, signal, dep, - I915_DEPENDENCY_EXTERNAL | - I915_DEPENDENCY_ALLOC)) + flags | I915_DEPENDENCY_ALLOC)) i915_dependency_free(dep); + local_bh_enable(); /* kick submission tasklet */ + return 0; } @@ -486,7 +484,7 @@ void i915_sched_node_fini(struct i915_sched_node *node) list_for_each_entry_safe(dep, tmp, &node->signalers_list, signal_link) { GEM_BUG_ON(!list_empty(&dep->dfs_link)); - list_del(&dep->wait_link); + list_del_rcu(&dep->wait_link); if (dep->flags & I915_DEPENDENCY_ALLOC) i915_dependency_free(dep); } @@ -497,7 +495,7 @@ void i915_sched_node_fini(struct i915_sched_node *node) GEM_BUG_ON(dep->signaler != node); GEM_BUG_ON(!list_empty(&dep->dfs_link)); - list_del(&dep->signal_link); + list_del_rcu(&dep->signal_link); if (dep->flags & I915_DEPENDENCY_ALLOC) i915_dependency_free(dep); } @@ -526,7 +524,8 @@ static struct i915_global_scheduler global = { { int __init i915_global_scheduler_init(void) { global.slab_dependencies = KMEM_CACHE(i915_dependency, - SLAB_HWCACHE_ALIGN); + SLAB_HWCACHE_ALIGN | + SLAB_TYPESAFE_BY_RCU); if (!global.slab_dependencies) return -ENOMEM; diff --git a/drivers/gpu/drm/i915/i915_scheduler.h b/drivers/gpu/drm/i915/i915_scheduler.h index d1dc4efef77b..6f0bf00fc569 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.h +++ b/drivers/gpu/drm/i915/i915_scheduler.h @@ -34,7 +34,8 @@ bool __i915_sched_node_add_dependency(struct i915_sched_node *node, unsigned long flags); int i915_sched_node_add_dependency(struct i915_sched_node *node, - struct i915_sched_node *signal); + struct i915_sched_node *signal, + unsigned long flags); void i915_sched_node_fini(struct i915_sched_node *node); diff --git a/drivers/gpu/drm/i915/i915_scheduler_types.h b/drivers/gpu/drm/i915/i915_scheduler_types.h index d18e70550054..f72e6c397b08 100644 --- a/drivers/gpu/drm/i915/i915_scheduler_types.h +++ b/drivers/gpu/drm/i915/i915_scheduler_types.h @@ -65,7 +65,7 @@ struct i915_sched_node { struct list_head link; struct i915_sched_attr attr; unsigned int flags; -#define I915_SCHED_HAS_SEMAPHORE_CHAIN BIT(0) +#define I915_SCHED_HAS_EXTERNAL_CHAIN BIT(0) intel_engine_mask_t semaphores; }; @@ -78,6 +78,7 @@ struct i915_dependency { unsigned long flags; #define I915_DEPENDENCY_ALLOC BIT(0) #define I915_DEPENDENCY_EXTERNAL BIT(1) +#define I915_DEPENDENCY_WEAK BIT(2) }; #endif /* _I915_SCHEDULER_TYPES_H_ */ diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h index 98bcb6fa0ab4..d53d207ab6eb 100644 --- a/drivers/gpu/drm/i915/i915_selftest.h +++ b/drivers/gpu/drm/i915/i915_selftest.h @@ -133,4 +133,6 @@ bool __igt_timeout(unsigned long timeout, const char *fmt, ...); #define igt_timeout(t, fmt, ...) \ __igt_timeout((t), KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__) +void igt_hexdump(const void *buf, size_t len); + #endif /* !__I915_SELFTEST_H__ */ diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 8812cdd9007f..ed2be3489f8e 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -24,8 +24,6 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include <drm/i915_drm.h> - #include "display/intel_fbc.h" #include "display/intel_gmbus.h" #include "display/intel_vga.h" diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c index a3d38e089b6e..295b9829e2da 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence.c +++ b/drivers/gpu/drm/i915/i915_sw_fence.c @@ -421,7 +421,7 @@ static void timer_i915_sw_fence_wake(struct timer_list *t) if (!fence) return; - pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%pS)\n", + pr_notice("Asynchronous wait on fence %s:%s:%llx timed out (hint:%ps)\n", cb->dma->ops->get_driver_name(cb->dma), cb->dma->ops->get_timeline_name(cb->dma), cb->dma->seqno, @@ -546,13 +546,11 @@ int __i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, cb->fence = fence; i915_sw_fence_await(fence); - ret = dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake); - if (ret == 0) { - ret = 1; - } else { + ret = 1; + if (dma_fence_add_callback(dma, &cb->base, __dma_i915_sw_fence_wake)) { + /* fence already signaled */ __dma_i915_sw_fence_wake(dma, &cb->base); - if (ret == -ENOENT) /* fence already signaled */ - ret = 0; + ret = 0; } return ret; diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.c b/drivers/gpu/drm/i915/i915_sw_fence_work.c index 997b2998f1f2..a3a81bb8f2c3 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence_work.c +++ b/drivers/gpu/drm/i915/i915_sw_fence_work.c @@ -38,7 +38,10 @@ fence_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) if (!f->dma.error) { dma_fence_get(&f->dma); - queue_work(system_unbound_wq, &f->work); + if (test_bit(DMA_FENCE_WORK_IMM, &f->dma.flags)) + fence_work(&f->work); + else + queue_work(system_unbound_wq, &f->work); } else { fence_complete(f); } diff --git a/drivers/gpu/drm/i915/i915_sw_fence_work.h b/drivers/gpu/drm/i915/i915_sw_fence_work.h index 3a22b287e201..2c409f11c5c5 100644 --- a/drivers/gpu/drm/i915/i915_sw_fence_work.h +++ b/drivers/gpu/drm/i915/i915_sw_fence_work.h @@ -32,6 +32,10 @@ struct dma_fence_work { const struct dma_fence_work_ops *ops; }; +enum { + DMA_FENCE_WORK_IMM = DMA_FENCE_FLAG_USER_BITS, +}; + void dma_fence_work_init(struct dma_fence_work *f, const struct dma_fence_work_ops *ops); int dma_fence_work_chain(struct dma_fence_work *f, struct dma_fence *signal); @@ -41,4 +45,23 @@ static inline void dma_fence_work_commit(struct dma_fence_work *f) i915_sw_fence_commit(&f->chain); } +/** + * dma_fence_work_commit_imm: Commit the fence, and if possible execute locally. + * @f: the fenced worker + * + * Instead of always scheduling a worker to execute the callback (see + * dma_fence_work_commit()), we try to execute the callback immediately in + * the local context. It is required that the fence be committed before it + * is published, and that no other threads try to tamper with the number + * of asynchronous waits on the fence (or else the callback will be + * executed in the wrong context, i.e. not the callers). + */ +static inline void dma_fence_work_commit_imm(struct dma_fence_work *f) +{ + if (atomic_read(&f->chain.pending) <= 1) + __set_bit(DMA_FENCE_WORK_IMM, &f->dma.flags); + + dma_fence_work_commit(f); +} + #endif /* I915_SW_FENCE_WORK_H */ diff --git a/drivers/gpu/drm/i915/i915_switcheroo.c b/drivers/gpu/drm/i915/i915_switcheroo.c index 39c79e1c5b52..b3a24eac21f1 100644 --- a/drivers/gpu/drm/i915/i915_switcheroo.c +++ b/drivers/gpu/drm/i915/i915_switcheroo.c @@ -20,14 +20,14 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, } if (state == VGA_SWITCHEROO_ON) { - pr_info("switched on\n"); + drm_info(&i915->drm, "switched on\n"); i915->drm.switch_power_state = DRM_SWITCH_POWER_CHANGING; /* i915 resume handler doesn't set to D0 */ pci_set_power_state(pdev, PCI_D0); i915_resume_switcheroo(i915); i915->drm.switch_power_state = DRM_SWITCH_POWER_ON; } else { - pr_info("switched off\n"); + drm_info(&i915->drm, "switched off\n"); i915->drm.switch_power_state = DRM_SWITCH_POWER_CHANGING; i915_suspend_switcheroo(i915, pmm); i915->drm.switch_power_state = DRM_SWITCH_POWER_OFF; @@ -43,7 +43,7 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev) * locking inversion with the driver load path. And the access here is * completely racy anyway. So don't bother with locking for now. */ - return i915 && i915->drm.open_count == 0; + return i915 && atomic_read(&i915->drm.open_count) == 0; } static const struct vga_switcheroo_client_ops i915_switcheroo_ops = { diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index c14d762bd652..45d32ef42787 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -32,6 +32,7 @@ #include "gt/intel_rc6.h" #include "gt/intel_rps.h" +#include "gt/sysfs_engines.h" #include "i915_drv.h" #include "i915_sysfs.h" @@ -606,6 +607,8 @@ void i915_setup_sysfs(struct drm_i915_private *dev_priv) drm_err(&dev_priv->drm, "RPS sysfs setup failed\n"); i915_setup_error_capture(kdev); + + intel_engines_add_sysfs(dev_priv); } void i915_teardown_sysfs(struct drm_i915_private *dev_priv) diff --git a/drivers/gpu/drm/i915/i915_utils.c b/drivers/gpu/drm/i915/i915_utils.c index 632d6953c78d..e28eae4a8f70 100644 --- a/drivers/gpu/drm/i915/i915_utils.c +++ b/drivers/gpu/drm/i915/i915_utils.c @@ -8,7 +8,6 @@ #include "i915_drv.h" #include "i915_utils.h" -#define FDO_BUG_URL "https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs" #define FDO_BUG_MSG "Please file a bug on drm/i915; see " FDO_BUG_URL " for details." void @@ -102,5 +101,6 @@ void set_timer_ms(struct timer_list *t, unsigned long timeout) */ barrier(); - mod_timer(t, jiffies + timeout); + /* Keep t->expires = 0 reserved to indicate a canceled timer. */ + mod_timer(t, jiffies + timeout ?: 1); } diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index b0ade76bec90..03a73d2bd50d 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -34,6 +34,8 @@ struct drm_i915_private; struct timer_list; +#define FDO_BUG_URL "https://gitlab.freedesktop.org/drm/intel/-/wikis/How-to-file-i915-bugs" + #undef WARN_ON /* Many gcc seem to no see through this and fall over :( */ #if 0 @@ -100,12 +102,24 @@ bool i915_error_injected(void); typeof(max) max__ = (max); \ (void)(&start__ == &size__); \ (void)(&start__ == &max__); \ - start__ > max__ || size__ > max__ - start__; \ + start__ >= max__ || size__ > max__ - start__; \ }) #define range_overflows_t(type, start, size, max) \ range_overflows((type)(start), (type)(size), (type)(max)) +#define range_overflows_end(start, size, max) ({ \ + typeof(start) start__ = (start); \ + typeof(size) size__ = (size); \ + typeof(max) max__ = (max); \ + (void)(&start__ == &size__); \ + (void)(&start__ == &max__); \ + start__ > max__ || size__ > max__ - start__; \ +}) + +#define range_overflows_end_t(type, start, size, max) \ + range_overflows_end((type)(start), (type)(size), (type)(max)) + /* Note we don't consider signbits :| */ #define overflows_type(x, T) \ (sizeof(x) > sizeof(T) && (x) >> BITS_PER_TYPE(T)) @@ -234,6 +248,11 @@ static inline u64 ptr_to_u64(const void *ptr) __idx; \ }) +static inline bool is_power_of_2_u64(u64 n) +{ + return (n != 0 && ((n & (n - 1)) == 0)); +} + static inline void __list_del_many(struct list_head *head, struct list_head *first) { @@ -241,6 +260,12 @@ static inline void __list_del_many(struct list_head *head, WRITE_ONCE(head->next, first); } +static inline int list_is_last_rcu(const struct list_head *list, + const struct list_head *head) +{ + return READ_ONCE(list->next) == head; +} + /* * Wait until the work is finally complete, even if it tries to postpone * by requeueing itself. Note, that if the worker never cancels itself, diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c index 4afe21662266..70fca72f5162 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.c +++ b/drivers/gpu/drm/i915/i915_vgpu.c @@ -21,6 +21,8 @@ * SOFTWARE. */ +#include "i915_drv.h" +#include "i915_pvinfo.h" #include "i915_vgpu.h" /** @@ -51,13 +53,13 @@ */ /** - * i915_detect_vgpu - detect virtual GPU + * intel_vgpu_detect - detect virtual GPU * @dev_priv: i915 device private * * This function is called at the initialization stage, to detect whether * running on a vGPU. */ -void i915_detect_vgpu(struct drm_i915_private *dev_priv) +void intel_vgpu_detect(struct drm_i915_private *dev_priv) { struct pci_dev *pdev = dev_priv->drm.pdev; u64 magic; @@ -102,11 +104,36 @@ out: pci_iounmap(pdev, shared_area); } +void intel_vgpu_register(struct drm_i915_private *i915) +{ + /* + * Notify a valid surface after modesetting, when running inside a VM. + */ + if (intel_vgpu_active(i915)) + intel_uncore_write(&i915->uncore, vgtif_reg(display_ready), + VGT_DRV_DISPLAY_READY); +} + +bool intel_vgpu_active(struct drm_i915_private *dev_priv) +{ + return dev_priv->vgpu.active; +} + bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv) { return dev_priv->vgpu.caps & VGT_CAPS_FULL_PPGTT; } +bool intel_vgpu_has_hwsp_emulation(struct drm_i915_private *dev_priv) +{ + return dev_priv->vgpu.caps & VGT_CAPS_HWSP_EMULATION; +} + +bool intel_vgpu_has_huge_gtt(struct drm_i915_private *dev_priv) +{ + return dev_priv->vgpu.caps & VGT_CAPS_HUGE_GTT; +} + struct _balloon_info_ { /* * There are up to 2 regions per mappable/unmappable graphic diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h index 8b3663dad193..ffbb77d08048 100644 --- a/drivers/gpu/drm/i915/i915_vgpu.h +++ b/drivers/gpu/drm/i915/i915_vgpu.h @@ -24,24 +24,17 @@ #ifndef _I915_VGPU_H_ #define _I915_VGPU_H_ -#include "i915_drv.h" -#include "i915_pvinfo.h" +#include <linux/types.h> -void i915_detect_vgpu(struct drm_i915_private *dev_priv); +struct drm_i915_private; +struct i915_ggtt; -bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *dev_priv); - -static inline bool -intel_vgpu_has_hwsp_emulation(struct drm_i915_private *dev_priv) -{ - return dev_priv->vgpu.caps & VGT_CAPS_HWSP_EMULATION; -} - -static inline bool -intel_vgpu_has_huge_gtt(struct drm_i915_private *dev_priv) -{ - return dev_priv->vgpu.caps & VGT_CAPS_HUGE_GTT; -} +void intel_vgpu_detect(struct drm_i915_private *i915); +bool intel_vgpu_active(struct drm_i915_private *i915); +void intel_vgpu_register(struct drm_i915_private *i915); +bool intel_vgpu_has_full_ppgtt(struct drm_i915_private *i915); +bool intel_vgpu_has_hwsp_emulation(struct drm_i915_private *i915); +bool intel_vgpu_has_huge_gtt(struct drm_i915_private *i915); int intel_vgt_balloon(struct i915_ggtt *ggtt); void intel_vgt_deballoon(struct i915_ggtt *ggtt); diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 74dc3ba59ce5..fc14ebf9a0b7 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -158,16 +158,18 @@ vma_create(struct drm_i915_gem_object *obj, GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); + spin_lock(&obj->vma.lock); + if (i915_is_ggtt(vm)) { if (unlikely(overflows_type(vma->size, u32))) - goto err_vma; + goto err_unlock; vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, i915_gem_object_get_tiling(obj), i915_gem_object_get_stride(obj)); if (unlikely(vma->fence_size < vma->size || /* overflow */ vma->fence_size > vm->total)) - goto err_vma; + goto err_unlock; GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); @@ -179,8 +181,6 @@ vma_create(struct drm_i915_gem_object *obj, __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma)); } - spin_lock(&obj->vma.lock); - rb = NULL; p = &obj->vma.tree.rb_node; while (*p) { @@ -225,6 +225,8 @@ vma_create(struct drm_i915_gem_object *obj, return vma; +err_unlock: + spin_unlock(&obj->vma.lock); err_vma: i915_vma_free(vma); return ERR_PTR(-E2BIG); @@ -520,7 +522,6 @@ void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags) GEM_BUG_ON(!obj); i915_vma_unpin(vma); - i915_vma_close(vma); if (flags & I915_VMA_RELEASE_MAP) i915_gem_object_unpin_map(obj); @@ -608,18 +609,6 @@ bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color) return true; } -static void assert_bind_count(const struct drm_i915_gem_object *obj) -{ - /* - * Combine the assertion that the object is bound and that we have - * pinned its pages. But we should never have bound the object - * more than we have pinned its pages. (For complete accuracy, we - * assume that no else is pinning the pages, but as a rough assertion - * that we will not run into problems later, this will do!) - */ - GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < atomic_read(&obj->bind_count)); -} - /** * i915_vma_insert - finds a slot for the vma in its address space * @vma: the vma @@ -641,7 +630,6 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) u64 start, end; int ret; - GEM_BUG_ON(i915_vma_is_closed(vma)); GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); @@ -739,12 +727,6 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color)); - if (vma->obj) { - struct drm_i915_gem_object *obj = vma->obj; - - atomic_inc(&obj->bind_count); - assert_bind_count(obj); - } list_add_tail(&vma->vm_link, &vma->vm->bound_list); return 0; @@ -762,12 +744,6 @@ i915_vma_detach(struct i915_vma *vma) * it to be reaped by the shrinker. */ list_del(&vma->vm_link); - if (vma->obj) { - struct drm_i915_gem_object *obj = vma->obj; - - assert_bind_count(obj); - atomic_dec(&obj->bind_count); - } } static bool try_qad_pin(struct i915_vma *vma, unsigned int flags) @@ -914,11 +890,35 @@ int i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) if (flags & PIN_GLOBAL) wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); - /* No more allocations allowed once we hold vm->mutex */ - err = mutex_lock_interruptible(&vma->vm->mutex); + /* + * Differentiate between user/kernel vma inside the aliasing-ppgtt. + * + * We conflate the Global GTT with the user's vma when using the + * aliasing-ppgtt, but it is still vitally important to try and + * keep the use cases distinct. For example, userptr objects are + * not allowed inside the Global GTT as that will cause lock + * inversions when we have to evict them the mmu_notifier callbacks - + * but they are allowed to be part of the user ppGTT which can never + * be mapped. As such we try to give the distinct users of the same + * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt + * and i915_ppgtt separate]. + * + * NB this may cause us to mask real lock inversions -- while the + * code is safe today, lockdep may not be able to spot future + * transgressions. + */ + err = mutex_lock_interruptible_nested(&vma->vm->mutex, + !(flags & PIN_GLOBAL)); if (err) goto err_fence; + /* No more allocations allowed now we hold vm->mutex */ + + if (unlikely(i915_vma_is_closed(vma))) { + err = -ENOENT; + goto err_unlock; + } + bound = atomic_read(&vma->flags); if (unlikely(bound & I915_VMA_ERROR)) { err = -ENOMEM; @@ -976,7 +976,7 @@ err_unlock: mutex_unlock(&vma->vm->mutex); err_fence: if (work) - dma_fence_work_commit(&work->base); + dma_fence_work_commit_imm(&work->base); if (wakeref) intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); err_pages: @@ -1022,13 +1022,8 @@ int i915_ggtt_pin(struct i915_vma *vma, u32 align, unsigned int flags) } while (1); } -void i915_vma_close(struct i915_vma *vma) +static void __vma_close(struct i915_vma *vma, struct intel_gt *gt) { - struct intel_gt *gt = vma->vm->gt; - unsigned long flags; - - GEM_BUG_ON(i915_vma_is_closed(vma)); - /* * We defer actually closing, unbinding and destroying the VMA until * the next idle point, or if the object is freed in the meantime. By @@ -1041,9 +1036,25 @@ void i915_vma_close(struct i915_vma *vma) * causing us to rebind the VMA once more. This ends up being a lot * of wasted work for the steady state. */ - spin_lock_irqsave(>->closed_lock, flags); + GEM_BUG_ON(i915_vma_is_closed(vma)); list_add(&vma->closed_link, >->closed_vma); - spin_unlock_irqrestore(>->closed_lock, flags); +} + +void i915_vma_close(struct i915_vma *vma) +{ + struct intel_gt *gt = vma->vm->gt; + unsigned long flags; + + if (i915_vma_is_ggtt(vma)) + return; + + GEM_BUG_ON(!atomic_read(&vma->open_count)); + if (atomic_dec_and_lock_irqsave(&vma->open_count, + >->closed_lock, + flags)) { + __vma_close(vma, gt); + spin_unlock_irqrestore(>->closed_lock, flags); + } } static void __i915_vma_remove_closed(struct i915_vma *vma) @@ -1093,6 +1104,7 @@ void i915_vma_release(struct kref *ref) void i915_vma_parked(struct intel_gt *gt) { struct i915_vma *vma, *next; + LIST_HEAD(closed); spin_lock_irq(>->closed_lock); list_for_each_entry_safe(vma, next, >->closed_vma, closed_link) { @@ -1104,28 +1116,26 @@ void i915_vma_parked(struct intel_gt *gt) if (!kref_get_unless_zero(&obj->base.refcount)) continue; - if (i915_vm_tryopen(vm)) { - list_del_init(&vma->closed_link); - } else { + if (!i915_vm_tryopen(vm)) { i915_gem_object_put(obj); - obj = NULL; + continue; } - spin_unlock_irq(>->closed_lock); + list_move(&vma->closed_link, &closed); + } + spin_unlock_irq(>->closed_lock); - if (obj) { - __i915_vma_put(vma); - i915_gem_object_put(obj); - } + /* As the GT is held idle, no vma can be reopened as we destroy them */ + list_for_each_entry_safe(vma, next, &closed, closed_link) { + struct drm_i915_gem_object *obj = vma->obj; + struct i915_address_space *vm = vma->vm; - i915_vm_close(vm); + INIT_LIST_HEAD(&vma->closed_link); + __i915_vma_put(vma); - /* Restart after dropping lock */ - spin_lock_irq(>->closed_lock); - next = list_first_entry(>->closed_vma, - typeof(*next), closed_link); + i915_gem_object_put(obj); + i915_vm_close(vm); } - spin_unlock_irq(>->closed_lock); } static void __i915_vma_iounmap(struct i915_vma *vma) @@ -1169,7 +1179,8 @@ int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) GEM_BUG_ON(!i915_vma_is_pinned(vma)); /* Wait for the vma to be bound before we start! */ - err = i915_request_await_active(rq, &vma->active); + err = i915_request_await_active(rq, &vma->active, + I915_ACTIVE_AWAIT_EXCL); if (err) return err; @@ -1210,6 +1221,10 @@ int i915_vma_move_to_active(struct i915_vma *vma, dma_resv_add_shared_fence(vma->resv, &rq->fence); obj->write_domain = 0; } + + if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence) + i915_active_add_request(&vma->fence->active, rq); + obj->read_domains |= I915_GEM_GPU_DOMAINS; obj->mm.dirty = true; @@ -1223,18 +1238,6 @@ int __i915_vma_unbind(struct i915_vma *vma) lockdep_assert_held(&vma->vm->mutex); - /* - * First wait upon any activity as retiring the request may - * have side-effects such as unpinning or even unbinding this vma. - * - * XXX Actually waiting under the vm->mutex is a hinderance and - * should be pipelined wherever possible. In cases where that is - * unavoidable, we should lift the wait to before the mutex. - */ - ret = i915_vma_sync(vma); - if (ret) - return ret; - if (i915_vma_is_pinned(vma)) { vma_print_allocator(vma, "is pinned"); return -EAGAIN; @@ -1256,6 +1259,9 @@ int __i915_vma_unbind(struct i915_vma *vma) GEM_BUG_ON(i915_vma_is_active(vma)); if (i915_vma_is_map_and_fenceable(vma)) { + /* Force a pagefault for domain tracking on next user access */ + i915_vma_revoke_mmap(vma); + /* * Check that we have flushed all writes through the GGTT * before the unbind, other due to non-strict nature of those @@ -1272,12 +1278,7 @@ int __i915_vma_unbind(struct i915_vma *vma) i915_vma_flush_writes(vma); /* release the fence reg _after_ flushing */ - ret = i915_vma_revoke_fence(vma); - if (ret) - return ret; - - /* Force a pagefault for domain tracking on next user access */ - i915_vma_revoke_mmap(vma); + i915_vma_revoke_fence(vma); __i915_vma_iounmap(vma); clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); @@ -1308,16 +1309,21 @@ int i915_vma_unbind(struct i915_vma *vma) if (!drm_mm_node_allocated(&vma->node)) return 0; - if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) - /* XXX not always required: nop_clear_range */ - wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); - /* Optimistic wait before taking the mutex */ err = i915_vma_sync(vma); if (err) goto out_rpm; - err = mutex_lock_interruptible(&vm->mutex); + if (i915_vma_is_pinned(vma)) { + vma_print_allocator(vma, "is pinned"); + return -EAGAIN; + } + + if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) + /* XXX not always required: nop_clear_range */ + wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); + + err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref); if (err) goto out_rpm; diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h index e1ced1df13e1..8ad1daabcd58 100644 --- a/drivers/gpu/drm/i915/i915_vma.h +++ b/drivers/gpu/drm/i915/i915_vma.h @@ -30,10 +30,10 @@ #include <drm/drm_mm.h> +#include "gt/intel_ggtt_fencing.h" #include "gem/i915_gem_object.h" #include "i915_gem_gtt.h" -#include "i915_gem_fence_reg.h" #include "i915_active.h" #include "i915_request.h" @@ -326,7 +326,7 @@ static inline struct page *i915_vma_first_page(struct i915_vma *vma) * True if the vma has a fence, false otherwise. */ int __must_check i915_vma_pin_fence(struct i915_vma *vma); -int __must_check i915_vma_revoke_fence(struct i915_vma *vma); +void i915_vma_revoke_fence(struct i915_vma *vma); int __i915_vma_pin_fence(struct i915_vma *vma); diff --git a/drivers/gpu/drm/i915/i915_vma_types.h b/drivers/gpu/drm/i915/i915_vma_types.h index e0942efd5236..63831cdb7402 100644 --- a/drivers/gpu/drm/i915/i915_vma_types.h +++ b/drivers/gpu/drm/i915/i915_vma_types.h @@ -273,21 +273,10 @@ struct i915_vma { struct rb_node obj_node; struct hlist_node obj_hash; - /** This vma's place in the execbuf reservation list */ - struct list_head exec_link; - struct list_head reloc_link; - /** This vma's place in the eviction list */ struct list_head evict_link; struct list_head closed_link; - - /** - * Used for performing relocations during execbuffer insertion. - */ - unsigned int *exec_flags; - struct hlist_node exec_node; - u32 exec_handle; }; #endif diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 8e99ad097830..91bb7891c70c 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -23,6 +23,7 @@ */ #include <drm/drm_print.h> +#include <drm/i915_pciids.h> #include "display/intel_cdclk.h" #include "intel_device_info.h" @@ -97,6 +98,7 @@ void intel_device_info_print_static(const struct intel_device_info *info, drm_printf(p, "platform: %s\n", intel_platform_name(info->platform)); drm_printf(p, "ppgtt-size: %d\n", info->ppgtt_size); drm_printf(p, "ppgtt-type: %d\n", info->ppgtt_type); + drm_printf(p, "dma_mask_size: %u\n", info->dma_mask_size); #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, yesno(info->name)); DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG); @@ -979,35 +981,32 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) drm_info(&dev_priv->drm, "Display fused off, disabling\n"); info->pipe_mask = 0; + info->cpu_transcoder_mask = 0; } else if (fuse_strap & IVB_PIPE_C_DISABLE) { drm_info(&dev_priv->drm, "PipeC fused off\n"); info->pipe_mask &= ~BIT(PIPE_C); + info->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); } } else if (HAS_DISPLAY(dev_priv) && INTEL_GEN(dev_priv) >= 9) { u32 dfsm = I915_READ(SKL_DFSM); - u8 enabled_mask = info->pipe_mask; - - if (dfsm & SKL_DFSM_PIPE_A_DISABLE) - enabled_mask &= ~BIT(PIPE_A); - if (dfsm & SKL_DFSM_PIPE_B_DISABLE) - enabled_mask &= ~BIT(PIPE_B); - if (dfsm & SKL_DFSM_PIPE_C_DISABLE) - enabled_mask &= ~BIT(PIPE_C); - if (INTEL_GEN(dev_priv) >= 12 && - (dfsm & TGL_DFSM_PIPE_D_DISABLE)) - enabled_mask &= ~BIT(PIPE_D); - /* - * At least one pipe should be enabled and if there are - * disabled pipes, they should be the last ones, with no holes - * in the mask. - */ - if (enabled_mask == 0 || !is_power_of_2(enabled_mask + 1)) - drm_err(&dev_priv->drm, - "invalid pipe fuse configuration: enabled_mask=0x%x\n", - enabled_mask); - else - info->pipe_mask = enabled_mask; + if (dfsm & SKL_DFSM_PIPE_A_DISABLE) { + info->pipe_mask &= ~BIT(PIPE_A); + info->cpu_transcoder_mask &= ~BIT(TRANSCODER_A); + } + if (dfsm & SKL_DFSM_PIPE_B_DISABLE) { + info->pipe_mask &= ~BIT(PIPE_B); + info->cpu_transcoder_mask &= ~BIT(TRANSCODER_B); + } + if (dfsm & SKL_DFSM_PIPE_C_DISABLE) { + info->pipe_mask &= ~BIT(PIPE_C); + info->cpu_transcoder_mask &= ~BIT(TRANSCODER_C); + } + if (INTEL_GEN(dev_priv) >= 12 && + (dfsm & TGL_DFSM_PIPE_D_DISABLE)) { + info->pipe_mask &= ~BIT(PIPE_D); + info->cpu_transcoder_mask &= ~BIT(TRANSCODER_D); + } if (dfsm & SKL_DFSM_DISPLAY_HDCP_DISABLE) info->display.has_hdcp = 0; diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h index 1ecb9df2de91..69c9257c6c6a 100644 --- a/drivers/gpu/drm/i915/intel_device_info.h +++ b/drivers/gpu/drm/i915/intel_device_info.h @@ -158,6 +158,8 @@ struct intel_device_info { enum intel_platform platform; + unsigned int dma_mask_size; /* available DMA address bits */ + enum intel_ppgtt_type ppgtt_type; unsigned int ppgtt_size; /* log2, e.g. 31/32/48 bits */ @@ -168,6 +170,7 @@ struct intel_device_info { u32 display_mmio_offset; u8 pipe_mask; + u8 cpu_transcoder_mask; #define DEFINE_FLAG(name) u8 name:1 DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c new file mode 100644 index 000000000000..8aa12cad93ce --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dram.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include "i915_drv.h" +#include "intel_dram.h" + +struct dram_dimm_info { + u8 size, width, ranks; +}; + +struct dram_channel_info { + struct dram_dimm_info dimm_l, dimm_s; + u8 ranks; + bool is_16gb_dimm; +}; + +#define DRAM_TYPE_STR(type) [INTEL_DRAM_ ## type] = #type + +static const char *intel_dram_type_str(enum intel_dram_type type) +{ + static const char * const str[] = { + DRAM_TYPE_STR(UNKNOWN), + DRAM_TYPE_STR(DDR3), + DRAM_TYPE_STR(DDR4), + DRAM_TYPE_STR(LPDDR3), + DRAM_TYPE_STR(LPDDR4), + }; + + if (type >= ARRAY_SIZE(str)) + type = INTEL_DRAM_UNKNOWN; + + return str[type]; +} + +#undef DRAM_TYPE_STR + +static int intel_dimm_num_devices(const struct dram_dimm_info *dimm) +{ + return dimm->ranks * 64 / (dimm->width ?: 1); +} + +/* Returns total GB for the whole DIMM */ +static int skl_get_dimm_size(u16 val) +{ + return val & SKL_DRAM_SIZE_MASK; +} + +static int skl_get_dimm_width(u16 val) +{ + if (skl_get_dimm_size(val) == 0) + return 0; + + switch (val & SKL_DRAM_WIDTH_MASK) { + case SKL_DRAM_WIDTH_X8: + case SKL_DRAM_WIDTH_X16: + case SKL_DRAM_WIDTH_X32: + val = (val & SKL_DRAM_WIDTH_MASK) >> SKL_DRAM_WIDTH_SHIFT; + return 8 << val; + default: + MISSING_CASE(val); + return 0; + } +} + +static int skl_get_dimm_ranks(u16 val) +{ + if (skl_get_dimm_size(val) == 0) + return 0; + + val = (val & SKL_DRAM_RANK_MASK) >> SKL_DRAM_RANK_SHIFT; + + return val + 1; +} + +/* Returns total GB for the whole DIMM */ +static int cnl_get_dimm_size(u16 val) +{ + return (val & CNL_DRAM_SIZE_MASK) / 2; +} + +static int cnl_get_dimm_width(u16 val) +{ + if (cnl_get_dimm_size(val) == 0) + return 0; + + switch (val & CNL_DRAM_WIDTH_MASK) { + case CNL_DRAM_WIDTH_X8: + case CNL_DRAM_WIDTH_X16: + case CNL_DRAM_WIDTH_X32: + val = (val & CNL_DRAM_WIDTH_MASK) >> CNL_DRAM_WIDTH_SHIFT; + return 8 << val; + default: + MISSING_CASE(val); + return 0; + } +} + +static int cnl_get_dimm_ranks(u16 val) +{ + if (cnl_get_dimm_size(val) == 0) + return 0; + + val = (val & CNL_DRAM_RANK_MASK) >> CNL_DRAM_RANK_SHIFT; + + return val + 1; +} + +static bool +skl_is_16gb_dimm(const struct dram_dimm_info *dimm) +{ + /* Convert total GB to Gb per DRAM device */ + return 8 * dimm->size / (intel_dimm_num_devices(dimm) ?: 1) == 16; +} + +static void +skl_dram_get_dimm_info(struct drm_i915_private *i915, + struct dram_dimm_info *dimm, + int channel, char dimm_name, u16 val) +{ + if (INTEL_GEN(i915) >= 10) { + dimm->size = cnl_get_dimm_size(val); + dimm->width = cnl_get_dimm_width(val); + dimm->ranks = cnl_get_dimm_ranks(val); + } else { + dimm->size = skl_get_dimm_size(val); + dimm->width = skl_get_dimm_width(val); + dimm->ranks = skl_get_dimm_ranks(val); + } + + drm_dbg_kms(&i915->drm, + "CH%u DIMM %c size: %u GB, width: X%u, ranks: %u, 16Gb DIMMs: %s\n", + channel, dimm_name, dimm->size, dimm->width, dimm->ranks, + yesno(skl_is_16gb_dimm(dimm))); +} + +static int +skl_dram_get_channel_info(struct drm_i915_private *i915, + struct dram_channel_info *ch, + int channel, u32 val) +{ + skl_dram_get_dimm_info(i915, &ch->dimm_l, + channel, 'L', val & 0xffff); + skl_dram_get_dimm_info(i915, &ch->dimm_s, + channel, 'S', val >> 16); + + if (ch->dimm_l.size == 0 && ch->dimm_s.size == 0) { + drm_dbg_kms(&i915->drm, "CH%u not populated\n", channel); + return -EINVAL; + } + + if (ch->dimm_l.ranks == 2 || ch->dimm_s.ranks == 2) + ch->ranks = 2; + else if (ch->dimm_l.ranks == 1 && ch->dimm_s.ranks == 1) + ch->ranks = 2; + else + ch->ranks = 1; + + ch->is_16gb_dimm = skl_is_16gb_dimm(&ch->dimm_l) || + skl_is_16gb_dimm(&ch->dimm_s); + + drm_dbg_kms(&i915->drm, "CH%u ranks: %u, 16Gb DIMMs: %s\n", + channel, ch->ranks, yesno(ch->is_16gb_dimm)); + + return 0; +} + +static bool +intel_is_dram_symmetric(const struct dram_channel_info *ch0, + const struct dram_channel_info *ch1) +{ + return !memcmp(ch0, ch1, sizeof(*ch0)) && + (ch0->dimm_s.size == 0 || + !memcmp(&ch0->dimm_l, &ch0->dimm_s, sizeof(ch0->dimm_l))); +} + +static int +skl_dram_get_channels_info(struct drm_i915_private *i915) +{ + struct dram_info *dram_info = &i915->dram_info; + struct dram_channel_info ch0 = {}, ch1 = {}; + u32 val; + int ret; + + val = intel_uncore_read(&i915->uncore, + SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN); + ret = skl_dram_get_channel_info(i915, &ch0, 0, val); + if (ret == 0) + dram_info->num_channels++; + + val = intel_uncore_read(&i915->uncore, + SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN); + ret = skl_dram_get_channel_info(i915, &ch1, 1, val); + if (ret == 0) + dram_info->num_channels++; + + if (dram_info->num_channels == 0) { + drm_info(&i915->drm, "Number of memory channels is zero\n"); + return -EINVAL; + } + + /* + * If any of the channel is single rank channel, worst case output + * will be same as if single rank memory, so consider single rank + * memory. + */ + if (ch0.ranks == 1 || ch1.ranks == 1) + dram_info->ranks = 1; + else + dram_info->ranks = max(ch0.ranks, ch1.ranks); + + if (dram_info->ranks == 0) { + drm_info(&i915->drm, "couldn't get memory rank information\n"); + return -EINVAL; + } + + dram_info->is_16gb_dimm = ch0.is_16gb_dimm || ch1.is_16gb_dimm; + + dram_info->symmetric_memory = intel_is_dram_symmetric(&ch0, &ch1); + + drm_dbg_kms(&i915->drm, "Memory configuration is symmetric? %s\n", + yesno(dram_info->symmetric_memory)); + + return 0; +} + +static enum intel_dram_type +skl_get_dram_type(struct drm_i915_private *i915) +{ + u32 val; + + val = intel_uncore_read(&i915->uncore, + SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN); + + switch (val & SKL_DRAM_DDR_TYPE_MASK) { + case SKL_DRAM_DDR_TYPE_DDR3: + return INTEL_DRAM_DDR3; + case SKL_DRAM_DDR_TYPE_DDR4: + return INTEL_DRAM_DDR4; + case SKL_DRAM_DDR_TYPE_LPDDR3: + return INTEL_DRAM_LPDDR3; + case SKL_DRAM_DDR_TYPE_LPDDR4: + return INTEL_DRAM_LPDDR4; + default: + MISSING_CASE(val); + return INTEL_DRAM_UNKNOWN; + } +} + +static int +skl_get_dram_info(struct drm_i915_private *i915) +{ + struct dram_info *dram_info = &i915->dram_info; + u32 mem_freq_khz, val; + int ret; + + dram_info->type = skl_get_dram_type(i915); + drm_dbg_kms(&i915->drm, "DRAM type: %s\n", + intel_dram_type_str(dram_info->type)); + + ret = skl_dram_get_channels_info(i915); + if (ret) + return ret; + + val = intel_uncore_read(&i915->uncore, + SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU); + mem_freq_khz = DIV_ROUND_UP((val & SKL_REQ_DATA_MASK) * + SKL_MEMORY_FREQ_MULTIPLIER_HZ, 1000); + + dram_info->bandwidth_kbps = dram_info->num_channels * + mem_freq_khz * 8; + + if (dram_info->bandwidth_kbps == 0) { + drm_info(&i915->drm, + "Couldn't get system memory bandwidth\n"); + return -EINVAL; + } + + dram_info->valid = true; + return 0; +} + +/* Returns Gb per DRAM device */ +static int bxt_get_dimm_size(u32 val) +{ + switch (val & BXT_DRAM_SIZE_MASK) { + case BXT_DRAM_SIZE_4GBIT: + return 4; + case BXT_DRAM_SIZE_6GBIT: + return 6; + case BXT_DRAM_SIZE_8GBIT: + return 8; + case BXT_DRAM_SIZE_12GBIT: + return 12; + case BXT_DRAM_SIZE_16GBIT: + return 16; + default: + MISSING_CASE(val); + return 0; + } +} + +static int bxt_get_dimm_width(u32 val) +{ + if (!bxt_get_dimm_size(val)) + return 0; + + val = (val & BXT_DRAM_WIDTH_MASK) >> BXT_DRAM_WIDTH_SHIFT; + + return 8 << val; +} + +static int bxt_get_dimm_ranks(u32 val) +{ + if (!bxt_get_dimm_size(val)) + return 0; + + switch (val & BXT_DRAM_RANK_MASK) { + case BXT_DRAM_RANK_SINGLE: + return 1; + case BXT_DRAM_RANK_DUAL: + return 2; + default: + MISSING_CASE(val); + return 0; + } +} + +static enum intel_dram_type bxt_get_dimm_type(u32 val) +{ + if (!bxt_get_dimm_size(val)) + return INTEL_DRAM_UNKNOWN; + + switch (val & BXT_DRAM_TYPE_MASK) { + case BXT_DRAM_TYPE_DDR3: + return INTEL_DRAM_DDR3; + case BXT_DRAM_TYPE_LPDDR3: + return INTEL_DRAM_LPDDR3; + case BXT_DRAM_TYPE_DDR4: + return INTEL_DRAM_DDR4; + case BXT_DRAM_TYPE_LPDDR4: + return INTEL_DRAM_LPDDR4; + default: + MISSING_CASE(val); + return INTEL_DRAM_UNKNOWN; + } +} + +static void bxt_get_dimm_info(struct dram_dimm_info *dimm, u32 val) +{ + dimm->width = bxt_get_dimm_width(val); + dimm->ranks = bxt_get_dimm_ranks(val); + + /* + * Size in register is Gb per DRAM device. Convert to total + * GB to match the way we report this for non-LP platforms. + */ + dimm->size = bxt_get_dimm_size(val) * intel_dimm_num_devices(dimm) / 8; +} + +static int bxt_get_dram_info(struct drm_i915_private *i915) +{ + struct dram_info *dram_info = &i915->dram_info; + u32 dram_channels; + u32 mem_freq_khz, val; + u8 num_active_channels; + int i; + + val = intel_uncore_read(&i915->uncore, BXT_P_CR_MC_BIOS_REQ_0_0_0); + mem_freq_khz = DIV_ROUND_UP((val & BXT_REQ_DATA_MASK) * + BXT_MEMORY_FREQ_MULTIPLIER_HZ, 1000); + + dram_channels = val & BXT_DRAM_CHANNEL_ACTIVE_MASK; + num_active_channels = hweight32(dram_channels); + + /* Each active bit represents 4-byte channel */ + dram_info->bandwidth_kbps = (mem_freq_khz * num_active_channels * 4); + + if (dram_info->bandwidth_kbps == 0) { + drm_info(&i915->drm, + "Couldn't get system memory bandwidth\n"); + return -EINVAL; + } + + /* + * Now read each DUNIT8/9/10/11 to check the rank of each dimms. + */ + for (i = BXT_D_CR_DRP0_DUNIT_START; i <= BXT_D_CR_DRP0_DUNIT_END; i++) { + struct dram_dimm_info dimm; + enum intel_dram_type type; + + val = intel_uncore_read(&i915->uncore, BXT_D_CR_DRP0_DUNIT(i)); + if (val == 0xFFFFFFFF) + continue; + + dram_info->num_channels++; + + bxt_get_dimm_info(&dimm, val); + type = bxt_get_dimm_type(val); + + drm_WARN_ON(&i915->drm, type != INTEL_DRAM_UNKNOWN && + dram_info->type != INTEL_DRAM_UNKNOWN && + dram_info->type != type); + + drm_dbg_kms(&i915->drm, + "CH%u DIMM size: %u GB, width: X%u, ranks: %u, type: %s\n", + i - BXT_D_CR_DRP0_DUNIT_START, + dimm.size, dimm.width, dimm.ranks, + intel_dram_type_str(type)); + + /* + * If any of the channel is single rank channel, + * worst case output will be same as if single rank + * memory, so consider single rank memory. + */ + if (dram_info->ranks == 0) + dram_info->ranks = dimm.ranks; + else if (dimm.ranks == 1) + dram_info->ranks = 1; + + if (type != INTEL_DRAM_UNKNOWN) + dram_info->type = type; + } + + if (dram_info->type == INTEL_DRAM_UNKNOWN || dram_info->ranks == 0) { + drm_info(&i915->drm, "couldn't get memory information\n"); + return -EINVAL; + } + + dram_info->valid = true; + + return 0; +} + +void intel_dram_detect(struct drm_i915_private *i915) +{ + struct dram_info *dram_info = &i915->dram_info; + int ret; + + /* + * Assume 16Gb DIMMs are present until proven otherwise. + * This is only used for the level 0 watermark latency + * w/a which does not apply to bxt/glk. + */ + dram_info->is_16gb_dimm = !IS_GEN9_LP(i915); + + if (INTEL_GEN(i915) < 9 || !HAS_DISPLAY(i915)) + return; + + if (IS_GEN9_LP(i915)) + ret = bxt_get_dram_info(i915); + else + ret = skl_get_dram_info(i915); + if (ret) + return; + + drm_dbg_kms(&i915->drm, "DRAM bandwidth: %u kBps, channels: %u\n", + dram_info->bandwidth_kbps, dram_info->num_channels); + + drm_dbg_kms(&i915->drm, "DRAM ranks: %u, 16Gb DIMMs: %s\n", + dram_info->ranks, yesno(dram_info->is_16gb_dimm)); +} + +static u32 gen9_edram_size_mb(struct drm_i915_private *i915, u32 cap) +{ + static const u8 ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 }; + static const u8 sets[4] = { 1, 1, 2, 2 }; + + return EDRAM_NUM_BANKS(cap) * + ways[EDRAM_WAYS_IDX(cap)] * + sets[EDRAM_SETS_IDX(cap)]; +} + +void intel_dram_edram_detect(struct drm_i915_private *i915) +{ + u32 edram_cap = 0; + + if (!(IS_HASWELL(i915) || IS_BROADWELL(i915) || INTEL_GEN(i915) >= 9)) + return; + + edram_cap = __raw_uncore_read32(&i915->uncore, HSW_EDRAM_CAP); + + /* NB: We can't write IDICR yet because we don't have gt funcs set up */ + + if (!(edram_cap & EDRAM_ENABLED)) + return; + + /* + * The needed capability bits for size calculation are not there with + * pre gen9 so return 128MB always. + */ + if (INTEL_GEN(i915) < 9) + i915->edram_size_mb = 128; + else + i915->edram_size_mb = gen9_edram_size_mb(i915, edram_cap); + + drm_info(&i915->drm, "Found %uMB of eDRAM\n", i915->edram_size_mb); +} diff --git a/drivers/gpu/drm/i915/intel_dram.h b/drivers/gpu/drm/i915/intel_dram.h new file mode 100644 index 000000000000..4ba13c13162c --- /dev/null +++ b/drivers/gpu/drm/i915/intel_dram.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __INTEL_DRAM_H__ +#define __INTEL_DRAM_H__ + +struct drm_i915_private; + +void intel_dram_edram_detect(struct drm_i915_private *i915); +void intel_dram_detect(struct drm_i915_private *i915); + +#endif /* __INTEL_DRAM_H__ */ diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c index 38ebd5562c7c..21b91313cc5d 100644 --- a/drivers/gpu/drm/i915/intel_gvt.c +++ b/drivers/gpu/drm/i915/intel_gvt.c @@ -22,6 +22,7 @@ */ #include "i915_drv.h" +#include "i915_vgpu.h" #include "intel_gvt.h" /** @@ -105,7 +106,7 @@ int intel_gvt_init(struct drm_i915_private *dev_priv) return 0; } - if (USES_GUC_SUBMISSION(dev_priv)) { + if (intel_uc_wants_guc_submission(&dev_priv->gt.uc)) { drm_err(&dev_priv->drm, "i915 GVT-g loading failed due to Graphics virtualization is not yet supported with GuC submission\n"); return -EIO; @@ -124,6 +125,11 @@ bail: return 0; } +static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) +{ + return dev_priv->gvt; +} + /** * intel_gvt_driver_remove - cleanup GVT components when i915 driver is * unbinding diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index ffac0b862ca5..f7bd1dbb625e 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -40,12 +40,37 @@ #include "gt/intel_llc.h" #include "i915_drv.h" +#include "i915_fixed.h" #include "i915_irq.h" #include "i915_trace.h" +#include "display/intel_bw.h" #include "intel_pm.h" #include "intel_sideband.h" #include "../../../platform/x86/intel_ips.h" +/* Stores plane specific WM parameters */ +struct skl_wm_params { + bool x_tiled, y_tiled; + bool rc_surface; + bool is_planar; + u32 width; + u8 cpp; + u32 plane_pixel_rate; + u32 y_min_scanlines; + u32 plane_bytes_per_line; + uint_fixed_16_16_t plane_blocks_per_line; + uint_fixed_16_16_t y_tile_minimum; + u32 linetime_us; + u32 dbuf_block_size; +}; + +/* used in computing the new watermarks state */ +struct intel_wm_config { + unsigned int num_pipes_active; + bool sprites_enabled; + bool sprites_scaled; +}; + static void gen9_init_clock_gating(struct drm_i915_private *dev_priv) { if (HAS_LLC(dev_priv)) { @@ -128,16 +153,6 @@ static void glk_init_clock_gating(struct drm_i915_private *dev_priv) */ I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) | PWM1_GATING_DIS | PWM2_GATING_DIS); - - /* WaDDIIOTimeout:glk */ - if (IS_GLK_REVID(dev_priv, 0, GLK_REVID_A1)) { - u32 val = I915_READ(CHICKEN_MISC_2); - val &= ~(GLK_CL0_PWR_DOWN | - GLK_CL1_PWR_DOWN | - GLK_CL2_PWR_DOWN); - I915_WRITE(CHICKEN_MISC_2, val); - } - } static void pnv_get_mem_freq(struct drm_i915_private *dev_priv) @@ -469,9 +484,9 @@ static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state) struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; enum pipe pipe = crtc->pipe; int sprite0_start, sprite1_start; + u32 dsparb, dsparb2, dsparb3; switch (pipe) { - u32 dsparb, dsparb2, dsparb3; case PIPE_A: dsparb = I915_READ(DSPARB); dsparb2 = I915_READ(DSPARB2); @@ -1969,6 +1984,7 @@ static void vlv_atomic_update_fifo(struct intel_atomic_state *state, const struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state; int sprite0_start, sprite1_start, fifo_size; + u32 dsparb, dsparb2, dsparb3; if (!crtc_state->fifo_changed) return; @@ -1994,7 +2010,6 @@ static void vlv_atomic_update_fifo(struct intel_atomic_state *state, spin_lock(&uncore->lock); switch (crtc->pipe) { - u32 dsparb, dsparb2, dsparb3; case PIPE_A: dsparb = intel_uncore_read_fw(uncore, DSPARB); dsparb2 = intel_uncore_read_fw(uncore, DSPARB2); @@ -2776,7 +2791,7 @@ static bool ilk_validate_wm_level(int level, } static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv, - const struct intel_crtc *intel_crtc, + const struct intel_crtc *crtc, int level, struct intel_crtc_state *crtc_state, const struct intel_plane_state *pristate, @@ -3107,7 +3122,7 @@ static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv, static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct intel_pipe_wm *pipe_wm; struct intel_plane *plane; const struct intel_plane_state *plane_state; @@ -3147,7 +3162,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state) usable_level = 0; memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm)); - ilk_compute_wm_level(dev_priv, intel_crtc, 0, crtc_state, + ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state, pristate, sprstate, curstate, &pipe_wm->wm[0]); if (!ilk_validate_pipe_wm(dev_priv, pipe_wm)) @@ -3158,7 +3173,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *crtc_state) for (level = 1; level <= usable_level; level++) { struct intel_wm_level *wm = &pipe_wm->wm[level]; - ilk_compute_wm_level(dev_priv, intel_crtc, level, crtc_state, + ilk_compute_wm_level(dev_priv, crtc, level, crtc_state, pristate, sprstate, curstate, wm); /* @@ -3743,42 +3758,71 @@ intel_disable_sagv(struct drm_i915_private *dev_priv) return 0; } -bool intel_can_enable_sagv(struct intel_atomic_state *state) +void intel_sagv_pre_plane_update(struct intel_atomic_state *state) { - struct drm_device *dev = state->base.dev; - struct drm_i915_private *dev_priv = to_i915(dev); - struct intel_crtc *crtc; - struct intel_plane *plane; - struct intel_crtc_state *crtc_state; - enum pipe pipe; - int level, latency; - - if (!intel_has_sagv(dev_priv)) - return false; + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_bw_state *new_bw_state; /* - * If there are no active CRTCs, no additional checks need be performed + * Just return if we can't control SAGV or don't have it. + * This is different from situation when we have SAGV but just can't + * afford it due to DBuf limitation - in case if SAGV is completely + * disabled in a BIOS, we are not even allowed to send a PCode request, + * as it will throw an error. So have to check it here. */ - if (hweight8(state->active_pipes) == 0) - return true; + if (!intel_has_sagv(dev_priv)) + return; + + new_bw_state = intel_atomic_get_new_bw_state(state); + if (!new_bw_state) + return; + + if (!intel_can_enable_sagv(dev_priv, new_bw_state)) + intel_disable_sagv(dev_priv); +} + +void intel_sagv_post_plane_update(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + const struct intel_bw_state *new_bw_state; /* - * SKL+ workaround: bspec recommends we disable SAGV when we have - * more then one pipe enabled + * Just return if we can't control SAGV or don't have it. + * This is different from situation when we have SAGV but just can't + * afford it due to DBuf limitation - in case if SAGV is completely + * disabled in a BIOS, we are not even allowed to send a PCode request, + * as it will throw an error. So have to check it here. */ - if (hweight8(state->active_pipes) > 1) + if (!intel_has_sagv(dev_priv)) + return; + + new_bw_state = intel_atomic_get_new_bw_state(state); + if (!new_bw_state) + return; + + if (intel_can_enable_sagv(dev_priv, new_bw_state)) + intel_enable_sagv(dev_priv); +} + +static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct intel_plane *plane; + const struct intel_plane_state *plane_state; + int level, latency; + + if (!intel_has_sagv(dev_priv)) return false; - /* Since we're now guaranteed to only have one active CRTC... */ - pipe = ffs(state->active_pipes) - 1; - crtc = intel_get_crtc_for_pipe(dev_priv, pipe); - crtc_state = to_intel_crtc_state(crtc->base.state); + if (!crtc_state->hw.active) + return true; if (crtc_state->hw.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) return false; - for_each_intel_plane_on_crtc(dev, crtc, plane) { - struct skl_plane_wm *wm = + intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { + const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane->id]; /* Skip this plane if it's not enabled */ @@ -3793,7 +3837,7 @@ bool intel_can_enable_sagv(struct intel_atomic_state *state) latency = dev_priv->wm.skl_latency[level]; if (skl_needs_memory_bw_wa(dev_priv) && - plane->base.state->fb->modifier == + plane_state->uapi.fb->modifier == I915_FORMAT_MOD_X_TILED) latency += 15; @@ -3809,6 +3853,71 @@ bool intel_can_enable_sagv(struct intel_atomic_state *state) return true; } +static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state) +{ + return skl_crtc_can_enable_sagv(crtc_state); +} + +bool intel_can_enable_sagv(struct drm_i915_private *dev_priv, + const struct intel_bw_state *bw_state) +{ + if (INTEL_GEN(dev_priv) < 11 && + bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes)) + return false; + + return bw_state->pipe_sagv_reject == 0; +} + +static int intel_compute_sagv_mask(struct intel_atomic_state *state) +{ + struct drm_i915_private *dev_priv = to_i915(state->base.dev); + int ret; + struct intel_crtc *crtc; + const struct intel_crtc_state *new_crtc_state; + struct intel_bw_state *new_bw_state = NULL; + const struct intel_bw_state *old_bw_state = NULL; + int i; + + for_each_new_intel_crtc_in_state(state, crtc, + new_crtc_state, i) { + new_bw_state = intel_atomic_get_bw_state(state); + if (IS_ERR(new_bw_state)) + return PTR_ERR(new_bw_state); + + old_bw_state = intel_atomic_get_old_bw_state(state); + + if (intel_crtc_can_enable_sagv(new_crtc_state)) + new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe); + else + new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe); + } + + if (!new_bw_state) + return 0; + + new_bw_state->active_pipes = + intel_calc_active_pipes(state, old_bw_state->active_pipes); + + if (new_bw_state->active_pipes != old_bw_state->active_pipes) { + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; + } + + if (intel_can_enable_sagv(dev_priv, new_bw_state) != + intel_can_enable_sagv(dev_priv, old_bw_state)) { + ret = intel_atomic_serialize_global_state(&new_bw_state->base); + if (ret) + return ret; + } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) { + ret = intel_atomic_lock_global_state(&new_bw_state->base); + if (ret) + return ret; + } + + return 0; +} + /* * Calculate initial DBuf slice offset, based on slice size * and mask(i.e if slice size is 1024 and second slice is enabled @@ -3843,7 +3952,7 @@ static u16 intel_get_ddb_size(struct drm_i915_private *dev_priv) } static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state, - u32 active_pipes); + u8 active_pipes); static void skl_ddb_get_pipe_allocation_limits(struct drm_i915_private *dev_priv, @@ -4002,6 +4111,7 @@ static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state, int color_plane); static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, int level, + unsigned int latency, const struct skl_wm_params *wp, const struct skl_wm_level *result_prev, struct skl_wm_level *result /* out */); @@ -4024,7 +4134,9 @@ skl_cursor_allocation(const struct intel_crtc_state *crtc_state, drm_WARN_ON(&dev_priv->drm, ret); for (level = 0; level <= max_level; level++) { - skl_compute_plane_wm(crtc_state, level, &wp, &wm, &wm); + unsigned int latency = dev_priv->wm.skl_latency[level]; + + skl_compute_plane_wm(crtc_state, level, latency, &wp, &wm, &wm); if (wm.min_ddb_alloc == U16_MAX) break; @@ -4178,56 +4290,57 @@ struct dbuf_slice_conf_entry { * as is from BSpec itself - that way it is at least easier * to compare, change and check. */ -static struct dbuf_slice_conf_entry icl_allowed_dbufs[] = +static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] = /* Autogenerated with igt/tools/intel_dbuf_map tool: */ { { .active_pipes = BIT(PIPE_A), .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) - } + [PIPE_A] = BIT(DBUF_S1), + }, }, { .active_pipes = BIT(PIPE_B), .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) - } + [PIPE_B] = BIT(DBUF_S1), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), - [PIPE_B] = BIT(DBUF_S2) - } + [PIPE_B] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_C), .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), .dbuf_mask = { [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, + {} }; /* @@ -4240,106 +4353,106 @@ static struct dbuf_slice_conf_entry icl_allowed_dbufs[] = * as is from BSpec itself - that way it is at least easier * to compare, change and check. */ -static struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = +static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = /* Autogenerated with igt/tools/intel_dbuf_map tool: */ { { .active_pipes = BIT(PIPE_A), .dbuf_mask = { - [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) - } + [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_B), .dbuf_mask = { - [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) - } + [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_B), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S2), - [PIPE_B] = BIT(DBUF_S1) - } + [PIPE_B] = BIT(DBUF_S1), + }, }, { .active_pipes = BIT(PIPE_C), .dbuf_mask = { - [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1) - } + [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_C), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_B) | BIT(PIPE_C), .dbuf_mask = { [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), [PIPE_B] = BIT(DBUF_S1), - [PIPE_C] = BIT(DBUF_S2) - } + [PIPE_C] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_D), .dbuf_mask = { - [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1) - } + [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_D), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_B) | BIT(PIPE_D), .dbuf_mask = { [PIPE_B] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), [PIPE_B] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_C) | BIT(PIPE_D), .dbuf_mask = { [PIPE_C] = BIT(DBUF_S1), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D), .dbuf_mask = { [PIPE_A] = BIT(DBUF_S1), [PIPE_C] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), .dbuf_mask = { [PIPE_B] = BIT(DBUF_S1), [PIPE_C] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, { .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), @@ -4347,19 +4460,18 @@ static struct dbuf_slice_conf_entry tgl_allowed_dbufs[] = [PIPE_A] = BIT(DBUF_S1), [PIPE_B] = BIT(DBUF_S1), [PIPE_C] = BIT(DBUF_S2), - [PIPE_D] = BIT(DBUF_S2) - } + [PIPE_D] = BIT(DBUF_S2), + }, }, + {} }; -static u8 compute_dbuf_slices(enum pipe pipe, - u32 active_pipes, - const struct dbuf_slice_conf_entry *dbuf_slices, - int size) +static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, + const struct dbuf_slice_conf_entry *dbuf_slices) { int i; - for (i = 0; i < size; i++) { + for (i = 0; i < dbuf_slices[i].active_pipes; i++) { if (dbuf_slices[i].active_pipes == active_pipes) return dbuf_slices[i].dbuf_mask[pipe]; } @@ -4371,8 +4483,7 @@ static u8 compute_dbuf_slices(enum pipe pipe, * returns correspondent DBuf slice mask as stated in BSpec for particular * platform. */ -static u32 icl_compute_dbuf_slices(enum pipe pipe, - u32 active_pipes) +static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes) { /* * FIXME: For ICL this is still a bit unclear as prev BSpec revision @@ -4386,32 +4497,25 @@ static u32 icl_compute_dbuf_slices(enum pipe pipe, * still here - we will need it once those additional constraints * pop up. */ - return compute_dbuf_slices(pipe, active_pipes, - icl_allowed_dbufs, - ARRAY_SIZE(icl_allowed_dbufs)); + return compute_dbuf_slices(pipe, active_pipes, icl_allowed_dbufs); } -static u32 tgl_compute_dbuf_slices(enum pipe pipe, - u32 active_pipes) +static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes) { - return compute_dbuf_slices(pipe, active_pipes, - tgl_allowed_dbufs, - ARRAY_SIZE(tgl_allowed_dbufs)); + return compute_dbuf_slices(pipe, active_pipes, tgl_allowed_dbufs); } static u8 skl_compute_dbuf_slices(const struct intel_crtc_state *crtc_state, - u32 active_pipes) + u8 active_pipes) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; if (IS_GEN(dev_priv, 12)) - return tgl_compute_dbuf_slices(pipe, - active_pipes); + return tgl_compute_dbuf_slices(pipe, active_pipes); else if (IS_GEN(dev_priv, 11)) - return icl_compute_dbuf_slices(pipe, - active_pipes); + return icl_compute_dbuf_slices(pipe, active_pipes); /* * For anything else just return one slice yet. * Should be extended for other platforms. @@ -4470,14 +4574,10 @@ skl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, u64 *plane_data_rate, u64 *uv_plane_data_rate) { - struct drm_atomic_state *state = crtc_state->uapi.state; struct intel_plane *plane; const struct intel_plane_state *plane_state; u64 total_data_rate = 0; - if (WARN_ON(!state)) - return 0; - /* Calculate and cache data rate for each plane */ intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { enum plane_id plane_id = plane->id; @@ -4505,9 +4605,6 @@ icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state; u64 total_data_rate = 0; - if (WARN_ON(!crtc_state->uapi.state)) - return 0; - /* Calculate and cache data rate for each plane */ intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) { enum plane_id plane_id = plane->id; @@ -4545,13 +4642,22 @@ icl_get_total_relative_data_rate(struct intel_crtc_state *crtc_state, return total_data_rate; } +static const struct skl_wm_level * +skl_plane_wm_level(const struct intel_crtc_state *crtc_state, + enum plane_id plane_id, + int level) +{ + const struct skl_plane_wm *wm = + &crtc_state->wm.skl.optimal.planes[plane_id]; + + return &wm->wm[level]; +} + static int skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) { - struct drm_atomic_state *state = crtc_state->uapi.state; - struct drm_crtc *crtc = crtc_state->uapi.crtc; - struct drm_i915_private *dev_priv = to_i915(crtc->dev); - struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); struct skl_ddb_entry *alloc = &crtc_state->wm.skl.ddb; u16 alloc_size, start = 0; u16 total[I915_MAX_PLANES] = {}; @@ -4568,9 +4674,6 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y)); memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv)); - if (drm_WARN_ON(&dev_priv->drm, !state)) - return 0; - if (!crtc_state->hw.active) { alloc->start = alloc->end = 0; return 0; @@ -4609,7 +4712,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) */ for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) { blocks = 0; - for_each_plane_id_on_crtc(intel_crtc, plane_id) { + for_each_plane_id_on_crtc(crtc, plane_id) { const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; @@ -4646,7 +4749,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) * watermark level, plus an extra share of the leftover blocks * proportional to its relative data rate. */ - for_each_plane_id_on_crtc(intel_crtc, plane_id) { + for_each_plane_id_on_crtc(crtc, plane_id) { const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; u64 rate; @@ -4685,7 +4788,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) /* Set the actual DDB start/end points for each plane */ start = alloc->start; - for_each_plane_id_on_crtc(intel_crtc, plane_id) { + for_each_plane_id_on_crtc(crtc, plane_id) { struct skl_ddb_entry *plane_alloc = &crtc_state->wm.skl.plane_ddb_y[plane_id]; struct skl_ddb_entry *uv_plane_alloc = @@ -4719,7 +4822,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) * that aren't actually possible. */ for (level++; level <= ilk_wm_max_level(dev_priv); level++) { - for_each_plane_id_on_crtc(intel_crtc, plane_id) { + for_each_plane_id_on_crtc(crtc, plane_id) { struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; @@ -4756,7 +4859,7 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *crtc_state) * Go back and disable the transition watermark if it turns out we * don't have enough DDB blocks for it. */ - for_each_plane_id_on_crtc(intel_crtc, plane_id) { + for_each_plane_id_on_crtc(crtc, plane_id) { struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id]; @@ -4786,7 +4889,7 @@ skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate, wm_intermediate_val = latency * pixel_rate * cpp; ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size); - if (INTEL_GEN(dev_priv) >= 10) + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) ret = add_fixed16_u32(ret, 1); return ret; @@ -4921,18 +5024,19 @@ skl_compute_wm_params(const struct intel_crtc_state *crtc_state, wp->y_min_scanlines, wp->dbuf_block_size); - if (INTEL_GEN(dev_priv) >= 10) + if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) interm_pbpl++; wp->plane_blocks_per_line = div_fixed16(interm_pbpl, wp->y_min_scanlines); - } else if (wp->x_tiled && IS_GEN(dev_priv, 9)) { - interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, - wp->dbuf_block_size); - wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); } else { interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line, - wp->dbuf_block_size) + 1; + wp->dbuf_block_size); + + if (!wp->x_tiled || + INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) + interm_pbpl++; + wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl); } @@ -4978,12 +5082,12 @@ static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level) static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, int level, + unsigned int latency, const struct skl_wm_params *wp, const struct skl_wm_level *result_prev, struct skl_wm_level *result /* out */) { struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); - u32 latency = dev_priv->wm.skl_latency[level]; uint_fixed_16_16_t method1, method2; uint_fixed_16_16_t selected_result; u32 res_blocks, res_lines, min_ddb_alloc = 0; @@ -4998,7 +5102,7 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state, * WaIncreaseLatencyIPCEnabled: kbl,cfl * Display WA #1141: kbl,cfl */ - if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) || + if ((IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) && dev_priv->ipc_enabled) latency += 4; @@ -5112,9 +5216,10 @@ skl_compute_wm_levels(const struct intel_crtc_state *crtc_state, for (level = 0; level <= max_level; level++) { struct skl_wm_level *result = &levels[level]; + unsigned int latency = dev_priv->wm.skl_latency[level]; - skl_compute_plane_wm(crtc_state, level, wm_params, - result_prev, result); + skl_compute_plane_wm(crtc_state, level, latency, + wm_params, result_prev, result); result_prev = result; } @@ -5126,21 +5231,30 @@ static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state, { struct drm_device *dev = crtc_state->uapi.crtc->dev; const struct drm_i915_private *dev_priv = to_i915(dev); - u16 trans_min, trans_y_tile_min; - const u16 trans_amount = 10; /* This is configurable amount */ + u16 trans_min, trans_amount, trans_y_tile_min; u16 wm0_sel_res_b, trans_offset_b, res_blocks; - /* Transition WM are not recommended by HW team for GEN9 */ - if (INTEL_GEN(dev_priv) <= 9) - return; - /* Transition WM don't make any sense if ipc is disabled */ if (!dev_priv->ipc_enabled) return; - trans_min = 14; + /* + * WaDisableTWM:skl,kbl,cfl,bxt + * Transition WM are not recommended by HW team for GEN9 + */ + if (IS_GEN9_BC(dev_priv) || IS_BROXTON(dev_priv)) + return; + if (INTEL_GEN(dev_priv) >= 11) trans_min = 4; + else + trans_min = 14; + + /* Display WA #1140: glk,cnl */ + if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv)) + trans_amount = 0; + else + trans_amount = 10; /* This is configurable amount */ trans_offset_b = trans_min + trans_amount; @@ -5163,11 +5277,6 @@ static void skl_compute_transition_wm(const struct intel_crtc_state *crtc_state, trans_offset_b; } else { res_blocks = wm0_sel_res_b + trans_offset_b; - - /* WA BUG:1938466 add one block for non y-tile planes */ - if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_A0)) - res_blocks += 1; - } /* @@ -5352,8 +5461,12 @@ void skl_write_plane_wm(struct intel_plane *plane, &crtc_state->wm.skl.plane_ddb_uv[plane_id]; for (level = 0; level <= max_level; level++) { + const struct skl_wm_level *wm_level; + + wm_level = skl_plane_wm_level(crtc_state, plane_id, level); + skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level), - &wm->wm[level]); + wm_level); } skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id), &wm->trans_wm); @@ -5386,8 +5499,12 @@ void skl_write_cursor_wm(struct intel_plane *plane, &crtc_state->wm.skl.plane_ddb_y[plane_id]; for (level = 0; level <= max_level; level++) { + const struct skl_wm_level *wm_level; + + wm_level = skl_plane_wm_level(crtc_state, plane_id, level); + skl_write_wm_level(dev_priv, CUR_WM(pipe, level), - &wm->wm[level]); + wm_level); } skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe), &wm->trans_wm); @@ -5410,16 +5527,20 @@ static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv, int level, max_level = ilk_wm_max_level(dev_priv); for (level = 0; level <= max_level; level++) { - if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]) || - !skl_wm_level_equals(&wm1->uv_wm[level], &wm2->uv_wm[level])) + /* + * We don't check uv_wm as the hardware doesn't actually + * use it. It only gets used for calculating the required + * ddb allocation. + */ + if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level])) return false; } return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm); } -static inline bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, - const struct skl_ddb_entry *b) +static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a, + const struct skl_ddb_entry *b) { return a->start < b->end && b->start < a->end; } @@ -5768,16 +5889,28 @@ skl_compute_wm(struct intel_atomic_state *state) ret = skl_build_pipe_wm(new_crtc_state); if (ret) return ret; - - ret = skl_wm_add_affected_planes(state, crtc); - if (ret) - return ret; } ret = skl_compute_ddb(state); if (ret) return ret; + ret = intel_compute_sagv_mask(state); + if (ret) + return ret; + + /* + * skl_compute_ddb() will have adjusted the final watermarks + * based on how much ddb is available. Now we can actually + * check if the final watermarks changed. + */ + for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, + new_crtc_state, i) { + ret = skl_wm_add_affected_planes(state, crtc); + if (ret) + return ret; + } + skl_print_wm_changes(state); return 0; @@ -5862,8 +5995,7 @@ static void ilk_optimize_watermarks(struct intel_atomic_state *state, mutex_unlock(&dev_priv->wm.wm_mutex); } -static inline void skl_wm_level_from_reg_val(u32 val, - struct skl_wm_level *level) +static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level) { level->plane_en = val & PLANE_WM_EN; level->ignore_lines = val & PLANE_WM_IGNORE_LINES; @@ -6812,21 +6944,6 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv) I915_WRITE(GEN10_DFR_RATIO_EN_AND_CHICKEN, I915_READ(GEN10_DFR_RATIO_EN_AND_CHICKEN) & ~DFR_DISABLE); - /* WaEnable32PlaneMode:icl */ - I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, - _MASKED_BIT_ENABLE(GEN11_ENABLE_32_PLANE_MODE)); - - /* - * Wa_1408615072:icl,ehl (vsunit) - * Wa_1407596294:icl,ehl (hsunit) - */ - intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE, - 0, VSUNIT_CLKGATE_DIS | HSUNIT_CLKGATE_DIS); - - /* Wa_1407352427:icl,ehl */ - intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2, - 0, PSDUNIT_CLKGATE_DIS); - /*Wa_14010594013:icl, ehl */ intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, CNL_DELAY_PMRSP); @@ -6837,10 +6954,6 @@ static void tgl_init_clock_gating(struct drm_i915_private *dev_priv) u32 vd_pg_enable = 0; unsigned int i; - /* Wa_1408615072:tgl */ - intel_uncore_rmw(&dev_priv->uncore, UNSLICE_UNIT_LEVEL_CLKGATE2, - 0, VSUNIT_CLKGATE_DIS_TGL); - /* This is not a WA. Enable VD HCP & MFX_ENC powergate */ for (i = 0; i < I915_MAX_VCS; i++) { if (HAS_ENGINE(dev_priv, _VCS(i))) @@ -6855,6 +6968,10 @@ static void tgl_init_clock_gating(struct drm_i915_private *dev_priv) if (IS_TGL_REVID(dev_priv, TGL_REVID_A0, TGL_REVID_A0)) I915_WRITE(GEN9_CLKGATE_DIS_3, I915_READ(GEN9_CLKGATE_DIS_3) | TGL_VRH_GATING_DIS); + + /* Wa_14011059788:tgl */ + intel_uncore_rmw(&dev_priv->uncore, GEN10_DFR_RATIO_EN_AND_CHICKEN, + 0, DFR_DISABLE); } static void cnp_init_clock_gating(struct drm_i915_private *dev_priv) @@ -6887,9 +7004,6 @@ static void cnl_init_clock_gating(struct drm_i915_private *dev_priv) val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE); /* ReadHitWriteOnlyDisable:cnl */ val |= RCCUNIT_CLKGATE_DIS; - /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */ - if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0)) - val |= SARBUNIT_CLKGATE_DIS; I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val); /* Wa_2201832410:cnl */ diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h index d60a85421c5a..614ac7f8d4cc 100644 --- a/drivers/gpu/drm/i915/intel_pm.h +++ b/drivers/gpu/drm/i915/intel_pm.h @@ -9,6 +9,7 @@ #include <linux/types.h> #include "i915_reg.h" +#include "display/intel_bw.h" struct drm_device; struct drm_i915_private; @@ -41,9 +42,12 @@ void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc, struct skl_pipe_wm *out); void g4x_wm_sanitize(struct drm_i915_private *dev_priv); void vlv_wm_sanitize(struct drm_i915_private *dev_priv); -bool intel_can_enable_sagv(struct intel_atomic_state *state); +bool intel_can_enable_sagv(struct drm_i915_private *dev_priv, + const struct intel_bw_state *bw_state); int intel_enable_sagv(struct drm_i915_private *dev_priv); int intel_disable_sagv(struct drm_i915_private *dev_priv); +void intel_sagv_pre_plane_update(struct intel_atomic_state *state); +void intel_sagv_post_plane_update(struct intel_atomic_state *state); bool skl_wm_level_equals(const struct skl_wm_level *l1, const struct skl_wm_level *l2); bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb, diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c index 1447e7516cb7..916ccd1c0e96 100644 --- a/drivers/gpu/drm/i915/intel_sideband.c +++ b/drivers/gpu/drm/i915/intel_sideband.c @@ -60,7 +60,7 @@ static void __vlv_punit_get(struct drm_i915_private *i915) * to the Valleyview P-unit and not all sideband communications. */ if (IS_VALLEYVIEW(i915)) { - pm_qos_update_request(&i915->sb_qos, 0); + cpu_latency_qos_update_request(&i915->sb_qos, 0); on_each_cpu(ping, NULL, 1); } } @@ -68,7 +68,8 @@ static void __vlv_punit_get(struct drm_i915_private *i915) static void __vlv_punit_put(struct drm_i915_private *i915) { if (IS_VALLEYVIEW(i915)) - pm_qos_update_request(&i915->sb_qos, PM_QOS_DEFAULT_VALUE); + cpu_latency_qos_update_request(&i915->sb_qos, + PM_QOS_DEFAULT_VALUE); iosf_mbi_punit_release(); } @@ -335,7 +336,7 @@ void intel_sbi_write(struct drm_i915_private *i915, u16 reg, u32 value, intel_sbi_rw(i915, reg, destination, &value, false); } -static inline int gen6_check_mailbox_status(u32 mbox) +static int gen6_check_mailbox_status(u32 mbox) { switch (mbox & GEN6_PCODE_ERROR_MASK) { case GEN6_PCODE_SUCCESS: @@ -355,7 +356,7 @@ static inline int gen6_check_mailbox_status(u32 mbox) } } -static inline int gen7_check_mailbox_status(u32 mbox) +static int gen7_check_mailbox_status(u32 mbox) { switch (mbox & GEN6_PCODE_ERROR_MASK) { case GEN6_PCODE_SUCCESS: @@ -370,6 +371,8 @@ static inline int gen7_check_mailbox_status(u32 mbox) return -ENXIO; case GEN11_PCODE_LOCKED: return -EBUSY; + case GEN11_PCODE_REJECTED: + return -EACCES; case GEN7_PCODE_MIN_FREQ_TABLE_GT_RATIO_OUT_OF_RANGE: return -EOVERFLOW; default: @@ -428,7 +431,7 @@ int sandybridge_pcode_read(struct drm_i915_private *i915, u32 mbox, mutex_lock(&i915->sb_lock); err = __sandybridge_pcode_rw(i915, mbox, val, val1, - 500, 0, + 500, 20, true); mutex_unlock(&i915->sb_lock); diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index abb18b90d7c3..a61cb8ca4d50 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -665,7 +665,7 @@ void intel_uncore_forcewake_user_put(struct intel_uncore *uncore) mmio_debug_resume(uncore->debug); if (check_for_unclaimed_mmio(uncore)) - dev_info(uncore->i915->drm.dev, + drm_info(&uncore->i915->drm, "Invalid mmio detected during user access\n"); spin_unlock(&uncore->debug->lock); @@ -735,6 +735,28 @@ void intel_uncore_forcewake_put(struct intel_uncore *uncore, } /** + * intel_uncore_forcewake_flush - flush the delayed release + * @uncore: the intel_uncore structure + * @fw_domains: forcewake domains to flush + */ +void intel_uncore_forcewake_flush(struct intel_uncore *uncore, + enum forcewake_domains fw_domains) +{ + struct intel_uncore_forcewake_domain *domain; + unsigned int tmp; + + if (!uncore->funcs.force_wake_put) + return; + + fw_domains &= uncore->fw_domains; + for_each_fw_domain_masked(domain, fw_domains, uncore, tmp) { + WRITE_ONCE(domain->active, false); + if (hrtimer_cancel(&domain->timer)) + intel_uncore_fw_release_timer(&domain->timer); + } +} + +/** * intel_uncore_forcewake_put__locked - grab forcewake domain references * @uncore: the intel_uncore structure * @fw_domains: forcewake domains to get reference on @@ -877,11 +899,6 @@ find_fw_domain(struct intel_uncore *uncore, u32 offset) #define GEN_FW_RANGE(s, e, d) \ { .start = (s), .end = (e), .domains = (d) } -#define HAS_FWTABLE(dev_priv) \ - (INTEL_GEN(dev_priv) >= 9 || \ - IS_CHERRYVIEW(dev_priv) || \ - IS_VALLEYVIEW(dev_priv)) - /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ static const struct intel_forcewake_range __vlv_fw_ranges[] = { GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER), @@ -1070,8 +1087,7 @@ static const struct intel_forcewake_range __gen9_fw_ranges[] = { /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ static const struct intel_forcewake_range __gen11_fw_ranges[] = { - GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER), - GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */ + GEN_FW_RANGE(0x0, 0x1fff, 0), /* uncore range */ GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER), GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER), @@ -1081,27 +1097,31 @@ static const struct intel_forcewake_range __gen11_fw_ranges[] = { GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER), GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER), - GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x8800, 0x8bff, 0), GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER), - GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER), - GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL), - GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x8d00, 0x94cf, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x94d0, 0x955f, FORCEWAKE_RENDER), + GEN_FW_RANGE(0x9560, 0x95ff, 0), + GEN_FW_RANGE(0x9600, 0xafff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER), GEN_FW_RANGE(0xb480, 0xdeff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0xdf00, 0xe8ff, FORCEWAKE_RENDER), GEN_FW_RANGE(0xe900, 0x16dff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0x16e00, 0x19fff, FORCEWAKE_RENDER), - GEN_FW_RANGE(0x1a000, 0x243ff, FORCEWAKE_BLITTER), - GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER), - GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x1a000, 0x23fff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x24000, 0x2407f, 0), + GEN_FW_RANGE(0x24080, 0x2417f, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x24180, 0x242ff, FORCEWAKE_RENDER), + GEN_FW_RANGE(0x24300, 0x243ff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x24400, 0x24fff, FORCEWAKE_RENDER), + GEN_FW_RANGE(0x25000, 0x3ffff, FORCEWAKE_BLITTER), GEN_FW_RANGE(0x40000, 0x1bffff, 0), GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0), - GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1), - GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0), - GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER), + GEN_FW_RANGE(0x1c4000, 0x1c7fff, 0), + GEN_FW_RANGE(0x1c8000, 0x1cffff, FORCEWAKE_MEDIA_VEBOX0), GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2), - GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3), - GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1) + GEN_FW_RANGE(0x1d4000, 0x1dbfff, 0) }; /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */ diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h index dcfa243892c6..8d3aa8b9acf9 100644 --- a/drivers/gpu/drm/i915/intel_uncore.h +++ b/drivers/gpu/drm/i915/intel_uncore.h @@ -209,7 +209,11 @@ void intel_uncore_forcewake_get(struct intel_uncore *uncore, enum forcewake_domains domains); void intel_uncore_forcewake_put(struct intel_uncore *uncore, enum forcewake_domains domains); -/* Like above but the caller must manage the uncore.lock itself. +void intel_uncore_forcewake_flush(struct intel_uncore *uncore, + enum forcewake_domains fw_domains); + +/* + * Like above but the caller must manage the uncore.lock itself. * Must be used with I915_READ_FW and friends. */ void intel_uncore_forcewake_get__locked(struct intel_uncore *uncore, diff --git a/drivers/gpu/drm/i915/intel_wakeref.c b/drivers/gpu/drm/i915/intel_wakeref.c index 8fbf6f4d3f26..dfd87d082218 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.c +++ b/drivers/gpu/drm/i915/intel_wakeref.c @@ -70,11 +70,12 @@ unlock: void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) { - INTEL_WAKEREF_BUG_ON(work_pending(&wf->work)); + INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work)); /* Assume we are not in process context and so cannot sleep. */ if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) { - schedule_work(&wf->work); + mod_delayed_work(system_wq, &wf->work, + FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags)); return; } @@ -83,7 +84,7 @@ void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags) static void __intel_wakeref_put_work(struct work_struct *wrk) { - struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work); + struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work); if (atomic_add_unless(&wf->count, -1, 1)) return; @@ -104,8 +105,9 @@ void __intel_wakeref_init(struct intel_wakeref *wf, atomic_set(&wf->count, 0); wf->wakeref = 0; - INIT_WORK(&wf->work, __intel_wakeref_put_work); - lockdep_init_map(&wf->work.lockdep_map, "wakeref.work", &key->work, 0); + INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work); + lockdep_init_map(&wf->work.work.lockdep_map, + "wakeref.work", &key->work, 0); } int intel_wakeref_wait_for_idle(struct intel_wakeref *wf) diff --git a/drivers/gpu/drm/i915/intel_wakeref.h b/drivers/gpu/drm/i915/intel_wakeref.h index 7d1e676b71ef..545c8f277c46 100644 --- a/drivers/gpu/drm/i915/intel_wakeref.h +++ b/drivers/gpu/drm/i915/intel_wakeref.h @@ -8,6 +8,7 @@ #define INTEL_WAKEREF_H #include <linux/atomic.h> +#include <linux/bitfield.h> #include <linux/bits.h> #include <linux/lockdep.h> #include <linux/mutex.h> @@ -41,7 +42,7 @@ struct intel_wakeref { struct intel_runtime_pm *rpm; const struct intel_wakeref_ops *ops; - struct work_struct work; + struct delayed_work work; }; struct intel_wakeref_lockclass { @@ -117,6 +118,11 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf) return atomic_inc_not_zero(&wf->count); } +enum { + INTEL_WAKEREF_PUT_ASYNC_BIT = 0, + __INTEL_WAKEREF_PUT_LAST_BIT__ +}; + /** * intel_wakeref_put_flags: Release the wakeref * @wf: the wakeref @@ -134,7 +140,9 @@ intel_wakeref_get_if_active(struct intel_wakeref *wf) */ static inline void __intel_wakeref_put(struct intel_wakeref *wf, unsigned long flags) -#define INTEL_WAKEREF_PUT_ASYNC BIT(0) +#define INTEL_WAKEREF_PUT_ASYNC BIT(INTEL_WAKEREF_PUT_ASYNC_BIT) +#define INTEL_WAKEREF_PUT_DELAY \ + GENMASK(BITS_PER_LONG - 1, __INTEL_WAKEREF_PUT_LAST_BIT__) { INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0); if (unlikely(!atomic_add_unless(&wf->count, -1, 1))) @@ -154,6 +162,14 @@ intel_wakeref_put_async(struct intel_wakeref *wf) __intel_wakeref_put(wf, INTEL_WAKEREF_PUT_ASYNC); } +static inline void +intel_wakeref_put_delay(struct intel_wakeref *wf, unsigned long delay) +{ + __intel_wakeref_put(wf, + INTEL_WAKEREF_PUT_ASYNC | + FIELD_PREP(INTEL_WAKEREF_PUT_DELAY, delay)); +} + /** * intel_wakeref_lock: Lock the wakeref (mutex) * @wf: the wakeref @@ -194,7 +210,7 @@ intel_wakeref_unlock_wait(struct intel_wakeref *wf) { mutex_lock(&wf->mutex); mutex_unlock(&wf->mutex); - flush_work(&wf->work); + flush_delayed_work(&wf->work); } /** diff --git a/drivers/gpu/drm/i915/intel_wopcm.c b/drivers/gpu/drm/i915/intel_wopcm.c index 2bb9f9f9a50a..ec776591e1cf 100644 --- a/drivers/gpu/drm/i915/intel_wopcm.c +++ b/drivers/gpu/drm/i915/intel_wopcm.c @@ -86,10 +86,10 @@ void intel_wopcm_init_early(struct intel_wopcm *wopcm) else wopcm->size = GEN9_WOPCM_SIZE; - DRM_DEV_DEBUG_DRIVER(i915->drm.dev, "WOPCM: %uK\n", wopcm->size / 1024); + drm_dbg(&i915->drm, "WOPCM: %uK\n", wopcm->size / 1024); } -static inline u32 context_reserved_size(struct drm_i915_private *i915) +static u32 context_reserved_size(struct drm_i915_private *i915) { if (IS_GEN9_LP(i915)) return BXT_WOPCM_RC6_CTX_RESERVED; @@ -99,8 +99,8 @@ static inline u32 context_reserved_size(struct drm_i915_private *i915) return 0; } -static inline bool gen9_check_dword_gap(struct drm_i915_private *i915, - u32 guc_wopcm_base, u32 guc_wopcm_size) +static bool gen9_check_dword_gap(struct drm_i915_private *i915, + u32 guc_wopcm_base, u32 guc_wopcm_size) { u32 offset; @@ -112,7 +112,7 @@ static inline bool gen9_check_dword_gap(struct drm_i915_private *i915, offset = guc_wopcm_base + GEN9_GUC_WOPCM_OFFSET; if (offset > guc_wopcm_size || (guc_wopcm_size - offset) < sizeof(u32)) { - dev_err(i915->drm.dev, + drm_err(&i915->drm, "WOPCM: invalid GuC region size: %uK < %uK\n", guc_wopcm_size / SZ_1K, (u32)(offset + sizeof(u32)) / SZ_1K); @@ -122,8 +122,8 @@ static inline bool gen9_check_dword_gap(struct drm_i915_private *i915, return true; } -static inline bool gen9_check_huc_fw_fits(struct drm_i915_private *i915, - u32 guc_wopcm_size, u32 huc_fw_size) +static bool gen9_check_huc_fw_fits(struct drm_i915_private *i915, + u32 guc_wopcm_size, u32 huc_fw_size) { /* * On Gen9 & CNL A0, hardware requires the total available GuC WOPCM @@ -131,7 +131,7 @@ static inline bool gen9_check_huc_fw_fits(struct drm_i915_private *i915, * firmware uploading would fail. */ if (huc_fw_size > guc_wopcm_size - GUC_WOPCM_RESERVED) { - dev_err(i915->drm.dev, "WOPCM: no space for %s: %uK < %uK\n", + drm_err(&i915->drm, "WOPCM: no space for %s: %uK < %uK\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_HUC), (guc_wopcm_size - GUC_WOPCM_RESERVED) / SZ_1K, huc_fw_size / 1024); @@ -141,32 +141,31 @@ static inline bool gen9_check_huc_fw_fits(struct drm_i915_private *i915, return true; } -static inline bool check_hw_restrictions(struct drm_i915_private *i915, - u32 guc_wopcm_base, u32 guc_wopcm_size, - u32 huc_fw_size) +static bool check_hw_restrictions(struct drm_i915_private *i915, + u32 guc_wopcm_base, u32 guc_wopcm_size, + u32 huc_fw_size) { if (IS_GEN(i915, 9) && !gen9_check_dword_gap(i915, guc_wopcm_base, guc_wopcm_size)) return false; - if ((IS_GEN(i915, 9) || - IS_CNL_REVID(i915, CNL_REVID_A0, CNL_REVID_A0)) && + if (IS_GEN(i915, 9) && !gen9_check_huc_fw_fits(i915, guc_wopcm_size, huc_fw_size)) return false; return true; } -static inline bool __check_layout(struct drm_i915_private *i915, u32 wopcm_size, - u32 guc_wopcm_base, u32 guc_wopcm_size, - u32 guc_fw_size, u32 huc_fw_size) +static bool __check_layout(struct drm_i915_private *i915, u32 wopcm_size, + u32 guc_wopcm_base, u32 guc_wopcm_size, + u32 guc_fw_size, u32 huc_fw_size) { const u32 ctx_rsvd = context_reserved_size(i915); u32 size; size = wopcm_size - ctx_rsvd; if (unlikely(range_overflows(guc_wopcm_base, guc_wopcm_size, size))) { - dev_err(i915->drm.dev, + drm_err(&i915->drm, "WOPCM: invalid GuC region layout: %uK + %uK > %uK\n", guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K, size / SZ_1K); @@ -175,7 +174,7 @@ static inline bool __check_layout(struct drm_i915_private *i915, u32 wopcm_size, size = guc_fw_size + GUC_WOPCM_RESERVED + GUC_WOPCM_STACK_RESERVED; if (unlikely(guc_wopcm_size < size)) { - dev_err(i915->drm.dev, "WOPCM: no space for %s: %uK < %uK\n", + drm_err(&i915->drm, "WOPCM: no space for %s: %uK < %uK\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_GUC), guc_wopcm_size / SZ_1K, size / SZ_1K); return false; @@ -183,7 +182,7 @@ static inline bool __check_layout(struct drm_i915_private *i915, u32 wopcm_size, size = huc_fw_size + WOPCM_RESERVED_SIZE; if (unlikely(guc_wopcm_base < size)) { - dev_err(i915->drm.dev, "WOPCM: no space for %s: %uK < %uK\n", + drm_err(&i915->drm, "WOPCM: no space for %s: %uK < %uK\n", intel_uc_fw_type_repr(INTEL_UC_FW_TYPE_HUC), guc_wopcm_base / SZ_1K, size / SZ_1K); return false; @@ -242,10 +241,8 @@ void intel_wopcm_init(struct intel_wopcm *wopcm) return; if (__wopcm_regs_locked(gt->uncore, &guc_wopcm_base, &guc_wopcm_size)) { - DRM_DEV_DEBUG_DRIVER(i915->drm.dev, - "GuC WOPCM is already locked [%uK, %uK)\n", - guc_wopcm_base / SZ_1K, - guc_wopcm_size / SZ_1K); + drm_dbg(&i915->drm, "GuC WOPCM is already locked [%uK, %uK)\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); goto check; } @@ -266,8 +263,8 @@ void intel_wopcm_init(struct intel_wopcm *wopcm) guc_wopcm_size = wopcm->size - ctx_rsvd - guc_wopcm_base; guc_wopcm_size &= GUC_WOPCM_SIZE_MASK; - DRM_DEV_DEBUG_DRIVER(i915->drm.dev, "Calculated GuC WOPCM [%uK, %uK)\n", - guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); + drm_dbg(&i915->drm, "Calculated GuC WOPCM [%uK, %uK)\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); check: if (__check_layout(i915, wopcm->size, guc_wopcm_base, guc_wopcm_size, diff --git a/drivers/gpu/drm/i915/oa/i915_oa_bdw.c b/drivers/gpu/drm/i915/oa/i915_oa_bdw.c deleted file mode 100644 index 14da5c3b569d..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_bdw.c +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_bdw.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x000000a0 }, - { _MMIO(0x9888), 0x198b0000 }, - { _MMIO(0x9888), 0x078b0066 }, - { _MMIO(0x9888), 0x118b0000 }, - { _MMIO(0x9888), 0x258b0000 }, - { _MMIO(0x9888), 0x21850008 }, - { _MMIO(0x9888), 0x0d834000 }, - { _MMIO(0x9888), 0x07844000 }, - { _MMIO(0x9888), 0x17804000 }, - { _MMIO(0x9888), 0x21800000 }, - { _MMIO(0x9888), 0x4f800000 }, - { _MMIO(0x9888), 0x41800000 }, - { _MMIO(0x9888), 0x31800000 }, - { _MMIO(0x9840), 0x00000080 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "d6de6f55-e526-4f79-a6a6-d7315c09044e", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "d6de6f55-e526-4f79-a6a6-d7315c09044e"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_bdw.h b/drivers/gpu/drm/i915/oa/i915_oa_bdw.h deleted file mode 100644 index 0cee3334f0a6..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_bdw.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_BDW_H__ -#define __I915_OA_BDW_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_bxt.c b/drivers/gpu/drm/i915/oa/i915_oa_bxt.c deleted file mode 100644 index 3e785bafcf99..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_bxt.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_bxt.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x19800000 }, - { _MMIO(0x9888), 0x07800063 }, - { _MMIO(0x9888), 0x11800000 }, - { _MMIO(0x9888), 0x23810008 }, - { _MMIO(0x9888), 0x1d950400 }, - { _MMIO(0x9888), 0x0f922000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x55900000 }, - { _MMIO(0x9888), 0x47900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "5ee72f5c-092f-421e-8b70-225f7c3e9612", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "5ee72f5c-092f-421e-8b70-225f7c3e9612"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_bxt.h b/drivers/gpu/drm/i915/oa/i915_oa_bxt.h deleted file mode 100644 index 0bdf391323ec..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_bxt.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_BXT_H__ -#define __I915_OA_BXT_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_cflgt2.c b/drivers/gpu/drm/i915/oa/i915_oa_cflgt2.c deleted file mode 100644 index 0ea86f70a06c..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_cflgt2.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_cflgt2.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810013 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "74fb4902-d3d3-4237-9e90-cbdc68d0a446", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "74fb4902-d3d3-4237-9e90-cbdc68d0a446"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_cflgt2.h b/drivers/gpu/drm/i915/oa/i915_oa_cflgt2.h deleted file mode 100644 index 6b862280ab78..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_cflgt2.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_CFLGT2_H__ -#define __I915_OA_CFLGT2_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_cflgt3.c b/drivers/gpu/drm/i915/oa/i915_oa_cflgt3.c deleted file mode 100644 index fc632dd890bf..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_cflgt3.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_cflgt3.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810013 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "577e8e2c-3fa0-4875-8743-3538d585e3b0", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "577e8e2c-3fa0-4875-8743-3538d585e3b0"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_cflgt3.h b/drivers/gpu/drm/i915/oa/i915_oa_cflgt3.h deleted file mode 100644 index 4ca9d8f89b2f..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_cflgt3.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_CFLGT3_H__ -#define __I915_OA_CFLGT3_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_chv.c b/drivers/gpu/drm/i915/oa/i915_oa_chv.c deleted file mode 100644 index 6cd4e9921a8a..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_chv.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_chv.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x000000a0 }, - { _MMIO(0x9888), 0x59800000 }, - { _MMIO(0x9888), 0x59800001 }, - { _MMIO(0x9888), 0x338b0000 }, - { _MMIO(0x9888), 0x258b0066 }, - { _MMIO(0x9888), 0x058b0000 }, - { _MMIO(0x9888), 0x038b0000 }, - { _MMIO(0x9888), 0x03844000 }, - { _MMIO(0x9888), 0x47800080 }, - { _MMIO(0x9888), 0x57800000 }, - { _MMIO(0x1823a4), 0x00000000 }, - { _MMIO(0x9888), 0x59800000 }, - { _MMIO(0x9840), 0x00000080 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "4a534b07-cba3-414d-8d60-874830e883aa", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "4a534b07-cba3-414d-8d60-874830e883aa"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_chv.h b/drivers/gpu/drm/i915/oa/i915_oa_chv.h deleted file mode 100644 index 3cac7bbc9c71..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_chv.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_CHV_H__ -#define __I915_OA_CHV_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_cnl.c b/drivers/gpu/drm/i915/oa/i915_oa_cnl.c deleted file mode 100644 index 1041e8914993..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_cnl.c +++ /dev/null @@ -1,101 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_cnl.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x0000ffff }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x0000ffff }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x0000ffff }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0xd04), 0x00000200 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x17060000 }, - { _MMIO(0x9840), 0x00000000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x13034000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x07060066 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x05060000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x0f080040 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x07091000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x0f041000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x1d004000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x35000000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x49000000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x3d000000 }, - { _MMIO(0x9884), 0x00000007 }, - { _MMIO(0x9888), 0x31000000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "db41edd4-d8e7-4730-ad11-b9a2d6833503", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "db41edd4-d8e7-4730-ad11-b9a2d6833503"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_cnl.h b/drivers/gpu/drm/i915/oa/i915_oa_cnl.h deleted file mode 100644 index db379f5fcbb9..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_cnl.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_CNL_H__ -#define __I915_OA_CNL_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_glk.c b/drivers/gpu/drm/i915/oa/i915_oa_glk.c deleted file mode 100644 index bd15ebe9aeeb..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_glk.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_glk.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x19800000 }, - { _MMIO(0x9888), 0x07800063 }, - { _MMIO(0x9888), 0x11800000 }, - { _MMIO(0x9888), 0x23810008 }, - { _MMIO(0x9888), 0x1d950400 }, - { _MMIO(0x9888), 0x0f922000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x55900000 }, - { _MMIO(0x9888), 0x47900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "dd3fd789-e783-4204-8cd0-b671bbccb0cf", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "dd3fd789-e783-4204-8cd0-b671bbccb0cf"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_glk.h b/drivers/gpu/drm/i915/oa/i915_oa_glk.h deleted file mode 100644 index 779f343efd11..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_glk.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_GLK_H__ -#define __I915_OA_GLK_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_hsw.c b/drivers/gpu/drm/i915/oa/i915_oa_hsw.c deleted file mode 100644 index 133721a8619f..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_hsw.c +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_hsw.h" - -static const struct i915_oa_reg b_counter_config_render_basic[] = { - { _MMIO(0x2724), 0x00800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2714), 0x00800000 }, - { _MMIO(0x2710), 0x00000000 }, -}; - -static const struct i915_oa_reg flex_eu_config_render_basic[] = { -}; - -static const struct i915_oa_reg mux_config_render_basic[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x253a4), 0x01600000 }, - { _MMIO(0x25440), 0x00100000 }, - { _MMIO(0x25128), 0x00000000 }, - { _MMIO(0x2691c), 0x00000800 }, - { _MMIO(0x26aa0), 0x01500000 }, - { _MMIO(0x26b9c), 0x00006000 }, - { _MMIO(0x2791c), 0x00000800 }, - { _MMIO(0x27aa0), 0x01500000 }, - { _MMIO(0x27b9c), 0x00006000 }, - { _MMIO(0x2641c), 0x00000400 }, - { _MMIO(0x25380), 0x00000010 }, - { _MMIO(0x2538c), 0x00000000 }, - { _MMIO(0x25384), 0x0800aaaa }, - { _MMIO(0x25400), 0x00000004 }, - { _MMIO(0x2540c), 0x06029000 }, - { _MMIO(0x25410), 0x00000002 }, - { _MMIO(0x25404), 0x5c30ffff }, - { _MMIO(0x25100), 0x00000016 }, - { _MMIO(0x25110), 0x00000400 }, - { _MMIO(0x25104), 0x00000000 }, - { _MMIO(0x26804), 0x00001211 }, - { _MMIO(0x26884), 0x00000100 }, - { _MMIO(0x26900), 0x00000002 }, - { _MMIO(0x26908), 0x00700000 }, - { _MMIO(0x26904), 0x00000000 }, - { _MMIO(0x26984), 0x00001022 }, - { _MMIO(0x26a04), 0x00000011 }, - { _MMIO(0x26a80), 0x00000006 }, - { _MMIO(0x26a88), 0x00000c02 }, - { _MMIO(0x26a84), 0x00000000 }, - { _MMIO(0x26b04), 0x00001000 }, - { _MMIO(0x26b80), 0x00000002 }, - { _MMIO(0x26b8c), 0x00000007 }, - { _MMIO(0x26b84), 0x00000000 }, - { _MMIO(0x27804), 0x00004844 }, - { _MMIO(0x27884), 0x00000400 }, - { _MMIO(0x27900), 0x00000002 }, - { _MMIO(0x27908), 0x0e000000 }, - { _MMIO(0x27904), 0x00000000 }, - { _MMIO(0x27984), 0x00004088 }, - { _MMIO(0x27a04), 0x00000044 }, - { _MMIO(0x27a80), 0x00000006 }, - { _MMIO(0x27a88), 0x00018040 }, - { _MMIO(0x27a84), 0x00000000 }, - { _MMIO(0x27b04), 0x00004000 }, - { _MMIO(0x27b80), 0x00000002 }, - { _MMIO(0x27b8c), 0x000000e0 }, - { _MMIO(0x27b84), 0x00000000 }, - { _MMIO(0x26104), 0x00002222 }, - { _MMIO(0x26184), 0x0c006666 }, - { _MMIO(0x26284), 0x04000000 }, - { _MMIO(0x26304), 0x04000000 }, - { _MMIO(0x26400), 0x00000002 }, - { _MMIO(0x26410), 0x000000a0 }, - { _MMIO(0x26404), 0x00000000 }, - { _MMIO(0x25420), 0x04108020 }, - { _MMIO(0x25424), 0x1284a420 }, - { _MMIO(0x2541c), 0x00000000 }, - { _MMIO(0x25428), 0x00042049 }, -}; - -static ssize_t -show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "403d8832-1a27-4aa6-a64e-f5389ce7b212", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_render_basic; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_render_basic); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_render_basic; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_render_basic); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_render_basic; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_render_basic); - - dev_priv->perf.test_config.sysfs_metric.name = "403d8832-1a27-4aa6-a64e-f5389ce7b212"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_render_basic_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_hsw.h b/drivers/gpu/drm/i915/oa/i915_oa_hsw.h deleted file mode 100644 index ba97f732f136..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_hsw.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_HSW_H__ -#define __I915_OA_HSW_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_icl.c b/drivers/gpu/drm/i915/oa/i915_oa_icl.c deleted file mode 100644 index 2d92041b754f..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_icl.c +++ /dev/null @@ -1,98 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_icl.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x0000ffff }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x0000ffff }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x0000ffff }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0xd04), 0x00000200 }, - { _MMIO(0x9840), 0x00000000 }, - { _MMIO(0x9884), 0x00000000 }, - { _MMIO(0x9888), 0x10060000 }, - { _MMIO(0x9888), 0x22060000 }, - { _MMIO(0x9888), 0x16060000 }, - { _MMIO(0x9888), 0x24060000 }, - { _MMIO(0x9888), 0x18060000 }, - { _MMIO(0x9888), 0x1a060000 }, - { _MMIO(0x9888), 0x12060000 }, - { _MMIO(0x9888), 0x14060000 }, - { _MMIO(0x9888), 0x10060000 }, - { _MMIO(0x9888), 0x22060000 }, - { _MMIO(0x9884), 0x00000003 }, - { _MMIO(0x9888), 0x16130000 }, - { _MMIO(0x9888), 0x24000001 }, - { _MMIO(0x9888), 0x0e130056 }, - { _MMIO(0x9888), 0x10130000 }, - { _MMIO(0x9888), 0x1a130000 }, - { _MMIO(0x9888), 0x541f0001 }, - { _MMIO(0x9888), 0x181f0000 }, - { _MMIO(0x9888), 0x4c1f0000 }, - { _MMIO(0x9888), 0x301f0000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "a291665e-244b-4b76-9b9a-01de9d3c8068", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "a291665e-244b-4b76-9b9a-01de9d3c8068"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_icl.h b/drivers/gpu/drm/i915/oa/i915_oa_icl.h deleted file mode 100644 index 5c64112d720e..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_icl.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_ICL_H__ -#define __I915_OA_ICL_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_icl(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_kblgt2.c b/drivers/gpu/drm/i915/oa/i915_oa_kblgt2.c deleted file mode 100644 index 1c3a67c9cfe0..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_kblgt2.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_kblgt2.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810013 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "baa3c7e4-52b6-4b85-801e-465a94b746dd", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "baa3c7e4-52b6-4b85-801e-465a94b746dd"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_kblgt2.h b/drivers/gpu/drm/i915/oa/i915_oa_kblgt2.h deleted file mode 100644 index 810532fa6b63..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_kblgt2.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_KBLGT2_H__ -#define __I915_OA_KBLGT2_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_kblgt3.c b/drivers/gpu/drm/i915/oa/i915_oa_kblgt3.c deleted file mode 100644 index ebbe5a9c9fdc..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_kblgt3.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_kblgt3.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810013 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "f1792f32-6db2-4b50-b4b2-557128f1688d", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "f1792f32-6db2-4b50-b4b2-557128f1688d"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_kblgt3.h b/drivers/gpu/drm/i915/oa/i915_oa_kblgt3.h deleted file mode 100644 index 13d70456fabd..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_kblgt3.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_KBLGT3_H__ -#define __I915_OA_KBLGT3_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_sklgt2.c b/drivers/gpu/drm/i915/oa/i915_oa_sklgt2.c deleted file mode 100644 index 1bc359ed34e8..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_sklgt2.c +++ /dev/null @@ -1,88 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_sklgt2.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810016 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "1651949f-0ac0-4cb1-a06f-dafd74a407d1", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "1651949f-0ac0-4cb1-a06f-dafd74a407d1"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_sklgt2.h b/drivers/gpu/drm/i915/oa/i915_oa_sklgt2.h deleted file mode 100644 index fda70c51a6ec..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_sklgt2.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_SKLGT2_H__ -#define __I915_OA_SKLGT2_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_sklgt3.c b/drivers/gpu/drm/i915/oa/i915_oa_sklgt3.c deleted file mode 100644 index 6e352f881310..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_sklgt3.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_sklgt3.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810013 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "2b985803-d3c9-4629-8a4f-634bfecba0e8", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "2b985803-d3c9-4629-8a4f-634bfecba0e8"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_sklgt3.h b/drivers/gpu/drm/i915/oa/i915_oa_sklgt3.h deleted file mode 100644 index df74eba5799e..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_sklgt3.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_SKLGT3_H__ -#define __I915_OA_SKLGT3_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_sklgt4.c b/drivers/gpu/drm/i915/oa/i915_oa_sklgt4.c deleted file mode 100644 index 8f345115a306..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_sklgt4.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_sklgt4.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0x2740), 0x00000000 }, - { _MMIO(0x2744), 0x00800000 }, - { _MMIO(0x2714), 0xf0800000 }, - { _MMIO(0x2710), 0x00000000 }, - { _MMIO(0x2724), 0xf0800000 }, - { _MMIO(0x2720), 0x00000000 }, - { _MMIO(0x2770), 0x00000004 }, - { _MMIO(0x2774), 0x00000000 }, - { _MMIO(0x2778), 0x00000003 }, - { _MMIO(0x277c), 0x00000000 }, - { _MMIO(0x2780), 0x00000007 }, - { _MMIO(0x2784), 0x00000000 }, - { _MMIO(0x2788), 0x00100002 }, - { _MMIO(0x278c), 0x0000fff7 }, - { _MMIO(0x2790), 0x00100002 }, - { _MMIO(0x2794), 0x0000ffcf }, - { _MMIO(0x2798), 0x00100082 }, - { _MMIO(0x279c), 0x0000ffef }, - { _MMIO(0x27a0), 0x001000c2 }, - { _MMIO(0x27a4), 0x0000ffe7 }, - { _MMIO(0x27a8), 0x00100001 }, - { _MMIO(0x27ac), 0x0000ffe7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x9840), 0x00000080 }, - { _MMIO(0x9888), 0x11810000 }, - { _MMIO(0x9888), 0x07810013 }, - { _MMIO(0x9888), 0x1f810000 }, - { _MMIO(0x9888), 0x1d810000 }, - { _MMIO(0x9888), 0x1b930040 }, - { _MMIO(0x9888), 0x07e54000 }, - { _MMIO(0x9888), 0x1f908000 }, - { _MMIO(0x9888), 0x11900000 }, - { _MMIO(0x9888), 0x37900000 }, - { _MMIO(0x9888), 0x53900000 }, - { _MMIO(0x9888), 0x45900000 }, - { _MMIO(0x9888), 0x33900000 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "882fa433-1f4a-4a67-a962-c741888fe5f5", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "882fa433-1f4a-4a67-a962-c741888fe5f5"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_sklgt4.h b/drivers/gpu/drm/i915/oa/i915_oa_sklgt4.h deleted file mode 100644 index 378ab7ab78d5..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_sklgt4.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018-2019 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_SKLGT4_H__ -#define __I915_OA_SKLGT4_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/oa/i915_oa_tgl.c b/drivers/gpu/drm/i915/oa/i915_oa_tgl.c deleted file mode 100644 index a29d93707345..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_tgl.c +++ /dev/null @@ -1,121 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2018 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#include <linux/sysfs.h> - -#include "i915_drv.h" -#include "i915_oa_tgl.h" - -static const struct i915_oa_reg b_counter_config_test_oa[] = { - { _MMIO(0xD920), 0x00000000 }, - { _MMIO(0xD900), 0x00000000 }, - { _MMIO(0xD904), 0xF0800000 }, - { _MMIO(0xD910), 0x00000000 }, - { _MMIO(0xD914), 0xF0800000 }, - { _MMIO(0xDC40), 0x00FF0000 }, - { _MMIO(0xD940), 0x00000004 }, - { _MMIO(0xD944), 0x0000FFFF }, - { _MMIO(0xDC00), 0x00000004 }, - { _MMIO(0xDC04), 0x0000FFFF }, - { _MMIO(0xD948), 0x00000003 }, - { _MMIO(0xD94C), 0x0000FFFF }, - { _MMIO(0xDC08), 0x00000003 }, - { _MMIO(0xDC0C), 0x0000FFFF }, - { _MMIO(0xD950), 0x00000007 }, - { _MMIO(0xD954), 0x0000FFFF }, - { _MMIO(0xDC10), 0x00000007 }, - { _MMIO(0xDC14), 0x0000FFFF }, - { _MMIO(0xD958), 0x00100002 }, - { _MMIO(0xD95C), 0x0000FFF7 }, - { _MMIO(0xDC18), 0x00100002 }, - { _MMIO(0xDC1C), 0x0000FFF7 }, - { _MMIO(0xD960), 0x00100002 }, - { _MMIO(0xD964), 0x0000FFCF }, - { _MMIO(0xDC20), 0x00100002 }, - { _MMIO(0xDC24), 0x0000FFCF }, - { _MMIO(0xD968), 0x00100082 }, - { _MMIO(0xD96C), 0x0000FFEF }, - { _MMIO(0xDC28), 0x00100082 }, - { _MMIO(0xDC2C), 0x0000FFEF }, - { _MMIO(0xD970), 0x001000C2 }, - { _MMIO(0xD974), 0x0000FFE7 }, - { _MMIO(0xDC30), 0x001000C2 }, - { _MMIO(0xDC34), 0x0000FFE7 }, - { _MMIO(0xD978), 0x00100001 }, - { _MMIO(0xD97C), 0x0000FFE7 }, - { _MMIO(0xDC38), 0x00100001 }, - { _MMIO(0xDC3C), 0x0000FFE7 }, -}; - -static const struct i915_oa_reg flex_eu_config_test_oa[] = { -}; - -static const struct i915_oa_reg mux_config_test_oa[] = { - { _MMIO(0x0D04), 0x00000200 }, - { _MMIO(0x9840), 0x00000000 }, - { _MMIO(0x9884), 0x00000000 }, - { _MMIO(0x9888), 0x280E0000 }, - { _MMIO(0x9888), 0x1E0E0147 }, - { _MMIO(0x9888), 0x180E0000 }, - { _MMIO(0x9888), 0x160E0000 }, - { _MMIO(0x9888), 0x1E0F1000 }, - { _MMIO(0x9888), 0x1E104000 }, - { _MMIO(0x9888), 0x2E020100 }, - { _MMIO(0x9888), 0x2C030004 }, - { _MMIO(0x9888), 0x38003000 }, - { _MMIO(0x9888), 0x1E0A8000 }, - { _MMIO(0x9884), 0x00000003 }, - { _MMIO(0x9888), 0x49110000 }, - { _MMIO(0x9888), 0x5D101400 }, - { _MMIO(0x9888), 0x1D140020 }, - { _MMIO(0x9888), 0x1D1103A3 }, - { _MMIO(0x9888), 0x01110000 }, - { _MMIO(0x9888), 0x61111000 }, - { _MMIO(0x9888), 0x1F128000 }, - { _MMIO(0x9888), 0x17100000 }, - { _MMIO(0x9888), 0x55100630 }, - { _MMIO(0x9888), 0x57100000 }, - { _MMIO(0x9888), 0x31100000 }, - { _MMIO(0x9884), 0x00000003 }, - { _MMIO(0x9888), 0x65100002 }, - { _MMIO(0x9884), 0x00000000 }, - { _MMIO(0x9888), 0x42000001 }, -}; - -static ssize_t -show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf) -{ - return sprintf(buf, "1\n"); -} - -void -i915_perf_load_test_config_tgl(struct drm_i915_private *dev_priv) -{ - strlcpy(dev_priv->perf.test_config.uuid, - "80a833f0-2504-4321-8894-e9277844ce7b", - sizeof(dev_priv->perf.test_config.uuid)); - dev_priv->perf.test_config.id = 1; - - dev_priv->perf.test_config.mux_regs = mux_config_test_oa; - dev_priv->perf.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa); - - dev_priv->perf.test_config.b_counter_regs = b_counter_config_test_oa; - dev_priv->perf.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa); - - dev_priv->perf.test_config.flex_regs = flex_eu_config_test_oa; - dev_priv->perf.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa); - - dev_priv->perf.test_config.sysfs_metric.name = "80a833f0-2504-4321-8894-e9277844ce7b"; - dev_priv->perf.test_config.sysfs_metric.attrs = dev_priv->perf.test_config.attrs; - - dev_priv->perf.test_config.attrs[0] = &dev_priv->perf.test_config.sysfs_metric_id.attr; - - dev_priv->perf.test_config.sysfs_metric_id.attr.name = "id"; - dev_priv->perf.test_config.sysfs_metric_id.attr.mode = 0444; - dev_priv->perf.test_config.sysfs_metric_id.show = show_test_oa_id; -} diff --git a/drivers/gpu/drm/i915/oa/i915_oa_tgl.h b/drivers/gpu/drm/i915/oa/i915_oa_tgl.h deleted file mode 100644 index 4c25f0be825c..000000000000 --- a/drivers/gpu/drm/i915/oa/i915_oa_tgl.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2018 Intel Corporation - * - * Autogenerated file by GPU Top : https://github.com/rib/gputop - * DO NOT EDIT manually! - */ - -#ifndef __I915_OA_TGL_H__ -#define __I915_OA_TGL_H__ - -struct drm_i915_private; - -void i915_perf_load_test_config_tgl(struct drm_i915_private *dev_priv); - -#endif diff --git a/drivers/gpu/drm/i915/selftests/i915_active.c b/drivers/gpu/drm/i915/selftests/i915_active.c index ef572a0c2566..4002c984c2e0 100644 --- a/drivers/gpu/drm/i915/selftests/i915_active.c +++ b/drivers/gpu/drm/i915/selftests/i915_active.c @@ -153,7 +153,7 @@ static int live_active_wait(void *arg) if (IS_ERR(active)) return PTR_ERR(active); - i915_active_wait(&active->base); + __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); if (!READ_ONCE(active->retired)) { struct drm_printer p = drm_err_printer(__func__); @@ -201,11 +201,57 @@ static int live_active_retire(void *arg) return err; } +static int live_active_barrier(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *engine; + struct live_active *active; + int err = 0; + + /* Check that we get a callback when requests retire upon waiting */ + + active = __live_alloc(i915); + if (!active) + return -ENOMEM; + + err = i915_active_acquire(&active->base); + if (err) + goto out; + + for_each_uabi_engine(engine, i915) { + err = i915_active_acquire_preallocate_barrier(&active->base, + engine); + if (err) + break; + + i915_active_acquire_barrier(&active->base); + } + + i915_active_release(&active->base); + if (err) + goto out; + + __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); + if (!READ_ONCE(active->retired)) { + pr_err("i915_active not retired after flushing barriers!\n"); + err = -EINVAL; + } + +out: + __live_put(active); + + if (igt_flush_test(i915)) + err = -EIO; + + return err; +} + int i915_active_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { SUBTEST(live_active_wait), SUBTEST(live_active_retire), + SUBTEST(live_active_barrier), }; if (intel_gt_is_wedged(&i915->gt)) @@ -231,7 +277,7 @@ static struct intel_engine_cs *node_to_barrier(struct active_node *it) void i915_active_print(struct i915_active *ref, struct drm_printer *m) { - drm_printf(m, "active %pS:%pS\n", ref->active, ref->retire); + drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire); drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count)); drm_printf(m, "\tpreallocated barriers? %s\n", yesno(!llist_empty(&ref->preallocated_barriers))); @@ -265,28 +311,40 @@ static void spin_unlock_wait(spinlock_t *lock) spin_unlock_irq(lock); } +static void active_flush(struct i915_active *ref, + struct i915_active_fence *active) +{ + struct dma_fence *fence; + + fence = xchg(__active_fence_slot(active), NULL); + if (!fence) + return; + + spin_lock_irq(fence->lock); + __list_del_entry(&active->cb.node); + spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */ + atomic_dec(&ref->count); + + GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); +} + void i915_active_unlock_wait(struct i915_active *ref) { if (i915_active_acquire_if_busy(ref)) { struct active_node *it, *n; + /* Wait for all active callbacks */ rcu_read_lock(); - rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { - struct dma_fence *f; - - /* Wait for all active callbacks */ - f = rcu_dereference(it->base.fence); - if (f) - spin_unlock_wait(f->lock); - } + active_flush(ref, &ref->excl); + rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) + active_flush(ref, &it->base); rcu_read_unlock(); i915_active_release(ref); } /* And wait for the retire callback */ - spin_lock_irq(&ref->tree_lock); - spin_unlock_irq(&ref->tree_lock); + spin_unlock_wait(&ref->tree_lock); /* ... which may have been on a thread instead */ flush_work(&ref->work); diff --git a/drivers/gpu/drm/i915/selftests/i915_buddy.c b/drivers/gpu/drm/i915/selftests/i915_buddy.c index 1b856bae67b5..939a6caebb03 100644 --- a/drivers/gpu/drm/i915/selftests/i915_buddy.c +++ b/drivers/gpu/drm/i915/selftests/i915_buddy.c @@ -298,10 +298,12 @@ static void igt_mm_config(u64 *size, u64 *chunk_size) static int igt_buddy_alloc_smoke(void *arg) { struct i915_buddy_mm mm; - int max_order; + IGT_TIMEOUT(end_time); + I915_RND_STATE(prng); u64 chunk_size; u64 mm_size; - int err; + int *order; + int err, i; igt_mm_config(&mm_size, &chunk_size); @@ -313,10 +315,16 @@ static int igt_buddy_alloc_smoke(void *arg) return err; } - for (max_order = mm.max_order; max_order >= 0; max_order--) { + order = i915_random_order(mm.max_order + 1, &prng); + if (!order) + goto out_fini; + + for (i = 0; i <= mm.max_order; ++i) { struct i915_buddy_block *block; - int order; + int max_order = order[i]; + bool timeout = false; LIST_HEAD(blocks); + int order; u64 total; err = igt_check_mm(&mm); @@ -360,6 +368,11 @@ retry: } total += i915_buddy_block_size(&mm, block); + + if (__igt_timeout(end_time, NULL)) { + timeout = true; + break; + } } while (total < mm.size); if (!err) @@ -373,7 +386,7 @@ retry: pr_err("post-mm check failed\n"); } - if (err) + if (err || timeout) break; cond_resched(); @@ -382,6 +395,8 @@ retry: if (err == -ENOMEM) err = 0; + kfree(order); +out_fini: i915_buddy_fini(&mm); return err; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index 623759b73bb4..88d400b9df88 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -125,8 +125,6 @@ static void pm_resume(struct drm_i915_private *i915) */ with_intel_runtime_pm(&i915->runtime_pm, wakeref) { i915_ggtt_resume(&i915->ggtt); - i915_gem_restore_fences(&i915->ggtt); - i915_gem_resume(i915); } } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c index 06ef88510209..028baae9631f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c @@ -45,8 +45,8 @@ static void quirk_add(struct drm_i915_gem_object *obj, static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects) { - unsigned long unbound, bound, count; struct drm_i915_gem_object *obj; + unsigned long count; count = 0; do { @@ -72,30 +72,6 @@ static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects) pr_debug("Filled GGTT with %lu pages [%llu total]\n", count, ggtt->vm.total / PAGE_SIZE); - bound = 0; - unbound = 0; - list_for_each_entry(obj, objects, st_link) { - GEM_BUG_ON(!obj->mm.quirked); - - if (atomic_read(&obj->bind_count)) - bound++; - else - unbound++; - } - GEM_BUG_ON(bound + unbound != count); - - if (unbound) { - pr_err("%s: Found %lu objects unbound, expected %u!\n", - __func__, unbound, 0); - return -EINVAL; - } - - if (bound != count) { - pr_err("%s: Found %lu objects bound, expected %lu!\n", - __func__, bound, count); - return -EINVAL; - } - if (list_empty(&ggtt->vm.bound_list)) { pr_err("No objects on the GGTT inactive list!\n"); return -EINVAL; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index b342bef5e7c9..2e471500a646 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -331,9 +331,6 @@ static void close_object_list(struct list_head *objects, vma = i915_vma_instance(obj, vm, NULL); if (!IS_ERR(vma)) ignored = i915_vma_unbind(vma); - /* Only ppgtt vma may be closed before the object is freed */ - if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma)) - i915_vma_close(vma); list_del(&obj->st_link); i915_gem_object_put(obj); @@ -591,7 +588,7 @@ static int walk_hole(struct i915_address_space *vm, pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", __func__, addr, vma->size, hole_start, hole_end, err); - goto err_close; + goto err_put; } i915_vma_unpin(vma); @@ -600,14 +597,14 @@ static int walk_hole(struct i915_address_space *vm, pr_err("%s incorrect at %llx + %llx\n", __func__, addr, vma->size); err = -EINVAL; - goto err_close; + goto err_put; } err = i915_vma_unbind(vma); if (err) { pr_err("%s unbind failed at %llx + %llx with err=%d\n", __func__, addr, vma->size, err); - goto err_close; + goto err_put; } GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); @@ -616,13 +613,10 @@ static int walk_hole(struct i915_address_space *vm, "%s timed out at %llx\n", __func__, addr)) { err = -EINTR; - goto err_close; + goto err_put; } } -err_close: - if (!i915_vma_is_ggtt(vma)) - i915_vma_close(vma); err_put: i915_gem_object_put(obj); if (err) @@ -675,7 +669,7 @@ static int pot_hole(struct i915_address_space *vm, addr, hole_start, hole_end, err); - goto err; + goto err_obj; } if (!drm_mm_node_allocated(&vma->node) || @@ -685,7 +679,7 @@ static int pot_hole(struct i915_address_space *vm, i915_vma_unpin(vma); err = i915_vma_unbind(vma); err = -EINVAL; - goto err; + goto err_obj; } i915_vma_unpin(vma); @@ -697,13 +691,10 @@ static int pot_hole(struct i915_address_space *vm, "%s timed out after %d/%d\n", __func__, pot, fls64(hole_end - 1) - 1)) { err = -EINTR; - goto err; + goto err_obj; } } -err: - if (!i915_vma_is_ggtt(vma)) - i915_vma_close(vma); err_obj: i915_gem_object_put(obj); return err; @@ -778,7 +769,7 @@ static int drunk_hole(struct i915_address_space *vm, addr, BIT_ULL(size), hole_start, hole_end, err); - goto err; + goto err_obj; } if (!drm_mm_node_allocated(&vma->node) || @@ -788,7 +779,7 @@ static int drunk_hole(struct i915_address_space *vm, i915_vma_unpin(vma); err = i915_vma_unbind(vma); err = -EINVAL; - goto err; + goto err_obj; } i915_vma_unpin(vma); @@ -799,13 +790,10 @@ static int drunk_hole(struct i915_address_space *vm, "%s timed out after %d/%d\n", __func__, n, count)) { err = -EINTR; - goto err; + goto err_obj; } } -err: - if (!i915_vma_is_ggtt(vma)) - i915_vma_close(vma); err_obj: i915_gem_object_put(obj); kfree(order); @@ -1229,7 +1217,6 @@ static void track_vma_bind(struct i915_vma *vma) { struct drm_i915_gem_object *obj = vma->obj; - atomic_inc(&obj->bind_count); /* track for eviction later */ __i915_gem_object_pin_pages(obj); GEM_BUG_ON(vma->pages); diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h index 34138c7bdd15..5dd5d81646c4 100644 --- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h @@ -37,12 +37,14 @@ selftest(gem, i915_gem_live_selftests) selftest(evict, i915_gem_evict_live_selftests) selftest(hugepages, i915_gem_huge_page_live_selftests) selftest(gem_contexts, i915_gem_context_live_selftests) +selftest(gem_execbuf, i915_gem_execbuffer_live_selftests) selftest(blt, i915_gem_object_blt_live_selftests) selftest(client, i915_gem_client_blt_live_selftests) selftest(reset, intel_reset_live_selftests) selftest(memory_region, intel_memory_region_live_selftests) selftest(hangcheck, intel_hangcheck_live_selftests) selftest(execlists, intel_execlists_live_selftests) +selftest(ring_submission, intel_ring_submission_live_selftests) selftest(perf, i915_perf_live_selftests) /* Here be dragons: keep last to run last! */ selftest(late_gt_pm, intel_gt_pm_late_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h index 5b39bab4da1d..6a2be7d0dd95 100644 --- a/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_mock_selftests.h @@ -16,6 +16,7 @@ * Tests are executed in order by igt/drv_selftest */ selftest(sanitycheck, i915_mock_sanitycheck) /* keep first (igt selfcheck) */ +selftest(shmem, shmem_utils_mock_selftests) selftest(fence, i915_sw_fence_mock_selftests) selftest(scatterlist, scatterlist_mock_selftests) selftest(syncmap, i915_syncmap_mock_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c index d1a1568c47ba..ca0c9dbab713 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf.c +++ b/drivers/gpu/drm/i915/selftests/i915_perf.c @@ -14,10 +14,85 @@ #include "igt_flush_test.h" #include "lib_sw_fence.h" +#define TEST_OA_CONFIG_UUID "12345678-1234-1234-1234-1234567890ab" + +static int +alloc_empty_config(struct i915_perf *perf) +{ + struct i915_oa_config *oa_config; + + oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL); + if (!oa_config) + return -ENOMEM; + + oa_config->perf = perf; + kref_init(&oa_config->ref); + + strlcpy(oa_config->uuid, TEST_OA_CONFIG_UUID, sizeof(oa_config->uuid)); + + mutex_lock(&perf->metrics_lock); + + oa_config->id = idr_alloc(&perf->metrics_idr, oa_config, 2, 0, GFP_KERNEL); + if (oa_config->id < 0) { + mutex_unlock(&perf->metrics_lock); + i915_oa_config_put(oa_config); + return -ENOMEM; + } + + mutex_unlock(&perf->metrics_lock); + + return 0; +} + +static void +destroy_empty_config(struct i915_perf *perf) +{ + struct i915_oa_config *oa_config = NULL, *tmp; + int id; + + mutex_lock(&perf->metrics_lock); + + idr_for_each_entry(&perf->metrics_idr, tmp, id) { + if (!strcmp(tmp->uuid, TEST_OA_CONFIG_UUID)) { + oa_config = tmp; + break; + } + } + + if (oa_config) + idr_remove(&perf->metrics_idr, oa_config->id); + + mutex_unlock(&perf->metrics_lock); + + if (oa_config) + i915_oa_config_put(oa_config); +} + +static struct i915_oa_config * +get_empty_config(struct i915_perf *perf) +{ + struct i915_oa_config *oa_config = NULL, *tmp; + int id; + + mutex_lock(&perf->metrics_lock); + + idr_for_each_entry(&perf->metrics_idr, tmp, id) { + if (!strcmp(tmp->uuid, TEST_OA_CONFIG_UUID)) { + oa_config = i915_oa_config_get(tmp); + break; + } + } + + mutex_unlock(&perf->metrics_lock); + + return oa_config; +} + static struct i915_perf_stream * test_stream(struct i915_perf *perf) { struct drm_i915_perf_open_param param = {}; + struct i915_oa_config *oa_config = get_empty_config(perf); struct perf_open_properties props = { .engine = intel_engine_lookup_user(perf->i915, I915_ENGINE_CLASS_RENDER, @@ -25,13 +100,19 @@ test_stream(struct i915_perf *perf) .sample_flags = SAMPLE_OA_REPORT, .oa_format = IS_GEN(perf->i915, 12) ? I915_OA_FORMAT_A32u40_A4u32_B8_C8 : I915_OA_FORMAT_C4_B8, - .metrics_set = 1, }; struct i915_perf_stream *stream; + if (!oa_config) + return NULL; + + props.metrics_set = oa_config->id; + stream = kzalloc(sizeof(*stream), GFP_KERNEL); - if (!stream) + if (!stream) { + i915_oa_config_put(oa_config); return NULL; + } stream->perf = perf; @@ -42,6 +123,8 @@ test_stream(struct i915_perf *perf) } mutex_unlock(&perf->lock); + i915_oa_config_put(oa_config); + return stream; } @@ -138,8 +221,7 @@ static int live_noa_delay(void *arg) goto out; } - if (rq->engine->emit_init_breadcrumb && - i915_request_timeline(rq)->has_initial_breadcrumb) { + if (rq->engine->emit_init_breadcrumb) { err = rq->engine->emit_init_breadcrumb(rq); if (err) { i915_request_add(rq); @@ -206,6 +288,7 @@ int i915_perf_live_selftests(struct drm_i915_private *i915) SUBTEST(live_noa_delay), }; struct i915_perf *perf = &i915->perf; + int err; if (!perf->metrics_kobj || !perf->ops.enable_metric_set) return 0; @@ -213,5 +296,13 @@ int i915_perf_live_selftests(struct drm_i915_private *i915) if (intel_gt_is_wedged(&i915->gt)) return 0; - return i915_subtests(tests, i915); + err = alloc_empty_config(&i915->perf); + if (err) + return err; + + err = i915_subtests(tests, i915); + + destroy_empty_config(&i915->perf); + + return err; } diff --git a/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h b/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h index 3bf7f53e9924..d8da142985eb 100644 --- a/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h +++ b/drivers/gpu/drm/i915/selftests/i915_perf_selftests.h @@ -16,5 +16,6 @@ * Tests are executed in order by igt/i915_selftest */ selftest(engine_cs, intel_engine_cs_perf_selftests) +selftest(request, i915_request_perf_selftests) selftest(blt, i915_gem_object_blt_perf_selftests) selftest(region, intel_memory_region_perf_selftests) diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index f89d9c42f1fa..6014e8dfcbb1 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -23,11 +23,13 @@ */ #include <linux/prime_numbers.h> +#include <linux/pm_qos.h> #include "gem/i915_gem_pm.h" #include "gem/selftests/mock_context.h" #include "gt/intel_engine_pm.h" +#include "gt/intel_engine_user.h" #include "gt/intel_gt.h" #include "i915_random.h" @@ -51,6 +53,11 @@ static unsigned int num_uabi_engines(struct drm_i915_private *i915) return count; } +static struct intel_engine_cs *rcs0(struct drm_i915_private *i915) +{ + return intel_engine_lookup_user(i915, I915_ENGINE_CLASS_RENDER, 0); +} + static int igt_add_request(void *arg) { struct drm_i915_private *i915 = arg; @@ -58,7 +65,7 @@ static int igt_add_request(void *arg) /* Basic preliminary test to create a request and let it loose! */ - request = mock_request(i915->engine[RCS0]->kernel_context, HZ / 10); + request = mock_request(rcs0(i915)->kernel_context, HZ / 10); if (!request) return -ENOMEM; @@ -76,7 +83,7 @@ static int igt_wait_request(void *arg) /* Submit a request, then wait upon it */ - request = mock_request(i915->engine[RCS0]->kernel_context, T); + request = mock_request(rcs0(i915)->kernel_context, T); if (!request) return -ENOMEM; @@ -145,7 +152,7 @@ static int igt_fence_wait(void *arg) /* Submit a request, treat it as a fence and wait upon it */ - request = mock_request(i915->engine[RCS0]->kernel_context, T); + request = mock_request(rcs0(i915)->kernel_context, T); if (!request) return -ENOMEM; @@ -420,7 +427,7 @@ static int mock_breadcrumbs_smoketest(void *arg) { struct drm_i915_private *i915 = arg; struct smoketest t = { - .engine = i915->engine[RCS0], + .engine = rcs0(i915), .ncontexts = 1024, .max_batch = 1024, .request_alloc = __mock_request_alloc @@ -809,10 +816,12 @@ static int recursive_batch_resolve(struct i915_vma *batch) return PTR_ERR(cmd); *cmd = MI_BATCH_BUFFER_END; - intel_gt_chipset_flush(batch->vm->gt); + __i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd)); i915_gem_object_unpin_map(batch->obj); + intel_gt_chipset_flush(batch->vm->gt); + return 0; } @@ -858,13 +867,6 @@ static int live_all_engines(void *arg) goto out_request; } - err = engine->emit_bb_start(request[idx], - batch->node.start, - batch->node.size, - 0); - GEM_BUG_ON(err); - request[idx]->batch = batch; - i915_vma_lock(batch); err = i915_request_await_object(request[idx], batch->obj, 0); if (err == 0) @@ -872,6 +874,13 @@ static int live_all_engines(void *arg) i915_vma_unlock(batch); GEM_BUG_ON(err); + err = engine->emit_bb_start(request[idx], + batch->node.start, + batch->node.size, + 0); + GEM_BUG_ON(err); + request[idx]->batch = batch; + i915_request_get(request[idx]); i915_request_add(request[idx]); idx++; @@ -986,13 +995,6 @@ static int live_sequential_engines(void *arg) } } - err = engine->emit_bb_start(request[idx], - batch->node.start, - batch->node.size, - 0); - GEM_BUG_ON(err); - request[idx]->batch = batch; - i915_vma_lock(batch); err = i915_request_await_object(request[idx], batch->obj, false); @@ -1001,6 +1003,13 @@ static int live_sequential_engines(void *arg) i915_vma_unlock(batch); GEM_BUG_ON(err); + err = engine->emit_bb_start(request[idx], + batch->node.start, + batch->node.size, + 0); + GEM_BUG_ON(err); + request[idx]->batch = batch; + i915_request_get(request[idx]); i915_request_add(request[idx]); @@ -1053,9 +1062,12 @@ out_request: I915_MAP_WC); if (!IS_ERR(cmd)) { *cmd = MI_BATCH_BUFFER_END; - intel_gt_chipset_flush(engine->gt); + __i915_gem_object_flush_map(request[idx]->batch->obj, + 0, sizeof(*cmd)); i915_gem_object_unpin_map(request[idx]->batch->obj); + + intel_gt_chipset_flush(engine->gt); } i915_vma_put(request[idx]->batch); @@ -1233,7 +1245,7 @@ static int live_parallel_engines(void *arg) struct igt_live_test t; unsigned int idx; - snprintf(name, sizeof(name), "%pS", fn); + snprintf(name, sizeof(name), "%ps", *fn); err = igt_live_test_begin(&t, i915, __func__, name); if (err) break; @@ -1470,3 +1482,572 @@ int i915_request_live_selftests(struct drm_i915_private *i915) return i915_subtests(tests, i915); } + +static int switch_to_kernel_sync(struct intel_context *ce, int err) +{ + struct i915_request *rq; + struct dma_fence *fence; + + rq = intel_engine_create_kernel_request(ce->engine); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + fence = i915_active_fence_get(&ce->timeline->last_request); + if (fence) { + i915_request_await_dma_fence(rq, fence); + dma_fence_put(fence); + } + + rq = i915_request_get(rq); + i915_request_add(rq); + if (i915_request_wait(rq, 0, HZ / 2) < 0 && !err) + err = -ETIME; + i915_request_put(rq); + + while (!err && !intel_engine_is_idle(ce->engine)) + intel_engine_flush_submission(ce->engine); + + return err; +} + +struct perf_stats { + struct intel_engine_cs *engine; + unsigned long count; + ktime_t time; + ktime_t busy; + u64 runtime; +}; + +struct perf_series { + struct drm_i915_private *i915; + unsigned int nengines; + struct intel_context *ce[]; +}; + +static int s_sync0(void *arg) +{ + struct perf_series *ps = arg; + IGT_TIMEOUT(end_time); + unsigned int idx = 0; + int err = 0; + + GEM_BUG_ON(!ps->nengines); + do { + struct i915_request *rq; + + rq = i915_request_create(ps->ce[idx]); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + i915_request_get(rq); + i915_request_add(rq); + + if (i915_request_wait(rq, 0, HZ / 5) < 0) + err = -ETIME; + i915_request_put(rq); + if (err) + break; + + if (++idx == ps->nengines) + idx = 0; + } while (!__igt_timeout(end_time, NULL)); + + return err; +} + +static int s_sync1(void *arg) +{ + struct perf_series *ps = arg; + struct i915_request *prev = NULL; + IGT_TIMEOUT(end_time); + unsigned int idx = 0; + int err = 0; + + GEM_BUG_ON(!ps->nengines); + do { + struct i915_request *rq; + + rq = i915_request_create(ps->ce[idx]); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + i915_request_get(rq); + i915_request_add(rq); + + if (prev && i915_request_wait(prev, 0, HZ / 5) < 0) + err = -ETIME; + i915_request_put(prev); + prev = rq; + if (err) + break; + + if (++idx == ps->nengines) + idx = 0; + } while (!__igt_timeout(end_time, NULL)); + i915_request_put(prev); + + return err; +} + +static int s_many(void *arg) +{ + struct perf_series *ps = arg; + IGT_TIMEOUT(end_time); + unsigned int idx = 0; + + GEM_BUG_ON(!ps->nengines); + do { + struct i915_request *rq; + + rq = i915_request_create(ps->ce[idx]); + if (IS_ERR(rq)) + return PTR_ERR(rq); + + i915_request_add(rq); + + if (++idx == ps->nengines) + idx = 0; + } while (!__igt_timeout(end_time, NULL)); + + return 0; +} + +static int perf_series_engines(void *arg) +{ + struct drm_i915_private *i915 = arg; + static int (* const func[])(void *arg) = { + s_sync0, + s_sync1, + s_many, + NULL, + }; + const unsigned int nengines = num_uabi_engines(i915); + struct intel_engine_cs *engine; + int (* const *fn)(void *arg); + struct pm_qos_request qos; + struct perf_stats *stats; + struct perf_series *ps; + unsigned int idx; + int err = 0; + + stats = kcalloc(nengines, sizeof(*stats), GFP_KERNEL); + if (!stats) + return -ENOMEM; + + ps = kzalloc(struct_size(ps, ce, nengines), GFP_KERNEL); + if (!ps) { + kfree(stats); + return -ENOMEM; + } + + cpu_latency_qos_add_request(&qos, 0); /* disable cstates */ + + ps->i915 = i915; + ps->nengines = nengines; + + idx = 0; + for_each_uabi_engine(engine, i915) { + struct intel_context *ce; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + goto out; + + err = intel_context_pin(ce); + if (err) { + intel_context_put(ce); + goto out; + } + + ps->ce[idx++] = ce; + } + GEM_BUG_ON(idx != ps->nengines); + + for (fn = func; *fn && !err; fn++) { + char name[KSYM_NAME_LEN]; + struct igt_live_test t; + + snprintf(name, sizeof(name), "%ps", *fn); + err = igt_live_test_begin(&t, i915, __func__, name); + if (err) + break; + + for (idx = 0; idx < nengines; idx++) { + struct perf_stats *p = + memset(&stats[idx], 0, sizeof(stats[idx])); + struct intel_context *ce = ps->ce[idx]; + + p->engine = ps->ce[idx]->engine; + intel_engine_pm_get(p->engine); + + if (intel_engine_supports_stats(p->engine)) + p->busy = intel_engine_get_busy_time(p->engine) + 1; + p->runtime = -intel_context_get_total_runtime_ns(ce); + p->time = ktime_get(); + } + + err = (*fn)(ps); + if (igt_live_test_end(&t)) + err = -EIO; + + for (idx = 0; idx < nengines; idx++) { + struct perf_stats *p = &stats[idx]; + struct intel_context *ce = ps->ce[idx]; + int integer, decimal; + u64 busy, dt; + + p->time = ktime_sub(ktime_get(), p->time); + if (p->busy) { + p->busy = ktime_sub(intel_engine_get_busy_time(p->engine), + p->busy - 1); + } + + err = switch_to_kernel_sync(ce, err); + p->runtime += intel_context_get_total_runtime_ns(ce); + intel_engine_pm_put(p->engine); + + busy = 100 * ktime_to_ns(p->busy); + dt = ktime_to_ns(p->time); + if (dt) { + integer = div64_u64(busy, dt); + busy -= integer * dt; + decimal = div64_u64(100 * busy, dt); + } else { + integer = 0; + decimal = 0; + } + + pr_info("%s %5s: { seqno:%d, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n", + name, p->engine->name, ce->timeline->seqno, + integer, decimal, + div_u64(p->runtime, 1000 * 1000), + div_u64(ktime_to_ns(p->time), 1000 * 1000)); + } + } + +out: + for (idx = 0; idx < nengines; idx++) { + if (IS_ERR_OR_NULL(ps->ce[idx])) + break; + + intel_context_unpin(ps->ce[idx]); + intel_context_put(ps->ce[idx]); + } + kfree(ps); + + cpu_latency_qos_remove_request(&qos); + kfree(stats); + return err; +} + +static int p_sync0(void *arg) +{ + struct perf_stats *p = arg; + struct intel_engine_cs *engine = p->engine; + struct intel_context *ce; + IGT_TIMEOUT(end_time); + unsigned long count; + bool busy; + int err = 0; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + err = intel_context_pin(ce); + if (err) { + intel_context_put(ce); + return err; + } + + busy = false; + if (intel_engine_supports_stats(engine)) { + p->busy = intel_engine_get_busy_time(engine); + busy = true; + } + + p->time = ktime_get(); + count = 0; + do { + struct i915_request *rq; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + i915_request_get(rq); + i915_request_add(rq); + + err = 0; + if (i915_request_wait(rq, 0, HZ / 5) < 0) + err = -ETIME; + i915_request_put(rq); + if (err) + break; + + count++; + } while (!__igt_timeout(end_time, NULL)); + p->time = ktime_sub(ktime_get(), p->time); + + if (busy) { + p->busy = ktime_sub(intel_engine_get_busy_time(engine), + p->busy); + } + + err = switch_to_kernel_sync(ce, err); + p->runtime = intel_context_get_total_runtime_ns(ce); + p->count = count; + + intel_context_unpin(ce); + intel_context_put(ce); + return err; +} + +static int p_sync1(void *arg) +{ + struct perf_stats *p = arg; + struct intel_engine_cs *engine = p->engine; + struct i915_request *prev = NULL; + struct intel_context *ce; + IGT_TIMEOUT(end_time); + unsigned long count; + bool busy; + int err = 0; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + err = intel_context_pin(ce); + if (err) { + intel_context_put(ce); + return err; + } + + busy = false; + if (intel_engine_supports_stats(engine)) { + p->busy = intel_engine_get_busy_time(engine); + busy = true; + } + + p->time = ktime_get(); + count = 0; + do { + struct i915_request *rq; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + i915_request_get(rq); + i915_request_add(rq); + + err = 0; + if (prev && i915_request_wait(prev, 0, HZ / 5) < 0) + err = -ETIME; + i915_request_put(prev); + prev = rq; + if (err) + break; + + count++; + } while (!__igt_timeout(end_time, NULL)); + i915_request_put(prev); + p->time = ktime_sub(ktime_get(), p->time); + + if (busy) { + p->busy = ktime_sub(intel_engine_get_busy_time(engine), + p->busy); + } + + err = switch_to_kernel_sync(ce, err); + p->runtime = intel_context_get_total_runtime_ns(ce); + p->count = count; + + intel_context_unpin(ce); + intel_context_put(ce); + return err; +} + +static int p_many(void *arg) +{ + struct perf_stats *p = arg; + struct intel_engine_cs *engine = p->engine; + struct intel_context *ce; + IGT_TIMEOUT(end_time); + unsigned long count; + int err = 0; + bool busy; + + ce = intel_context_create(engine); + if (IS_ERR(ce)) + return PTR_ERR(ce); + + err = intel_context_pin(ce); + if (err) { + intel_context_put(ce); + return err; + } + + busy = false; + if (intel_engine_supports_stats(engine)) { + p->busy = intel_engine_get_busy_time(engine); + busy = true; + } + + count = 0; + p->time = ktime_get(); + do { + struct i915_request *rq; + + rq = i915_request_create(ce); + if (IS_ERR(rq)) { + err = PTR_ERR(rq); + break; + } + + i915_request_add(rq); + count++; + } while (!__igt_timeout(end_time, NULL)); + p->time = ktime_sub(ktime_get(), p->time); + + if (busy) { + p->busy = ktime_sub(intel_engine_get_busy_time(engine), + p->busy); + } + + err = switch_to_kernel_sync(ce, err); + p->runtime = intel_context_get_total_runtime_ns(ce); + p->count = count; + + intel_context_unpin(ce); + intel_context_put(ce); + return err; +} + +static int perf_parallel_engines(void *arg) +{ + struct drm_i915_private *i915 = arg; + static int (* const func[])(void *arg) = { + p_sync0, + p_sync1, + p_many, + NULL, + }; + const unsigned int nengines = num_uabi_engines(i915); + struct intel_engine_cs *engine; + int (* const *fn)(void *arg); + struct pm_qos_request qos; + struct { + struct perf_stats p; + struct task_struct *tsk; + } *engines; + int err = 0; + + engines = kcalloc(nengines, sizeof(*engines), GFP_KERNEL); + if (!engines) + return -ENOMEM; + + cpu_latency_qos_add_request(&qos, 0); + + for (fn = func; *fn; fn++) { + char name[KSYM_NAME_LEN]; + struct igt_live_test t; + unsigned int idx; + + snprintf(name, sizeof(name), "%ps", *fn); + err = igt_live_test_begin(&t, i915, __func__, name); + if (err) + break; + + atomic_set(&i915->selftest.counter, nengines); + + idx = 0; + for_each_uabi_engine(engine, i915) { + intel_engine_pm_get(engine); + + memset(&engines[idx].p, 0, sizeof(engines[idx].p)); + engines[idx].p.engine = engine; + + engines[idx].tsk = kthread_run(*fn, &engines[idx].p, + "igt:%s", engine->name); + if (IS_ERR(engines[idx].tsk)) { + err = PTR_ERR(engines[idx].tsk); + intel_engine_pm_put(engine); + break; + } + get_task_struct(engines[idx++].tsk); + } + + yield(); /* start all threads before we kthread_stop() */ + + idx = 0; + for_each_uabi_engine(engine, i915) { + int status; + + if (IS_ERR(engines[idx].tsk)) + break; + + status = kthread_stop(engines[idx].tsk); + if (status && !err) + err = status; + + intel_engine_pm_put(engine); + put_task_struct(engines[idx++].tsk); + } + + if (igt_live_test_end(&t)) + err = -EIO; + if (err) + break; + + idx = 0; + for_each_uabi_engine(engine, i915) { + struct perf_stats *p = &engines[idx].p; + u64 busy = 100 * ktime_to_ns(p->busy); + u64 dt = ktime_to_ns(p->time); + int integer, decimal; + + if (dt) { + integer = div64_u64(busy, dt); + busy -= integer * dt; + decimal = div64_u64(100 * busy, dt); + } else { + integer = 0; + decimal = 0; + } + + GEM_BUG_ON(engine != p->engine); + pr_info("%s %5s: { count:%lu, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n", + name, engine->name, p->count, integer, decimal, + div_u64(p->runtime, 1000 * 1000), + div_u64(ktime_to_ns(p->time), 1000 * 1000)); + idx++; + } + } + + cpu_latency_qos_remove_request(&qos); + kfree(engines); + return err; +} + +int i915_request_perf_selftests(struct drm_i915_private *i915) +{ + static const struct i915_subtest tests[] = { + SUBTEST(perf_series_engines), + SUBTEST(perf_parallel_engines), + }; + + if (intel_gt_is_wedged(&i915->gt)) + return 0; + + return i915_subtests(tests, i915); +} diff --git a/drivers/gpu/drm/i915/selftests/i915_selftest.c b/drivers/gpu/drm/i915/selftests/i915_selftest.c index d3bf9eefb682..1bc11c09faef 100644 --- a/drivers/gpu/drm/i915/selftests/i915_selftest.c +++ b/drivers/gpu/drm/i915/selftests/i915_selftest.c @@ -396,6 +396,35 @@ bool __igt_timeout(unsigned long timeout, const char *fmt, ...) return true; } +void igt_hexdump(const void *buf, size_t len) +{ + const size_t rowsize = 8 * sizeof(u32); + const void *prev = NULL; + bool skip = false; + size_t pos; + + for (pos = 0; pos < len; pos += rowsize) { + char line[128]; + + if (prev && !memcmp(prev, buf + pos, rowsize)) { + if (!skip) { + pr_info("*\n"); + skip = true; + } + continue; + } + + WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos, + rowsize, sizeof(u32), + line, sizeof(line), + false) >= sizeof(line)); + pr_info("[%04zx] %s\n", pos, line); + + prev = buf + pos; + skip = false; + } +} + module_param_named(st_random_seed, i915_selftest.random_seed, uint, 0400); module_param_named(st_timeout, i915_selftest.timeout_ms, uint, 0400); module_param_named(st_filter, i915_selftest.filter, charp, 0400); diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index 58b5f40a07dd..af89c7fc8f59 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -173,7 +173,7 @@ static int igt_vma_create(void *arg) } nc = 0; - for_each_prime_number(num_ctx, 2 * NUM_CONTEXT_TAG) { + for_each_prime_number(num_ctx, 2 * BITS_PER_LONG) { for (; nc < num_ctx; nc++) { ctx = mock_context(i915, "mock"); if (!ctx) diff --git a/drivers/gpu/drm/i915/selftests/igt_spinner.c b/drivers/gpu/drm/i915/selftests/igt_spinner.c index e8a58fe49c39..e35ba5f9e73f 100644 --- a/drivers/gpu/drm/i915/selftests/igt_spinner.c +++ b/drivers/gpu/drm/i915/selftests/igt_spinner.c @@ -169,8 +169,7 @@ igt_spinner_create_request(struct igt_spinner *spin, intel_gt_chipset_flush(engine->gt); - if (engine->emit_init_breadcrumb && - i915_request_timeline(rq)->has_initial_breadcrumb) { + if (engine->emit_init_breadcrumb) { err = engine->emit_init_breadcrumb(rq); if (err) goto cancel_rq; @@ -183,7 +182,7 @@ igt_spinner_create_request(struct igt_spinner *spin, cancel_rq: if (err) { - i915_request_skip(rq, err); + i915_request_set_error_once(rq, err); i915_request_add(rq); } unpin_hws: diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index 2a1d4ba1f9f3..6e80d99048e4 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -594,8 +594,11 @@ create_region_for_mapping(struct intel_memory_region *mr, u64 size, u32 type, void *addr; obj = i915_gem_object_create_region(mr, size, 0); - if (IS_ERR(obj)) + if (IS_ERR(obj)) { + if (PTR_ERR(obj) == -ENOSPC) /* Stolen memory */ + return ERR_PTR(-ENODEV); return obj; + } addr = i915_gem_object_pin_map(obj, type); if (IS_ERR(addr)) { diff --git a/drivers/gpu/drm/i915/selftests/librapl.c b/drivers/gpu/drm/i915/selftests/librapl.c new file mode 100644 index 000000000000..58710ac3f979 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/librapl.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include <asm/msr.h> + +#include "librapl.h" + +u64 librapl_energy_uJ(void) +{ + unsigned long long power; + u32 units; + + if (rdmsrl_safe(MSR_RAPL_POWER_UNIT, &power)) + return 0; + + units = (power & 0x1f00) >> 8; + + if (rdmsrl_safe(MSR_PP1_ENERGY_STATUS, &power)) + return 0; + + return (1000000 * power) >> units; /* convert to uJ */ +} diff --git a/drivers/gpu/drm/i915/selftests/librapl.h b/drivers/gpu/drm/i915/selftests/librapl.h new file mode 100644 index 000000000000..887f3e91dd05 --- /dev/null +++ b/drivers/gpu/drm/i915/selftests/librapl.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef SELFTEST_LIBRAPL_H +#define SELFTEST_LIBRAPL_H + +#include <linux/types.h> + +u64 librapl_energy_uJ(void); + +#endif /* SELFTEST_LIBRAPL_H */ diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 3b8986983afc..47fde54150f4 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -144,7 +144,6 @@ struct drm_i915_private *mock_gem_device(void) goto put_device; } i915->drm.pdev = pdev; - i915->drm.dev_private = i915; intel_runtime_pm_init_early(&i915->runtime_pm); @@ -179,11 +178,11 @@ struct drm_i915_private *mock_gem_device(void) mkwrite_device_info(i915)->engine_mask = BIT(0); - i915->engine[RCS0] = mock_engine(i915, "mock", RCS0); - if (!i915->engine[RCS0]) + i915->gt.engine[RCS0] = mock_engine(i915, "mock", RCS0); + if (!i915->gt.engine[RCS0]) goto err_unlock; - if (mock_engine_init(i915->engine[RCS0])) + if (mock_engine_init(i915->gt.engine[RCS0])) goto err_context; __clear_bit(I915_WEDGED, &i915->gt.reset.flags); |