drm/nouveau/fifo: merge gk104_fifo_func into nvkm_host_func

This makes it easier to transition everything.

- a couple of function renames for collisions

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>

14 files changed, 219 insertions, 111 deletions

diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index b06db3b3d471..4e099bf0a3fa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -30,8 +30,6 @@
 #include <nvif/cl0080.h>
 #include <nvif/unpack.h>
 
-#include "gk104.h"
-
 void
 nvkm_fifo_recover_chan(struct nvkm_fifo *fifo, int chid)
 {
@@ -58,7 +56,7 @@ nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags)
 void
 nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info)
 {
-	return fifo->func->fault(fifo, info);
+	return fifo->func->mmu_fault->recover(fifo, info);
 }
 
 void
@@ -179,9 +177,6 @@ nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_dev
 	const struct nvkm_fifo_func_chan *chan = &fifo->func->chan;
 	int c = 0;
 
-	if (fifo->func->engine_id == gk104_fifo_engine_id)
-		chan = &gk104_fifo(fifo)->func->chan;
-
 	/* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */
 	if (chan->user.oclass) {
 		if (c++ == index) {
@@ -289,7 +284,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device,
 
 	INIT_LIST_HEAD(&fifo->chan);
 
-	nr = func->chid_nr ? func->chid_nr(fifo) : gk104_fifo(fifo)->func->chid_nr(fifo);
+	nr = func->chid_nr(fifo);
 	if (WARN_ON(fifo->nr > NVKM_FIFO_CHID_NR))
 		fifo->nr = NVKM_FIFO_CHID_NR;
 	else
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index bd0b0c66e28f..1a641b1fdfbf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -359,6 +359,11 @@ gf100_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info)
 	nvkm_fifo_chan_put(&fifo->base, flags, &chan);
 }
 
+static const struct nvkm_fifo_func_mmu_fault
+gf100_fifo_mmu_fault = {
+	.recover = gf100_fifo_fault,
+};
+
 static const struct nvkm_enum
 gf100_fifo_sched_reason[] = {
 	{ 0x0a, "CTXSW_TIMEOUT" },
@@ -422,7 +427,7 @@ gf100_fifo_intr_sched(struct gf100_fifo *fifo)
 }
 
 void
-gf100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit)
+gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit)
 {
 	struct nvkm_device *device = fifo->engine.subdev.device;
 	u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10));
@@ -541,7 +546,7 @@ gf100_fifo_intr(struct nvkm_fifo *base)
 		u32 mask = nvkm_rd32(device, 0x00259c);
 		while (mask) {
 			u32 unit = __ffs(mask);
-			gf100_fifo_intr_fault(&fifo->base, unit);
+			gf100_fifo_intr_mmu_fault_unit(&fifo->base, unit);
 			nvkm_wr32(device, 0x00259c, (1 << unit));
 			mask &= ~(1 << unit);
 		}
@@ -679,7 +684,7 @@ gf100_fifo = {
 	.init = gf100_fifo_init,
 	.fini = gf100_fifo_fini,
 	.intr = gf100_fifo_intr,
-	.fault = gf100_fifo_fault,
+	.mmu_fault = &gf100_fifo_mmu_fault,
 	.engine_id = gf100_fifo_engine_id,
 	.id_engine = gf100_fifo_id_engine,
 	.uevent_init = gf100_fifo_uevent_init,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index da26c5219e53..880ddfec22f7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -397,7 +397,7 @@ gk104_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid)
 	return NULL;
 }
 
-static void
+void
 gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid)
 {
 	struct gk104_fifo *fifo = gk104_fifo(base);
@@ -629,7 +629,7 @@ gk104_fifo_fault_gpcclient[] = {
 	{}
 };
 
-static void
+void
 gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info)
 {
 	struct gk104_fifo *fifo = gk104_fifo(base);
@@ -709,6 +709,11 @@ gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info)
 	spin_unlock_irqrestore(&fifo->base.lock, flags);
 }
 
+const struct nvkm_fifo_func_mmu_fault
+gk104_fifo_mmu_fault = {
+	.recover = gk104_fifo_fault,
+};
+
 static const struct nvkm_enum
 gk104_fifo_bind_reason[] = {
 	{ 0x01, "BIND_NOT_UNBOUND" },
@@ -869,7 +874,7 @@ gk104_fifo_intr_engine(struct gk104_fifo *fifo)
 	nvkm_fifo_uevent(&fifo->base);
 }
 
-static void
+void
 gk104_fifo_intr(struct nvkm_fifo *base)
 {
 	struct gk104_fifo *fifo = gk104_fifo(base);
@@ -924,7 +929,7 @@ gk104_fifo_intr(struct nvkm_fifo *base)
 		u32 mask = nvkm_rd32(device, 0x00259c);
 		while (mask) {
 			u32 unit = __ffs(mask);
-			fifo->func->intr.fault(&fifo->base, unit);
+			fifo->func->intr_mmu_fault_unit(&fifo->base, unit);
 			nvkm_wr32(device, 0x00259c, (1 << unit));
 			mask &= ~(1 << unit);
 		}
@@ -1168,22 +1173,6 @@ gk104_fifo_dtor(struct nvkm_fifo *base)
 	return fifo;
 }
 
-static const struct nvkm_fifo_func
-gk104_fifo_ = {
-	.dtor = gk104_fifo_dtor,
-	.oneinit = gk104_fifo_oneinit,
-	.info = gk104_fifo_info,
-	.init = gk104_fifo_init,
-	.fini = gk104_fifo_fini,
-	.intr = gk104_fifo_intr,
-	.fault = gk104_fifo_fault,
-	.engine_id = gk104_fifo_engine_id,
-	.id_engine = gk104_fifo_id_engine,
-	.uevent_init = gk104_fifo_uevent_init,
-	.uevent_fini = gk104_fifo_uevent_fini,
-	.recover_chan = gk104_fifo_recover_chan,
-};
-
 int
 gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device,
 		enum nvkm_subdev_type type, int inst, int nr, struct nvkm_fifo **pfifo)
@@ -1196,20 +1185,32 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device,
 	INIT_WORK(&fifo->recover.work, gk104_fifo_recover_work);
 	*pfifo = &fifo->base;
 
-	return nvkm_fifo_ctor(&gk104_fifo_, device, type, inst, &fifo->base);
+	return nvkm_fifo_ctor(func, device, type, inst, &fifo->base);
 }
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gk104_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk104_fifo_chid_nr,
-	.intr.fault = gf100_fifo_intr_fault,
-	.pbdma = &gk104_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gk104_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk104_fifo_runlist,
+	.pbdma = &gk104_fifo_pbdma,
 	.cgrp = {{                               }, &nv04_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
index be426734da02..962187740010 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h
@@ -8,6 +8,8 @@ struct nvkm_fifo_cgrp;
 #include <core/enum.h>
 #include <subdev/mmu.h>
 
+#define gk104_fifo_func nvkm_fifo_func
+
 struct gk104_fifo_chan;
 struct gk104_fifo {
 	const struct gk104_fifo_func *func;
@@ -45,41 +47,6 @@ struct gk104_fifo {
 	} user;
 };
 
-struct gk104_fifo_func {
-	int (*chid_nr)(struct nvkm_fifo *);
-
-	struct {
-		void (*fault)(struct nvkm_fifo *, int unit);
-	} intr;
-
-	const struct gk104_fifo_pbdma_func {
-		int (*nr)(struct gk104_fifo *);
-		void (*init)(struct gk104_fifo *);
-		void (*init_timeout)(struct gk104_fifo *);
-	} *pbdma;
-
-	struct {
-		const struct nvkm_enum *access;
-		const struct nvkm_enum *engine;
-		const struct nvkm_enum *reason;
-		const struct nvkm_enum *hubclient;
-		const struct nvkm_enum *gpcclient;
-	} fault;
-
-	const struct gk104_fifo_runlist_func {
-		u8 size;
-		void (*cgrp)(struct nvkm_fifo_cgrp *,
-			     struct nvkm_memory *, u32 offset);
-		void (*chan)(struct gk104_fifo_chan *,
-			     struct nvkm_memory *, u32 offset);
-		void (*commit)(struct gk104_fifo *, int runl,
-			       struct nvkm_memory *, int entries);
-	} *runlist;
-
-	struct nvkm_fifo_func_cgrp cgrp;
-	struct nvkm_fifo_func_chan chan;
-};
-
 struct gk104_fifo_engine_status {
 	bool busy;
 	bool faulted;
@@ -135,14 +102,12 @@ void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *,
 extern const struct gk104_fifo_pbdma_func gk208_fifo_pbdma;
 void gk208_fifo_pbdma_init_timeout(struct gk104_fifo *);
 
-void gm107_fifo_intr_fault(struct nvkm_fifo *, int);
 extern const struct nvkm_enum gm107_fifo_fault_engine[];
 extern const struct gk104_fifo_runlist_func gm107_fifo_runlist;
 
 extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma;
 int gm200_fifo_pbdma_nr(struct gk104_fifo *);
 
-void gp100_fifo_intr_fault(struct nvkm_fifo *, int);
 extern const struct nvkm_enum gp100_fifo_fault_engine[];
 
 extern const struct nvkm_enum gv100_fifo_fault_access[];
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
index df9b83cf382f..9dcc54f1d016 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c
@@ -56,17 +56,29 @@ gk110_fifo_runlist = {
 	.commit = gk104_fifo_runlist_commit,
 };
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gk110_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk104_fifo_chid_nr,
-	.intr.fault = gf100_fifo_intr_fault,
-	.pbdma = &gk104_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gk104_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
+	.pbdma = &gk104_fifo_pbdma,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
index 003f244f89db..28791baf77f8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c
@@ -49,17 +49,29 @@ gk208_fifo_chid_nr(struct nvkm_fifo *fifo)
 	return 1024;
 }
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gk208_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gk208_fifo_chid_nr,
-	.intr.fault = gf100_fifo_intr_fault,
-	.pbdma = &gk208_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gk104_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
+	.pbdma = &gk208_fifo_pbdma,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
index 661c9f0cf7b9..b62e3bb0294c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c
@@ -24,17 +24,29 @@
 
 #include <nvif/class.h>
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gk20a_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = nv50_fifo_chid_nr,
-	.intr.fault = gf100_fifo_intr_fault,
-	.pbdma = &gk208_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gk104_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gk104_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gk110_fifo_runlist,
+	.pbdma = &gk208_fifo_pbdma,
 	.cgrp = {{                               }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
index 910d97911a98..f996f1c1141a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c
@@ -74,8 +74,13 @@ gm107_fifo_fault_engine[] = {
 	{}
 };
 
+const struct nvkm_fifo_func_mmu_fault
+gm107_fifo_mmu_fault = {
+	.recover = gk104_fifo_fault,
+};
+
 void
-gm107_fifo_intr_fault(struct nvkm_fifo *fifo, int unit)
+gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit)
 {
 	struct nvkm_device *device = fifo->engine.subdev.device;
 	u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10));
@@ -104,17 +109,29 @@ gm107_fifo_chid_nr(struct nvkm_fifo *fifo)
 	return 2048;
 }
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gm107_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm107_fifo_chid_nr,
-	.intr.fault = gm107_fifo_intr_fault,
-	.pbdma = &gk208_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gm107_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gm107_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
+	.pbdma = &gk208_fifo_pbdma,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
index 101027536a93..8f3da21ce1e7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c
@@ -46,17 +46,29 @@ gm200_fifo_chid_nr(struct nvkm_fifo *fifo)
 	return nvkm_rd32(fifo->engine.subdev.device, 0x002008);
 }
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gm200_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
-	.intr.fault = gm107_fifo_intr_fault,
-	.pbdma = &gm200_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gm107_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gm107_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
+	.pbdma = &gm200_fifo_pbdma,
 	.cgrp = {{ 0, 0,  KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp },
 	.chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
index c52a4a0b361b..dc590848c25c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -52,8 +52,13 @@ gp100_fifo_fault_engine[] = {
 	{}
 };
 
+static const struct nvkm_fifo_func_mmu_fault
+gp100_fifo_mmu_fault = {
+	.recover = gk104_fifo_fault,
+};
+
 void
-gp100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit)
+gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit)
 {
 	struct nvkm_device *device = fifo->engine.subdev.device;
 	u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10));
@@ -76,17 +81,29 @@ gp100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit)
 	nvkm_fifo_fault(fifo, &info);
 }
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func
 gp100_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
-	.intr.fault = gp100_fifo_intr_fault,
-	.pbdma = &gm200_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.intr_mmu_fault_unit = gp100_fifo_intr_mmu_fault_unit,
+	.mmu_fault = &gp100_fifo_mmu_fault,
 	.fault.access = gk104_fifo_fault_access,
 	.fault.engine = gp100_fifo_fault_engine,
 	.fault.reason = gk104_fifo_fault_reason,
 	.fault.hubclient = gk104_fifo_fault_hubclient,
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gm107_fifo_runlist,
+	.pbdma = &gm200_fifo_pbdma,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
index ddca29852865..36c02b5aac0f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c
@@ -291,16 +291,33 @@ gv100_fifo_fault_access[] = {
 	{}
 };
 
-static const struct gk104_fifo_func
+static const struct nvkm_fifo_func_mmu_fault
+gv100_fifo_mmu_fault = {
+	.recover = gk104_fifo_fault,
+};
+
+static const struct nvkm_fifo_func
 gv100_fifo = {
+	.dtor = gk104_fifo_dtor,
+	.oneinit = gk104_fifo_oneinit,
 	.chid_nr = gm200_fifo_chid_nr,
-	.pbdma = &gm200_fifo_pbdma,
+	.info = gk104_fifo_info,
+	.init = gk104_fifo_init,
+	.fini = gk104_fifo_fini,
+	.intr = gk104_fifo_intr,
+	.mmu_fault = &gv100_fifo_mmu_fault,
 	.fault.access = gv100_fifo_fault_access,
 	.fault.engine = gv100_fifo_fault_engine,
 	.fault.reason = gv100_fifo_fault_reason,
 	.fault.hubclient = gv100_fifo_fault_hubclient,
 	.fault.gpcclient = gv100_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = gk104_fifo_recover_chan,
 	.runlist = &gv100_fifo_runlist,
+	.pbdma = &gm200_fifo_pbdma,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0,  VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new },
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index b8031c212c8b..ba3dbc86435e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -4,7 +4,9 @@
 #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine)
 #include <engine/fifo.h>
 struct nvkm_cgrp;
+struct nvkm_memory;
 struct gk104_fifo;
+struct gk104_fifo_chan;
 
 void nvkm_fifo_uevent(struct nvkm_fifo *);
 void nvkm_fifo_kevent(struct nvkm_fifo *, int chid);
@@ -23,8 +25,22 @@ struct nvkm_fifo_func {
 	int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data);
 	void (*init)(struct nvkm_fifo *);
 	void (*fini)(struct nvkm_fifo *);
+
 	void (*intr)(struct nvkm_fifo *);
-	void (*fault)(struct nvkm_fifo *, struct nvkm_fault_data *);
+	void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit);
+
+	const struct nvkm_fifo_func_mmu_fault {
+		void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *);
+	} *mmu_fault;
+
+	struct {
+		const struct nvkm_enum *access;
+		const struct nvkm_enum *engine;
+		const struct nvkm_enum *reason;
+		const struct nvkm_enum *hubclient;
+		const struct nvkm_enum *gpcclient;
+	} fault;
+
 	int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *);
 	struct nvkm_engine *(*id_engine)(struct nvkm_fifo *, int engi);
 	void (*pause)(struct nvkm_fifo *, unsigned long *);
@@ -33,6 +49,22 @@ struct nvkm_fifo_func {
 	void (*uevent_fini)(struct nvkm_fifo *);
 	void (*recover_chan)(struct nvkm_fifo *, int chid);
 
+	const struct gk104_fifo_runlist_func {
+		u8 size;
+		void (*cgrp)(struct nvkm_fifo_cgrp *,
+			     struct nvkm_memory *, u32 offset);
+		void (*chan)(struct gk104_fifo_chan *,
+			     struct nvkm_memory *, u32 offset);
+		void (*commit)(struct gk104_fifo *, int runl,
+			       struct nvkm_memory *, int entries);
+	} *runlist;
+
+	const struct gk104_fifo_pbdma_func {
+		int (*nr)(struct gk104_fifo *);
+		void (*init)(struct gk104_fifo *);
+		void (*init_timeout)(struct gk104_fifo *);
+	} *pbdma;
+
 	struct nvkm_fifo_func_cgrp {
 		struct nvkm_sclass user;
 		const struct nvkm_cgrp_func *func;
@@ -67,19 +99,29 @@ int nv50_fifo_chid_nr(struct nvkm_fifo *);
 
 extern const struct nvkm_chan_func g84_chan;
 
-void gf100_fifo_intr_fault(struct nvkm_fifo *, int);
+void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
 
 int gk104_fifo_chid_nr(struct nvkm_fifo *);
+void gk104_fifo_intr(struct nvkm_fifo *);
+extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault;
+void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *);
+void gk104_fifo_recover_chan(struct nvkm_fifo *, int);
 int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *);
 struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int);
 
 extern const struct nvkm_cgrp_func gk110_cgrp;
 extern const struct nvkm_chan_func gk110_chan;
 
+void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
+extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault;
 extern const struct nvkm_chan_func gm107_chan;
 
 int gm200_fifo_chid_nr(struct nvkm_fifo *);
 
+void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int);
+
+extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault;
+
 int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *,
 		   void *argv, u32 argc, struct nvkm_object **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
index db9b802bd67c..274096316a1b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c
@@ -339,6 +339,11 @@ tu102_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info)
 	spin_unlock_irqrestore(&fifo->base.lock, flags);
 }
 
+const struct nvkm_fifo_func_mmu_fault
+tu102_fifo_mmu_fault = {
+	.recover = tu102_fifo_fault,
+};
+
 static void
 tu102_fifo_intr_ctxsw_timeout(struct gk104_fifo *fifo)
 {
@@ -433,31 +438,27 @@ tu102_fifo_intr(struct nvkm_fifo *base)
 }
 
 static const struct nvkm_fifo_func
-tu102_fifo_ = {
+tu102_fifo = {
 	.dtor = gk104_fifo_dtor,
 	.oneinit = gk104_fifo_oneinit,
+	.chid_nr = gm200_fifo_chid_nr,
 	.info = gk104_fifo_info,
 	.init = gk104_fifo_init,
 	.fini = gk104_fifo_fini,
 	.intr = tu102_fifo_intr,
-	.fault = tu102_fifo_fault,
-	.engine_id = gk104_fifo_engine_id,
-	.id_engine = gk104_fifo_id_engine,
-	.uevent_init = gk104_fifo_uevent_init,
-	.uevent_fini = gk104_fifo_uevent_fini,
-	.recover_chan = tu102_fifo_recover_chan,
-};
-
-static const struct gk104_fifo_func
-tu102_fifo = {
-	.chid_nr = gm200_fifo_chid_nr,
-	.pbdma = &tu102_fifo_pbdma,
+	.mmu_fault = &tu102_fifo_mmu_fault,
 	.fault.access = gv100_fifo_fault_access,
 	.fault.engine = tu102_fifo_fault_engine,
 	.fault.reason = gv100_fifo_fault_reason,
 	.fault.hubclient = gv100_fifo_fault_hubclient,
 	.fault.gpcclient = gv100_fifo_fault_gpcclient,
+	.engine_id = gk104_fifo_engine_id,
+	.id_engine = gk104_fifo_id_engine,
+	.uevent_init = gk104_fifo_uevent_init,
+	.uevent_fini = gk104_fifo_uevent_fini,
+	.recover_chan = tu102_fifo_recover_chan,
 	.runlist = &tu102_fifo_runlist,
+	.pbdma = &tu102_fifo_pbdma,
 	.cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A  }, &gk110_cgrp, .force = true },
 	.chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new },
 };
@@ -474,5 +475,5 @@ tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst,
 	INIT_WORK(&fifo->recover.work, tu102_fifo_recover_work);
 	*pfifo = &fifo->base;
 
-	return nvkm_fifo_ctor(&tu102_fifo_, device, type, inst, &fifo->base);
+	return nvkm_fifo_ctor(&tu102_fifo, device, type, inst, &fifo->base);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c
index a1d93f28b634..42b49043deef 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c
@@ -162,8 +162,8 @@ nvkm_uchan_new(struct nvkm_fifo *fifo, struct nvkm_cgrp *cgrp, const struct nvkm
 	nvkm_object_ctor(&nvkm_uchan, oclass, &uchan->object);
 	*pobject = &uchan->object;
 
-	if (!fifo->func->chan.func)
-		ret = gk104_fifo(fifo)->func->chan.ctor(gk104_fifo(fifo), oclass, argv, argc, &object);
+	if (fifo->func->chan.ctor)
+		ret = fifo->func->chan.ctor(gk104_fifo(fifo), oclass, argv, argc, &object);
 	else
 		ret = fifo->func->chan.oclass->ctor(fifo, oclass, argv, argc, &object);
 	if (!object)