1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
|
/*
* Copyright 2014 Advanced Micro Devices, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
#ifndef KFD_PRIV_H_INCLUDED
#define KFD_PRIV_H_INCLUDED
#include <linux/hashtable.h>
#include <linux/mmu_notifier.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include <linux/atomic.h>
#include <linux/workqueue.h>
#include <linux/spinlock.h>
#include <linux/kfd_ioctl.h>
#include <linux/idr.h>
#include <linux/kfifo.h>
#include <linux/seq_file.h>
#include <linux/kref.h>
#include <linux/sysfs.h>
#include <linux/device_cgroup.h>
#include <drm/drm_file.h>
#include <drm/drm_drv.h>
#include <drm/drm_device.h>
#include <drm/drm_ioctl.h>
#include <kgd_kfd_interface.h>
#include <linux/swap.h>
#include "amd_shared.h"
#define KFD_MAX_RING_ENTRY_SIZE 8
#define KFD_SYSFS_FILE_MODE 0444
/* GPU ID hash width in bits */
#define KFD_GPU_ID_HASH_WIDTH 16
/* Use upper bits of mmap offset to store KFD driver specific information.
* BITS[63:62] - Encode MMAP type
* BITS[61:46] - Encode gpu_id. To identify to which GPU the offset belongs to
* BITS[45:0] - MMAP offset value
*
* NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these
* defines are w.r.t to PAGE_SIZE
*/
#define KFD_MMAP_TYPE_SHIFT 62
#define KFD_MMAP_TYPE_MASK (0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_DOORBELL (0x3ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_EVENTS (0x2ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_RESERVED_MEM (0x1ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_TYPE_MMIO (0x0ULL << KFD_MMAP_TYPE_SHIFT)
#define KFD_MMAP_GPU_ID_SHIFT 46
#define KFD_MMAP_GPU_ID_MASK (((1ULL << KFD_GPU_ID_HASH_WIDTH) - 1) \
<< KFD_MMAP_GPU_ID_SHIFT)
#define KFD_MMAP_GPU_ID(gpu_id) ((((uint64_t)gpu_id) << KFD_MMAP_GPU_ID_SHIFT)\
& KFD_MMAP_GPU_ID_MASK)
#define KFD_MMAP_GET_GPU_ID(offset) ((offset & KFD_MMAP_GPU_ID_MASK) \
>> KFD_MMAP_GPU_ID_SHIFT)
/*
* When working with cp scheduler we should assign the HIQ manually or via
* the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
* definitions for Kaveri. In Kaveri only the first ME queues participates
* in the cp scheduling taking that in mind we set the HIQ slot in the
* second ME.
*/
#define KFD_CIK_HIQ_PIPE 4
#define KFD_CIK_HIQ_QUEUE 0
/* Macro for allocating structures */
#define kfd_alloc_struct(ptr_to_struct) \
((typeof(ptr_to_struct)) kzalloc(sizeof(*ptr_to_struct), GFP_KERNEL))
#define KFD_MAX_NUM_OF_PROCESSES 512
#define KFD_MAX_NUM_OF_QUEUES_PER_PROCESS 1024
/*
* Size of the per-process TBA+TMA buffer: 2 pages
*
* The first page is the TBA used for the CWSR ISA code. The second
* page is used as TMA for user-mode trap handler setup in daisy-chain mode.
*/
#define KFD_CWSR_TBA_TMA_SIZE (PAGE_SIZE * 2)
#define KFD_CWSR_TMA_OFFSET PAGE_SIZE
#define KFD_MAX_NUM_OF_QUEUES_PER_DEVICE \
(KFD_MAX_NUM_OF_PROCESSES * \
KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
#define KFD_KERNEL_QUEUE_SIZE 2048
#define KFD_UNMAP_LATENCY_MS (4000)
/*
* 512 = 0x200
* The doorbell index distance between SDMA RLC (2*i) and (2*i+1) in the
* same SDMA engine on SOC15, which has 8-byte doorbells for SDMA.
* 512 8-byte doorbell distance (i.e. one page away) ensures that SDMA RLC
* (2*i+1) doorbells (in terms of the lower 12 bit address) lie exactly in
* the OFFSET and SIZE set in registers like BIF_SDMA0_DOORBELL_RANGE.
*/
#define KFD_QUEUE_DOORBELL_MIRROR_OFFSET 512
/*
* Kernel module parameter to specify maximum number of supported queues per
* device
*/
extern int max_num_of_queues_per_device;
/* Kernel module parameter to specify the scheduling policy */
extern int sched_policy;
/*
* Kernel module parameter to specify the maximum process
* number per HW scheduler
*/
extern int hws_max_conc_proc;
extern int cwsr_enable;
/*
* Kernel module parameter to specify whether to send sigterm to HSA process on
* unhandled exception
*/
extern int send_sigterm;
/*
* This kernel module is used to simulate large bar machine on non-large bar
* enabled machines.
*/
extern int debug_largebar;
/*
* Ignore CRAT table during KFD initialization, can be used to work around
* broken CRAT tables on some AMD systems
*/
extern int ignore_crat;
/* Set sh_mem_config.retry_disable on GFX v9 */
extern int amdgpu_noretry;
/* Halt if HWS hang is detected */
extern int halt_if_hws_hang;
/* Whether MEC FW support GWS barriers */
extern bool hws_gws_support;
/* Queue preemption timeout in ms */
extern int queue_preemption_timeout_ms;
/* Enable eviction debug messages */
extern bool debug_evictions;
enum cache_policy {
cache_policy_coherent,
cache_policy_noncoherent
};
#define KFD_IS_SOC15(chip) ((chip) >= CHIP_VEGA10)
struct kfd_event_interrupt_class {
bool (*interrupt_isr)(struct kfd_dev *dev,
const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
bool *patched_flag);
void (*interrupt_wq)(struct kfd_dev *dev,
const uint32_t *ih_ring_entry);
};
struct kfd_device_info {
enum amd_asic_type asic_family;
const char *asic_name;
const struct kfd_event_interrupt_class *event_interrupt_class;
unsigned int max_pasid_bits;
unsigned int max_no_of_hqd;
unsigned int doorbell_size;
size_t ih_ring_entry_size;
uint8_t num_of_watch_points;
uint16_t mqd_size_aligned;
bool supports_cwsr;
bool needs_iommu_device;
bool needs_pci_atomics;
unsigned int num_sdma_engines;
unsigned int num_xgmi_sdma_engines;
unsigned int num_sdma_queues_per_engine;
};
struct kfd_mem_obj {
uint32_t range_start;
uint32_t range_end;
uint64_t gpu_addr;
uint32_t *cpu_ptr;
void *gtt_mem;
};
struct kfd_vmid_info {
uint32_t first_vmid_kfd;
uint32_t last_vmid_kfd;
uint32_t vmid_num_kfd;
};
struct kfd_dev {
struct kgd_dev *kgd;
const struct kfd_device_info *device_info;
struct pci_dev *pdev;
struct drm_device *ddev;
unsigned int id; /* topology stub index */
phys_addr_t doorbell_base; /* Start of actual doorbells used by
* KFD. It is aligned for mapping
* into user mode
*/
size_t doorbell_base_dw_offset; /* Offset from the start of the PCI
* doorbell BAR to the first KFD
* doorbell in dwords. GFX reserves
* the segment before this offset.
*/
u32 __iomem *doorbell_kernel_ptr; /* This is a pointer for a doorbells
* page used by kernel queue
*/
struct kgd2kfd_shared_resources shared_resources;
struct kfd_vmid_info vm_info;
const struct kfd2kgd_calls *kfd2kgd;
struct mutex doorbell_mutex;
DECLARE_BITMAP(doorbell_available_index,
KFD_MAX_NUM_OF_QUEUES_PER_PROCESS);
void *gtt_mem;
uint64_t gtt_start_gpu_addr;
void *gtt_start_cpu_ptr;
void *gtt_sa_bitmap;
struct mutex gtt_sa_lock;
unsigned int gtt_sa_chunk_size;
unsigned int gtt_sa_num_of_chunks;
/* Interrupts */
struct kfifo ih_fifo;
struct workqueue_struct *ih_wq;
struct work_struct interrupt_work;
spinlock_t interrupt_lock;
/* QCM Device instance */
struct device_queue_manager *dqm;
bool init_complete;
/*
* Interrupts of interest to KFD are copied
* from the HW ring into a SW ring.
*/
bool interrupts_active;
/* Debug manager */
struct kfd_dbgmgr *dbgmgr;
/* Firmware versions */
uint16_t mec_fw_version;
uint16_t mec2_fw_version;
uint16_t sdma_fw_version;
/* Maximum process number mapped to HW scheduler */
unsigned int max_proc_per_quantum;
/* CWSR */
bool cwsr_enabled;
const void *cwsr_isa;
unsigned int cwsr_isa_size;
/* xGMI */
uint64_t hive_id;
/* UUID */
uint64_t unique_id;
bool pci_atomic_requested;
/* Use IOMMU v2 flag */
bool use_iommu_v2;
/* SRAM ECC flag */
atomic_t sram_ecc_flag;
/* Compute Profile ref. count */
atomic_t compute_profile;
/* Global GWS resource shared between processes */
void *gws;
/* Clients watching SMI events */
struct list_head smi_clients;
spinlock_t smi_lock;
uint32_t reset_seq_num;
};
enum kfd_mempool {
KFD_MEMPOOL_SYSTEM_CACHEABLE = 1,
KFD_MEMPOOL_SYSTEM_WRITECOMBINE = 2,
KFD_MEMPOOL_FRAMEBUFFER = 3,
};
/* Character device interface */
int kfd_chardev_init(void);
void kfd_chardev_exit(void);
struct device *kfd_chardev(void);
/**
* enum kfd_unmap_queues_filter - Enum for queue filters.
*
* @KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE: Preempts single queue.
*
* @KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES: Preempts all queues in the
* running queues list.
*
* @KFD_UNMAP_QUEUES_FILTER_BY_PASID: Preempts queues that belongs to
* specific process.
*
*/
enum kfd_unmap_queues_filter {
KFD_UNMAP_QUEUES_FILTER_SINGLE_QUEUE,
KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES,
KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES,
KFD_UNMAP_QUEUES_FILTER_BY_PASID
};
/**
* enum kfd_queue_type - Enum for various queue types.
*
* @KFD_QUEUE_TYPE_COMPUTE: Regular user mode queue type.
*
* @KFD_QUEUE_TYPE_SDMA: SDMA user mode queue type.
*
* @KFD_QUEUE_TYPE_HIQ: HIQ queue type.
*
* @KFD_QUEUE_TYPE_DIQ: DIQ queue type.
*
* @KFD_QUEUE_TYPE_SDMA_XGMI: Special SDMA queue for XGMI interface.
*/
enum kfd_queue_type {
KFD_QUEUE_TYPE_COMPUTE,
KFD_QUEUE_TYPE_SDMA,
KFD_QUEUE_TYPE_HIQ,
KFD_QUEUE_TYPE_DIQ,
KFD_QUEUE_TYPE_SDMA_XGMI
};
enum kfd_queue_format {
KFD_QUEUE_FORMAT_PM4,
KFD_QUEUE_FORMAT_AQL
};
enum KFD_QUEUE_PRIORITY {
KFD_QUEUE_PRIORITY_MINIMUM = 0,
KFD_QUEUE_PRIORITY_MAXIMUM = 15
};
/**
* struct queue_properties
*
* @type: The queue type.
*
* @queue_id: Queue identifier.
*
* @queue_address: Queue ring buffer address.
*
* @queue_size: Queue ring buffer size.
*
* @priority: Defines the queue priority relative to other queues in the
* process.
* This is just an indication and HW scheduling may override the priority as
* necessary while keeping the relative prioritization.
* the priority granularity is from 0 to f which f is the highest priority.
* currently all queues are initialized with the highest priority.
*
* @queue_percent: This field is partially implemented and currently a zero in
* this field defines that the queue is non active.
*
* @read_ptr: User space address which points to the number of dwords the
* cp read from the ring buffer. This field updates automatically by the H/W.
*
* @write_ptr: Defines the number of dwords written to the ring buffer.
*
* @doorbell_ptr: Notifies the H/W of new packet written to the queue ring
* buffer. This field should be similar to write_ptr and the user should
* update this field after updating the write_ptr.
*
* @doorbell_off: The doorbell offset in the doorbell pci-bar.
*
* @is_interop: Defines if this is a interop queue. Interop queue means that
* the queue can access both graphics and compute resources.
*
* @is_evicted: Defines if the queue is evicted. Only active queues
* are evicted, rendering them inactive.
*
* @is_active: Defines if the queue is active or not. @is_active and
* @is_evicted are protected by the DQM lock.
*
* @is_gws: Defines if the queue has been updated to be GWS-capable or not.
* @is_gws should be protected by the DQM lock, since changing it can yield the
* possibility of updating DQM state on number of GWS queues.
*
* @vmid: If the scheduling mode is no cp scheduling the field defines the vmid
* of the queue.
*
* This structure represents the queue properties for each queue no matter if
* it's user mode or kernel mode queue.
*
*/
struct queue_properties {
enum kfd_queue_type type;
enum kfd_queue_format format;
unsigned int queue_id;
uint64_t queue_address;
uint64_t queue_size;
uint32_t priority;
uint32_t queue_percent;
uint32_t *read_ptr;
uint32_t *write_ptr;
void __iomem *doorbell_ptr;
uint32_t doorbell_off;
bool is_interop;
bool is_evicted;
bool is_active;
bool is_gws;
/* Not relevant for user mode queues in cp scheduling */
unsigned int vmid;
/* Relevant only for sdma queues*/
uint32_t sdma_engine_id;
uint32_t sdma_queue_id;
uint32_t sdma_vm_addr;
/* Relevant only for VI */
uint64_t eop_ring_buffer_address;
uint32_t eop_ring_buffer_size;
uint64_t ctx_save_restore_area_address;
uint32_t ctx_save_restore_area_size;
uint32_t ctl_stack_size;
uint64_t tba_addr;
uint64_t tma_addr;
/* Relevant for CU */
uint32_t cu_mask_count; /* Must be a multiple of 32 */
uint32_t *cu_mask;
};
#define QUEUE_IS_ACTIVE(q) ((q).queue_size > 0 && \
(q).queue_address != 0 && \
(q).queue_percent > 0 && \
!(q).is_evicted)
/**
* struct queue
*
* @list: Queue linked list.
*
* @mqd: The queue MQD (memory queue descriptor).
*
* @mqd_mem_obj: The MQD local gpu memory object.
*
* @gart_mqd_addr: The MQD gart mc address.
*
* @properties: The queue properties.
*
* @mec: Used only in no cp scheduling mode and identifies to micro engine id
* that the queue should be executed on.
*
* @pipe: Used only in no cp scheduling mode and identifies the queue's pipe
* id.
*
* @queue: Used only in no cp scheduliong mode and identifies the queue's slot.
*
* @process: The kfd process that created this queue.
*
* @device: The kfd device that created this queue.
*
* @gws: Pointing to gws kgd_mem if this is a gws control queue; NULL
* otherwise.
*
* This structure represents user mode compute queues.
* It contains all the necessary data to handle such queues.
*
*/
struct queue {
struct list_head list;
void *mqd;
struct kfd_mem_obj *mqd_mem_obj;
uint64_t gart_mqd_addr;
struct queue_properties properties;
uint32_t mec;
uint32_t pipe;
uint32_t queue;
unsigned int sdma_id;
unsigned int doorbell_id;
struct kfd_process *process;
struct kfd_dev *device;
void *gws;
/* procfs */
struct kobject kobj;
};
enum KFD_MQD_TYPE {
KFD_MQD_TYPE_HIQ = 0, /* for hiq */
KFD_MQD_TYPE_CP, /* for cp queues and diq */
KFD_MQD_TYPE_SDMA, /* for sdma queues */
KFD_MQD_TYPE_DIQ, /* for diq */
KFD_MQD_TYPE_MAX
};
enum KFD_PIPE_PRIORITY {
KFD_PIPE_PRIORITY_CS_LOW = 0,
KFD_PIPE_PRIORITY_CS_MEDIUM,
KFD_PIPE_PRIORITY_CS_HIGH
};
struct scheduling_resources {
unsigned int vmid_mask;
enum kfd_queue_type type;
uint64_t queue_mask;
uint64_t gws_mask;
uint32_t oac_mask;
uint32_t gds_heap_base;
uint32_t gds_heap_size;
};
struct process_queue_manager {
/* data */
struct kfd_process *process;
struct list_head queues;
unsigned long *queue_slot_bitmap;
};
struct qcm_process_device {
/* The Device Queue Manager that owns this data */
struct device_queue_manager *dqm;
struct process_queue_manager *pqm;
/* Queues list */
struct list_head queues_list;
struct list_head priv_queue_list;
unsigned int queue_count;
unsigned int vmid;
bool is_debug;
unsigned int evicted; /* eviction counter, 0=active */
/* This flag tells if we should reset all wavefronts on
* process termination
*/
bool reset_wavefronts;
/* This flag tells us if this process has a GWS-capable
* queue that will be mapped into the runlist. It's
* possible to request a GWS BO, but not have the queue
* currently mapped, and this changes how the MAP_PROCESS
* PM4 packet is configured.
*/
bool mapped_gws_queue;
/* All the memory management data should be here too */
uint64_t gds_context_area;
/* Contains page table flags such as AMDGPU_PTE_VALID since gfx9 */
uint64_t page_table_base;
uint32_t sh_mem_config;
uint32_t sh_mem_bases;
uint32_t sh_mem_ape1_base;
uint32_t sh_mem_ape1_limit;
uint32_t gds_size;
uint32_t num_gws;
uint32_t num_oac;
uint32_t sh_hidden_private_base;
/* CWSR memory */
void *cwsr_kaddr;
uint64_t cwsr_base;
uint64_t tba_addr;
uint64_t tma_addr;
/* IB memory */
uint64_t ib_base;
void *ib_kaddr;
/* doorbell resources per process per device */
unsigned long *doorbell_bitmap;
};
/* KFD Memory Eviction */
/* Approx. wait time before attempting to restore evicted BOs */
#define PROCESS_RESTORE_TIME_MS 100
/* Approx. back off time if restore fails due to lack of memory */
#define PROCESS_BACK_OFF_TIME_MS 100
/* Approx. time before evicting the process again */
#define PROCESS_ACTIVE_TIME_MS 10
/* 8 byte handle containing GPU ID in the most significant 4 bytes and
* idr_handle in the least significant 4 bytes
*/
#define MAKE_HANDLE(gpu_id, idr_handle) \
(((uint64_t)(gpu_id) << 32) + idr_handle)
#define GET_GPU_ID(handle) (handle >> 32)
#define GET_IDR_HANDLE(handle) (handle & 0xFFFFFFFF)
enum kfd_pdd_bound {
PDD_UNBOUND = 0,
PDD_BOUND,
PDD_BOUND_SUSPENDED,
};
#define MAX_SYSFS_FILENAME_LEN 15
/*
* SDMA counter runs at 100MHz frequency.
* We display SDMA activity in microsecond granularity in sysfs.
* As a result, the divisor is 100.
*/
#define SDMA_ACTIVITY_DIVISOR 100
/* Data that is per-process-per device. */
struct kfd_process_device {
/*
* List of all per-device data for a process.
* Starts from kfd_process.per_device_data.
*/
struct list_head per_device_list;
/* The device that owns this data. */
struct kfd_dev *dev;
/* The process that owns this kfd_process_device. */
struct kfd_process *process;
/* per-process-per device QCM data structure */
struct qcm_process_device qpd;
/*Apertures*/
uint64_t lds_base;
uint64_t lds_limit;
uint64_t gpuvm_base;
uint64_t gpuvm_limit;
uint64_t scratch_base;
uint64_t scratch_limit;
/* VM context for GPUVM allocations */
struct file *drm_file;
void *vm;
/* GPUVM allocations storage */
struct idr alloc_idr;
/* Flag used to tell the pdd has dequeued from the dqm.
* This is used to prevent dev->dqm->ops.process_termination() from
* being called twice when it is already called in IOMMU callback
* function.
*/
bool already_dequeued;
bool runtime_inuse;
/* Is this process/pasid bound to this device? (amd_iommu_bind_pasid) */
enum kfd_pdd_bound bound;
/* VRAM usage */
uint64_t vram_usage;
struct attribute attr_vram;
char vram_filename[MAX_SYSFS_FILENAME_LEN];
/* SDMA activity tracking */
uint64_t sdma_past_activity_counter;
struct attribute attr_sdma;
char sdma_filename[MAX_SYSFS_FILENAME_LEN];
/* Eviction activity tracking */
uint64_t last_evict_timestamp;
atomic64_t evict_duration_counter;
struct attribute attr_evict;
struct kobject *kobj_stats;
};
#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
/* Process data */
struct kfd_process {
/*
* kfd_process are stored in an mm_struct*->kfd_process*
* hash table (kfd_processes in kfd_process.c)
*/
struct hlist_node kfd_processes;
/*
* Opaque pointer to mm_struct. We don't hold a reference to
* it so it should never be dereferenced from here. This is
* only used for looking up processes by their mm.
*/
void *mm;
struct kref ref;
struct work_struct release_work;
struct mutex mutex;
/*
* In any process, the thread that started main() is the lead
* thread and outlives the rest.
* It is here because amd_iommu_bind_pasid wants a task_struct.
* It can also be used for safely getting a reference to the
* mm_struct of the process.
*/
struct task_struct *lead_thread;
/* We want to receive a notification when the mm_struct is destroyed */
struct mmu_notifier mmu_notifier;
uint16_t pasid;
unsigned int doorbell_index;
/*
* List of kfd_process_device structures,
* one for each device the process is using.
*/
struct list_head per_device_data;
struct process_queue_manager pqm;
/*Is the user space process 32 bit?*/
bool is_32bit_user_mode;
/* Event-related data */
struct mutex event_mutex;
/* Event ID allocator and lookup */
struct idr event_idr;
/* Event page */
struct kfd_signal_page *signal_page;
size_t signal_mapped_size;
size_t signal_event_count;
bool signal_event_limit_reached;
/* Information used for memory eviction */
void *kgd_process_info;
/* Eviction fence that is attached to all the BOs of this process. The
* fence will be triggered during eviction and new one will be created
* during restore
*/
struct dma_fence *ef;
/* Work items for evicting and restoring BOs */
struct delayed_work eviction_work;
struct delayed_work restore_work;
/* seqno of the last scheduled eviction */
unsigned int last_eviction_seqno;
/* Approx. the last timestamp (in jiffies) when the process was
* restored after an eviction
*/
unsigned long last_restore_timestamp;
/* Kobj for our procfs */
struct kobject *kobj;
struct kobject *kobj_queues;
struct attribute attr_pasid;
};
#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
extern DECLARE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
extern struct srcu_struct kfd_processes_srcu;
/**
* typedef amdkfd_ioctl_t - typedef for ioctl function pointer.
*
* @filep: pointer to file structure.
* @p: amdkfd process pointer.
* @data: pointer to arg that was copied from user.
*
* Return: returns ioctl completion code.
*/
typedef int amdkfd_ioctl_t(struct file *filep, struct kfd_process *p,
void *data);
struct amdkfd_ioctl_desc {
unsigned int cmd;
int flags;
amdkfd_ioctl_t *func;
unsigned int cmd_drv;
const char *name;
};
bool kfd_dev_is_large_bar(struct kfd_dev *dev);
int kfd_process_create_wq(void);
void kfd_process_destroy_wq(void);
struct kfd_process *kfd_create_process(struct file *filep);
struct kfd_process *kfd_get_process(const struct task_struct *);
struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid);
struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
void kfd_unref_process(struct kfd_process *p);
int kfd_process_evict_queues(struct kfd_process *p);
int kfd_process_restore_queues(struct kfd_process *p);
void kfd_suspend_all_processes(void);
int kfd_resume_all_processes(void);
int kfd_process_device_init_vm(struct kfd_process_device *pdd,
struct file *drm_file);
struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
struct kfd_process *p);
struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
struct kfd_process *p);
struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
struct kfd_process *p);
int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
struct vm_area_struct *vma);
/* KFD process API for creating and translating handles */
int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
void *mem);
void *kfd_process_device_translate_handle(struct kfd_process_device *p,
int handle);
void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
int handle);
/* Process device data iterator */
struct kfd_process_device *kfd_get_first_process_device_data(
struct kfd_process *p);
struct kfd_process_device *kfd_get_next_process_device_data(
struct kfd_process *p,
struct kfd_process_device *pdd);
bool kfd_has_process_device_data(struct kfd_process *p);
/* PASIDs */
int kfd_pasid_init(void);
void kfd_pasid_exit(void);
bool kfd_set_pasid_limit(unsigned int new_limit);
unsigned int kfd_get_pasid_limit(void);
unsigned int kfd_pasid_alloc(void);
void kfd_pasid_free(unsigned int pasid);
/* Doorbells */
size_t kfd_doorbell_process_slice(struct kfd_dev *kfd);
int kfd_doorbell_init(struct kfd_dev *kfd);
void kfd_doorbell_fini(struct kfd_dev *kfd);
int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
struct vm_area_struct *vma);
void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd,
unsigned int *doorbell_off);
void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr);
u32 read_kernel_doorbell(u32 __iomem *db);
void write_kernel_doorbell(void __iomem *db, u32 value);
void write_kernel_doorbell64(void __iomem *db, u64 value);
unsigned int kfd_get_doorbell_dw_offset_in_bar(struct kfd_dev *kfd,
struct kfd_process *process,
unsigned int doorbell_id);
phys_addr_t kfd_get_process_doorbells(struct kfd_dev *dev,
struct kfd_process *process);
int kfd_alloc_process_doorbells(struct kfd_process *process);
void kfd_free_process_doorbells(struct kfd_process *process);
/* GTT Sub-Allocator */
int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size,
struct kfd_mem_obj **mem_obj);
int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj);
extern struct device *kfd_device;
/* KFD's procfs */
void kfd_procfs_init(void);
void kfd_procfs_shutdown(void);
int kfd_procfs_add_queue(struct queue *q);
void kfd_procfs_del_queue(struct queue *q);
/* Topology */
int kfd_topology_init(void);
void kfd_topology_shutdown(void);
int kfd_topology_add_device(struct kfd_dev *gpu);
int kfd_topology_remove_device(struct kfd_dev *gpu);
struct kfd_topology_device *kfd_topology_device_by_proximity_domain(
uint32_t proximity_domain);
struct kfd_topology_device *kfd_topology_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_id(uint32_t gpu_id);
struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev);
struct kfd_dev *kfd_device_by_kgd(const struct kgd_dev *kgd);
int kfd_topology_enum_kfd_devices(uint8_t idx, struct kfd_dev **kdev);
int kfd_numa_node_to_apic_id(int numa_node_id);
void kfd_double_confirm_iommu_support(struct kfd_dev *gpu);
/* Interrupts */
int kfd_interrupt_init(struct kfd_dev *dev);
void kfd_interrupt_exit(struct kfd_dev *dev);
bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
bool interrupt_is_wanted(struct kfd_dev *dev,
const uint32_t *ih_ring_entry,
uint32_t *patched_ihre, bool *flag);
/* amdkfd Apertures */
int kfd_init_apertures(struct kfd_process *process);
/* Queue Context Management */
int init_queue(struct queue **q, const struct queue_properties *properties);
void uninit_queue(struct queue *q);
void print_queue_properties(struct queue_properties *q);
void print_queue(struct queue *q);
struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_cik_hawaii(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_vi_tonga(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct mqd_manager *mqd_manager_init_v10(enum KFD_MQD_TYPE type,
struct kfd_dev *dev);
struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev);
void device_queue_manager_uninit(struct device_queue_manager *dqm);
struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
enum kfd_queue_type type);
void kernel_queue_uninit(struct kernel_queue *kq, bool hanging);
int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
/* Process Queue Manager */
struct process_queue_node {
struct queue *q;
struct kernel_queue *kq;
struct list_head process_queue_list;
};
void kfd_process_dequeue_from_device(struct kfd_process_device *pdd);
void kfd_process_dequeue_from_all_devices(struct kfd_process *p);
int pqm_init(struct process_queue_manager *pqm, struct kfd_process *p);
void pqm_uninit(struct process_queue_manager *pqm);
int pqm_create_queue(struct process_queue_manager *pqm,
struct kfd_dev *dev,
struct file *f,
struct queue_properties *properties,
unsigned int *qid,
uint32_t *p_doorbell_offset_in_process);
int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
struct queue_properties *p);
int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
struct queue_properties *p);
int pqm_set_gws(struct process_queue_manager *pqm, unsigned int qid,
void *gws);
struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
unsigned int qid);
struct queue *pqm_get_user_queue(struct process_queue_manager *pqm,
unsigned int qid);
int pqm_get_wave_state(struct process_queue_manager *pqm,
unsigned int qid,
void __user *ctl_stack,
u32 *ctl_stack_used_size,
u32 *save_area_used_size);
int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
unsigned int fence_value,
unsigned int timeout_ms);
/* Packet Manager */
#define KFD_FENCE_COMPLETED (100)
#define KFD_FENCE_INIT (10)
struct packet_manager {
struct device_queue_manager *dqm;
struct kernel_queue *priv_queue;
struct mutex lock;
bool allocated;
struct kfd_mem_obj *ib_buffer_obj;
unsigned int ib_size_bytes;
bool is_over_subscription;
const struct packet_manager_funcs *pmf;
};
struct packet_manager_funcs {
/* Support ASIC-specific packet formats for PM4 packets */
int (*map_process)(struct packet_manager *pm, uint32_t *buffer,
struct qcm_process_device *qpd);
int (*runlist)(struct packet_manager *pm, uint32_t *buffer,
uint64_t ib, size_t ib_size_in_dwords, bool chain);
int (*set_resources)(struct packet_manager *pm, uint32_t *buffer,
struct scheduling_resources *res);
int (*map_queues)(struct packet_manager *pm, uint32_t *buffer,
struct queue *q, bool is_static);
int (*unmap_queues)(struct packet_manager *pm, uint32_t *buffer,
enum kfd_queue_type type,
enum kfd_unmap_queues_filter mode,
uint32_t filter_param, bool reset,
unsigned int sdma_engine);
int (*query_status)(struct packet_manager *pm, uint32_t *buffer,
uint64_t fence_address, uint32_t fence_value);
int (*release_mem)(uint64_t gpu_addr, uint32_t *buffer);
/* Packet sizes */
int map_process_size;
int runlist_size;
int set_resources_size;
int map_queues_size;
int unmap_queues_size;
int query_status_size;
int release_mem_size;
};
extern const struct packet_manager_funcs kfd_vi_pm_funcs;
extern const struct packet_manager_funcs kfd_v9_pm_funcs;
int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
void pm_uninit(struct packet_manager *pm, bool hanging);
int pm_send_set_resources(struct packet_manager *pm,
struct scheduling_resources *res);
int pm_send_runlist(struct packet_manager *pm, struct list_head *dqm_queues);
int pm_send_query_status(struct packet_manager *pm, uint64_t fence_address,
uint32_t fence_value);
int pm_send_unmap_queue(struct packet_manager *pm, enum kfd_queue_type type,
enum kfd_unmap_queues_filter mode,
uint32_t filter_param, bool reset,
unsigned int sdma_engine);
void pm_release_ib(struct packet_manager *pm);
/* Following PM funcs can be shared among VI and AI */
unsigned int pm_build_pm4_header(unsigned int opcode, size_t packet_size);
uint64_t kfd_get_number_elems(struct kfd_dev *kfd);
/* Events */
extern const struct kfd_event_interrupt_class event_interrupt_class_cik;
extern const struct kfd_event_interrupt_class event_interrupt_class_v9;
extern const struct kfd_device_global_init_class device_global_init_class_cik;
void kfd_event_init_process(struct kfd_process *p);
void kfd_event_free_process(struct kfd_process *p);
int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma);
int kfd_wait_on_events(struct kfd_process *p,
uint32_t num_events, void __user *data,
bool all, uint32_t user_timeout_ms,
uint32_t *wait_result);
void kfd_signal_event_interrupt(unsigned int pasid, uint32_t partial_id,
uint32_t valid_id_bits);
void kfd_signal_iommu_event(struct kfd_dev *dev,
unsigned int pasid, unsigned long address,
bool is_write_requested, bool is_execute_requested);
void kfd_signal_hw_exception_event(unsigned int pasid);
int kfd_set_event(struct kfd_process *p, uint32_t event_id);
int kfd_reset_event(struct kfd_process *p, uint32_t event_id);
int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
uint64_t size);
int kfd_event_create(struct file *devkfd, struct kfd_process *p,
uint32_t event_type, bool auto_reset, uint32_t node_id,
uint32_t *event_id, uint32_t *event_trigger_data,
uint64_t *event_page_offset, uint32_t *event_slot_index);
int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
struct kfd_vm_fault_info *info);
void kfd_signal_reset_event(struct kfd_dev *dev);
void kfd_flush_tlb(struct kfd_process_device *pdd);
int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
bool kfd_is_locked(void);
/* Compute profile */
void kfd_inc_compute_active(struct kfd_dev *dev);
void kfd_dec_compute_active(struct kfd_dev *dev);
/* Cgroup Support */
/* Check with device cgroup if @kfd device is accessible */
static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd)
{
#if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF)
struct drm_device *ddev = kfd->ddev;
return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR,
ddev->render->index,
DEVCG_ACC_WRITE | DEVCG_ACC_READ);
#else
return 0;
#endif
}
/* Debugfs */
#if defined(CONFIG_DEBUG_FS)
void kfd_debugfs_init(void);
void kfd_debugfs_fini(void);
int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data);
int pqm_debugfs_mqds(struct seq_file *m, void *data);
int kfd_debugfs_hqds_by_device(struct seq_file *m, void *data);
int dqm_debugfs_hqds(struct seq_file *m, void *data);
int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
int pm_debugfs_runlist(struct seq_file *m, void *data);
int kfd_debugfs_hang_hws(struct kfd_dev *dev);
int pm_debugfs_hang_hws(struct packet_manager *pm);
int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);
#else
static inline void kfd_debugfs_init(void) {}
static inline void kfd_debugfs_fini(void) {}
#endif
#endif
|