summaryrefslogtreecommitdiffstats
path: root/tools/workqueue/wq_monitor.py
blob: a8856a9c45dcc954df5bf3587abfbb475afb8d9c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
#!/usr/bin/env drgn
#
# Copyright (C) 2023 Tejun Heo <tj@kernel.org>
# Copyright (C) 2023 Meta Platforms, Inc. and affiliates.

desc = """
This is a drgn script to monitor workqueues. For more info on drgn, visit
https://github.com/osandov/drgn.

  total    Total number of work items executed by the workqueue.

  infl     The number of currently in-flight work items.

  CPUtime  Total CPU time consumed by the workqueue in seconds. This is
           sampled from scheduler ticks and only provides ballpark
           measurement. "nohz_full=" CPUs are excluded from measurement.

  CPUitsv  The number of times a concurrency-managed work item hogged CPU
           longer than the threshold (workqueue.cpu_intensive_thresh_us)
           and got excluded from concurrency management to avoid stalling
           other work items.

  CMW/RPR  For per-cpu workqueues, the number of concurrency-management
           wake-ups while executing a work item of the workqueue. For
           unbound workqueues, the number of times a worker was repatriated
           to its affinity scope after being migrated to an off-scope CPU by
           the scheduler.

  mayday   The number of times the rescuer was requested while waiting for
           new worker creation.

  rescued  The number of work items executed by the rescuer.
"""

import sys
import signal
import os
import re
import time
import json

import drgn
from drgn.helpers.linux.list import list_for_each_entry,list_empty
from drgn.helpers.linux.cpumask import for_each_possible_cpu

import argparse
parser = argparse.ArgumentParser(description=desc,
                                 formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('workqueue', metavar='REGEX', nargs='*',
                    help='Target workqueue name patterns (all if empty)')
parser.add_argument('-i', '--interval', metavar='SECS', type=float, default=1,
                    help='Monitoring interval (0 to print once and exit)')
parser.add_argument('-j', '--json', action='store_true',
                    help='Output in json')
args = parser.parse_args()

def err(s):
    print(s, file=sys.stderr, flush=True)
    sys.exit(1)

workqueues              = prog['workqueues']

WQ_UNBOUND              = prog['WQ_UNBOUND']
WQ_MEM_RECLAIM          = prog['WQ_MEM_RECLAIM']

PWQ_STAT_STARTED        = prog['PWQ_STAT_STARTED']      # work items started execution
PWQ_STAT_COMPLETED      = prog['PWQ_STAT_COMPLETED']	# work items completed execution
PWQ_STAT_CPU_TIME       = prog['PWQ_STAT_CPU_TIME']     # total CPU time consumed
PWQ_STAT_CPU_INTENSIVE  = prog['PWQ_STAT_CPU_INTENSIVE'] # wq_cpu_intensive_thresh_us violations
PWQ_STAT_CM_WAKEUP      = prog['PWQ_STAT_CM_WAKEUP']    # concurrency-management worker wakeups
PWQ_STAT_REPATRIATED    = prog['PWQ_STAT_REPATRIATED']  # unbound workers brought back into scope
PWQ_STAT_MAYDAY         = prog['PWQ_STAT_MAYDAY']	# maydays to rescuer
PWQ_STAT_RESCUED        = prog['PWQ_STAT_RESCUED']	# linked work items executed by rescuer
PWQ_NR_STATS            = prog['PWQ_NR_STATS']

class WqStats:
    def __init__(self, wq):
        self.name = wq.name.string_().decode()
        self.unbound = wq.flags & WQ_UNBOUND != 0
        self.mem_reclaim = wq.flags & WQ_MEM_RECLAIM != 0
        self.stats = [0] * PWQ_NR_STATS
        for pwq in list_for_each_entry('struct pool_workqueue', wq.pwqs.address_of_(), 'pwqs_node'):
            for i in range(PWQ_NR_STATS):
                self.stats[i] += int(pwq.stats[i])

    def dict(self, now):
        return { 'timestamp'            : now,
                 'name'                 : self.name,
                 'unbound'              : self.unbound,
                 'mem_reclaim'          : self.mem_reclaim,
                 'started'              : self.stats[PWQ_STAT_STARTED],
                 'completed'            : self.stats[PWQ_STAT_COMPLETED],
                 'cpu_time'             : self.stats[PWQ_STAT_CPU_TIME],
                 'cpu_intensive'        : self.stats[PWQ_STAT_CPU_INTENSIVE],
                 'cm_wakeup'            : self.stats[PWQ_STAT_CM_WAKEUP],
                 'repatriated'          : self.stats[PWQ_STAT_REPATRIATED],
                 'mayday'               : self.stats[PWQ_STAT_MAYDAY],
                 'rescued'              : self.stats[PWQ_STAT_RESCUED], }

    def table_header_str():
        return f'{"":>24} {"total":>8} {"infl":>5} {"CPUtime":>8} '\
            f'{"CPUitsv":>7} {"CMW/RPR":>7} {"mayday":>7} {"rescued":>7}'

    def table_row_str(self):
        cpu_intensive = '-'
        cmw_rpr = '-'
        mayday = '-'
        rescued = '-'

        if self.unbound:
            cmw_rpr = str(self.stats[PWQ_STAT_REPATRIATED]);
        else:
            cpu_intensive = str(self.stats[PWQ_STAT_CPU_INTENSIVE])
            cmw_rpr = str(self.stats[PWQ_STAT_CM_WAKEUP])

        if self.mem_reclaim:
            mayday = str(self.stats[PWQ_STAT_MAYDAY])
            rescued = str(self.stats[PWQ_STAT_RESCUED])

        out = f'{self.name[-24:]:24} ' \
              f'{self.stats[PWQ_STAT_STARTED]:8} ' \
              f'{max(self.stats[PWQ_STAT_STARTED] - self.stats[PWQ_STAT_COMPLETED], 0):5} ' \
              f'{self.stats[PWQ_STAT_CPU_TIME] / 1000000:8.1f} ' \
              f'{cpu_intensive:>7} ' \
              f'{cmw_rpr:>7} ' \
              f'{mayday:>7} ' \
              f'{rescued:>7} '
        return out.rstrip(':')

exit_req = False

def sigint_handler(signr, frame):
    global exit_req
    exit_req = True

def main():
    # handle args
    table_fmt = not args.json
    interval = args.interval

    re_str = None
    if args.workqueue:
        for r in args.workqueue:
            if re_str is None:
                re_str = r
            else:
                re_str += '|' + r

    filter_re = re.compile(re_str) if re_str else None

    # monitoring loop
    signal.signal(signal.SIGINT, sigint_handler)

    while not exit_req:
        now = time.time()

        if table_fmt:
            print()
            print(WqStats.table_header_str())

        for wq in list_for_each_entry('struct workqueue_struct', workqueues.address_of_(), 'list'):
            stats = WqStats(wq)
            if filter_re and not filter_re.search(stats.name):
                continue
            if table_fmt:
                print(stats.table_row_str())
            else:
                print(stats.dict(now))

        if interval == 0:
            break
        time.sleep(interval)

if __name__ == "__main__":
    main()