Skip to content

Commit a0773b8

Browse files
vstinnermdboom
vstinner
and
mdboom
authored
gh-108753: Enhance pystats (#108754)
Statistics gathering is now off by default. Use the "-X pystats" command line option or set the new PYTHONSTATS environment variable to 1 to turn statistics gathering on at Python startup. Statistics are no longer dumped at exit if statistics gathering was off or statistics have been cleared. Changes: * Add PYTHONSTATS environment variable. * sys._stats_dump() now returns False if statistics are not dumped because they are all equal to zero. * Add PyConfig._pystats member. * Add tests on sys functions and on setting PyConfig._pystats to 1. * Add Include/cpython/pystats.h and Include/internal/pycore_pystats.h header files. * Rename '_py_stats' variable to '_Py_stats'. * Exclude Include/cpython/pystats.h from the Py_LIMITED_API. * Move pystats.h include from object.h to Python.h. * Add _Py_StatsOn() and _Py_StatsOff() functions. Remove '_py_stats_struct' variable from the API: make it static in specialize.c. * Document API in Include/pystats.h and Include/cpython/pystats.h. * Complete pystats documentation in Doc/using/configure.rst. * Don't write "all zeros" stats: if _stats_off() and _stats_clear() or _stats_dump() were called. * _PyEval_Fini() now always call _Py_PrintSpecializationStats() which does nothing if stats are all zeros. Co-authored-by: Michael Droettboom <[email protected]>
1 parent 8ff1142 commit a0773b8

File tree

19 files changed

+402
-183
lines changed

19 files changed

+402
-183
lines changed

‎Doc/using/configure.rst‎

Lines changed: 58 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,14 +192,69 @@ General Options
192192

193193
.. cmdoption:: --enable-pystats
194194

195-
Turn on internal statistics gathering.
195+
Turn on internal Python performance statistics gathering.
196+
197+
By default, statistics gathering is off. Use ``python3 -X pystats`` command
198+
or set ``PYTHONSTATS=1`` environment variable to turn on statistics
199+
gathering at Python startup.
200+
201+
At Python exit, dump statistics if statistics gathering was on and not
202+
cleared.
203+
204+
Effects:
205+
206+
* Add :option:`-X pystats <-X>` command line option.
207+
* Add :envvar:`!PYTHONSTATS` environment variable.
208+
* Define the ``Py_STATS`` macro.
209+
* Add functions to the :mod:`sys` module:
210+
211+
* :func:`!sys._stats_on`: Turns on statistics gathering.
212+
* :func:`!sys._stats_off`: Turns off statistics gathering.
213+
* :func:`!sys._stats_clear`: Clears the statistics.
214+
* :func:`!sys._stats_dump`: Dump statistics to file, and clears the statistics.
196215

197216
The statistics will be dumped to a arbitrary (probably unique) file in
198-
``/tmp/py_stats/``, or ``C:\temp\py_stats\`` on Windows. If that directory
199-
does not exist, results will be printed on stdout.
217+
``/tmp/py_stats/`` (Unix) or ``C:\temp\py_stats\`` (Windows). If that
218+
directory does not exist, results will be printed on stderr.
200219

201220
Use ``Tools/scripts/summarize_stats.py`` to read the stats.
202221

222+
Statistics:
223+
224+
* Opcode:
225+
226+
* Specialization: success, failure, hit, deferred, miss, deopt, failures;
227+
* Execution count;
228+
* Pair count.
229+
230+
* Call:
231+
232+
* Inlined Python calls;
233+
* PyEval calls;
234+
* Frames pushed;
235+
* Frame object created;
236+
* Eval calls: vector, generator, legacy, function VECTORCALL, build class,
237+
slot, function "ex", API, method.
238+
239+
* Object:
240+
241+
* incref and decref;
242+
* interpreter incref and decref;
243+
* allocations: all, 512 bytes, 4 kiB, big;
244+
* free;
245+
* to/from free lists;
246+
* dictionary materialized/dematerialized;
247+
* type cache;
248+
* optimization attemps;
249+
* optimization traces created/executed;
250+
* uops executed.
251+
252+
* Garbage collector:
253+
254+
* Garbage collections;
255+
* Objects visited;
256+
* Objects collected.
257+
203258
.. versionadded:: 3.11
204259

205260
.. cmdoption:: --disable-gil

‎Include/cpython/initconfig.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,11 @@ typedef struct PyConfig {
215215

216216
// If non-zero, we believe we're running from a source tree.
217217
int _is_python_build;
218+
219+
#ifdef Py_STATS
220+
// If non-zero, turns on statistics gathering.
221+
int _pystats;
222+
#endif
218223
} PyConfig;
219224

220225
PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

‎Include/cpython/pystats.h‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Statistics on Python performance.
2+
//
3+
// API:
4+
//
5+
// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
6+
// and Py_DECREF().
7+
// - _Py_stats variable
8+
//
9+
// Functions of the sys module:
10+
//
11+
// - sys._stats_on()
12+
// - sys._stats_off()
13+
// - sys._stats_clear()
14+
// - sys._stats_dump()
15+
//
16+
// Python must be built with ./configure --enable-pystats to define the
17+
// Py_STATS macro.
18+
//
19+
// Define _PY_INTERPRETER macro to increment interpreter_increfs and
20+
// interpreter_decrefs. Otherwise, increment increfs and decrefs.
21+
22+
#ifndef Py_CPYTHON_PYSTATS_H
23+
# error "this header file must not be included directly"
24+
#endif
25+
26+
#define SPECIALIZATION_FAILURE_KINDS 36
27+
28+
/* Stats for determining who is calling PyEval_EvalFrame */
29+
#define EVAL_CALL_TOTAL 0
30+
#define EVAL_CALL_VECTOR 1
31+
#define EVAL_CALL_GENERATOR 2
32+
#define EVAL_CALL_LEGACY 3
33+
#define EVAL_CALL_FUNCTION_VECTORCALL 4
34+
#define EVAL_CALL_BUILD_CLASS 5
35+
#define EVAL_CALL_SLOT 6
36+
#define EVAL_CALL_FUNCTION_EX 7
37+
#define EVAL_CALL_API 8
38+
#define EVAL_CALL_METHOD 9
39+
40+
#define EVAL_CALL_KINDS 10
41+
42+
typedef struct _specialization_stats {
43+
uint64_t success;
44+
uint64_t failure;
45+
uint64_t hit;
46+
uint64_t deferred;
47+
uint64_t miss;
48+
uint64_t deopt;
49+
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
50+
} SpecializationStats;
51+
52+
typedef struct _opcode_stats {
53+
SpecializationStats specialization;
54+
uint64_t execution_count;
55+
uint64_t pair_count[256];
56+
} OpcodeStats;
57+
58+
typedef struct _call_stats {
59+
uint64_t inlined_py_calls;
60+
uint64_t pyeval_calls;
61+
uint64_t frames_pushed;
62+
uint64_t frame_objects_created;
63+
uint64_t eval_calls[EVAL_CALL_KINDS];
64+
} CallStats;
65+
66+
typedef struct _object_stats {
67+
uint64_t increfs;
68+
uint64_t decrefs;
69+
uint64_t interpreter_increfs;
70+
uint64_t interpreter_decrefs;
71+
uint64_t allocations;
72+
uint64_t allocations512;
73+
uint64_t allocations4k;
74+
uint64_t allocations_big;
75+
uint64_t frees;
76+
uint64_t to_freelist;
77+
uint64_t from_freelist;
78+
uint64_t new_values;
79+
uint64_t dict_materialized_on_request;
80+
uint64_t dict_materialized_new_key;
81+
uint64_t dict_materialized_too_big;
82+
uint64_t dict_materialized_str_subclass;
83+
uint64_t dict_dematerialized;
84+
uint64_t type_cache_hits;
85+
uint64_t type_cache_misses;
86+
uint64_t type_cache_dunder_hits;
87+
uint64_t type_cache_dunder_misses;
88+
uint64_t type_cache_collisions;
89+
uint64_t optimization_attempts;
90+
uint64_t optimization_traces_created;
91+
uint64_t optimization_traces_executed;
92+
uint64_t optimization_uops_executed;
93+
/* Temporary value used during GC */
94+
uint64_t object_visits;
95+
} ObjectStats;
96+
97+
typedef struct _gc_stats {
98+
uint64_t collections;
99+
uint64_t object_visits;
100+
uint64_t objects_collected;
101+
} GCStats;
102+
103+
typedef struct _stats {
104+
OpcodeStats opcode_stats[256];
105+
CallStats call_stats;
106+
ObjectStats object_stats;
107+
GCStats *gc_stats;
108+
} PyStats;
109+
110+
111+
// Export for shared extensions like 'math'
112+
PyAPI_DATA(PyStats*) _Py_stats;
113+
114+
#ifdef _PY_INTERPRETER
115+
# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0)
116+
# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0)
117+
#else
118+
# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0)
119+
# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0)
120+
#endif

‎Include/internal/pycore_code.h‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -268,17 +268,17 @@ extern int _PyStaticCode_Init(PyCodeObject *co);
268268

269269
#ifdef Py_STATS
270270

271-
#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
272-
#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
273-
#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
274-
#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
275-
#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
271+
#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0)
272+
#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0)
273+
#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0)
274+
#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0)
275+
#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0)
276276
#define OBJECT_STAT_INC_COND(name, cond) \
277-
do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
278-
#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
277+
do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0)
278+
#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0)
279279
#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
280-
do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)
281-
#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0)
280+
do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0)
281+
#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0)
282282

283283
// Export for '_opcode' shared extension
284284
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);

‎Include/internal/pycore_pystats.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#ifndef Py_INTERNAL_PYSTATS_H
2+
#define Py_INTERNAL_PYSTATS_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
#ifndef Py_BUILD_CORE
8+
# error "this header requires Py_BUILD_CORE define"
9+
#endif
10+
11+
#ifdef Py_STATS
12+
extern void _Py_StatsOn(void);
13+
extern void _Py_StatsOff(void);
14+
extern void _Py_StatsClear(void);
15+
extern int _Py_PrintSpecializationStats(int to_file);
16+
#endif
17+
18+
#ifdef __cplusplus
19+
}
20+
#endif
21+
#endif // !Py_INTERNAL_PYSTATS_H

‎Include/pystats.h‎

Lines changed: 13 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,124 +1,26 @@
1-
1+
// Statistics on Python performance (public API).
2+
//
3+
// Define _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
4+
// and Py_DECREF().
5+
//
6+
// See Include/cpython/pystats.h for the full API.
27

38
#ifndef Py_PYSTATS_H
49
#define Py_PYSTATS_H
510
#ifdef __cplusplus
611
extern "C" {
712
#endif
813

9-
#ifdef Py_STATS
10-
11-
#define SPECIALIZATION_FAILURE_KINDS 36
12-
13-
/* Stats for determining who is calling PyEval_EvalFrame */
14-
#define EVAL_CALL_TOTAL 0
15-
#define EVAL_CALL_VECTOR 1
16-
#define EVAL_CALL_GENERATOR 2
17-
#define EVAL_CALL_LEGACY 3
18-
#define EVAL_CALL_FUNCTION_VECTORCALL 4
19-
#define EVAL_CALL_BUILD_CLASS 5
20-
#define EVAL_CALL_SLOT 6
21-
#define EVAL_CALL_FUNCTION_EX 7
22-
#define EVAL_CALL_API 8
23-
#define EVAL_CALL_METHOD 9
24-
25-
#define EVAL_CALL_KINDS 10
26-
27-
typedef struct _specialization_stats {
28-
uint64_t success;
29-
uint64_t failure;
30-
uint64_t hit;
31-
uint64_t deferred;
32-
uint64_t miss;
33-
uint64_t deopt;
34-
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
35-
} SpecializationStats;
36-
37-
typedef struct _opcode_stats {
38-
SpecializationStats specialization;
39-
uint64_t execution_count;
40-
uint64_t pair_count[256];
41-
} OpcodeStats;
42-
43-
typedef struct _call_stats {
44-
uint64_t inlined_py_calls;
45-
uint64_t pyeval_calls;
46-
uint64_t frames_pushed;
47-
uint64_t frame_objects_created;
48-
uint64_t eval_calls[EVAL_CALL_KINDS];
49-
} CallStats;
50-
51-
typedef struct _object_stats {
52-
uint64_t increfs;
53-
uint64_t decrefs;
54-
uint64_t interpreter_increfs;
55-
uint64_t interpreter_decrefs;
56-
uint64_t allocations;
57-
uint64_t allocations512;
58-
uint64_t allocations4k;
59-
uint64_t allocations_big;
60-
uint64_t frees;
61-
uint64_t to_freelist;
62-
uint64_t from_freelist;
63-
uint64_t new_values;
64-
uint64_t dict_materialized_on_request;
65-
uint64_t dict_materialized_new_key;
66-
uint64_t dict_materialized_too_big;
67-
uint64_t dict_materialized_str_subclass;
68-
uint64_t dict_dematerialized;
69-
uint64_t type_cache_hits;
70-
uint64_t type_cache_misses;
71-
uint64_t type_cache_dunder_hits;
72-
uint64_t type_cache_dunder_misses;
73-
uint64_t type_cache_collisions;
74-
uint64_t optimization_attempts;
75-
uint64_t optimization_traces_created;
76-
uint64_t optimization_traces_executed;
77-
uint64_t optimization_uops_executed;
78-
/* Temporary value used during GC */
79-
uint64_t object_visits;
80-
} ObjectStats;
81-
82-
typedef struct _gc_stats {
83-
uint64_t collections;
84-
uint64_t object_visits;
85-
uint64_t objects_collected;
86-
} GCStats;
87-
88-
typedef struct _stats {
89-
OpcodeStats opcode_stats[256];
90-
CallStats call_stats;
91-
ObjectStats object_stats;
92-
GCStats *gc_stats;
93-
} PyStats;
94-
95-
96-
PyAPI_DATA(PyStats) _py_stats_struct;
97-
PyAPI_DATA(PyStats *) _py_stats;
98-
99-
extern void _Py_StatsClear(void);
100-
extern void _Py_PrintSpecializationStats(int to_file);
101-
102-
#ifdef _PY_INTERPRETER
103-
104-
#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0)
105-
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0)
106-
14+
#if defined(Py_STATS) && !defined(Py_LIMITED_API)
15+
# define Py_CPYTHON_PYSTATS_H
16+
# include "cpython/pystats.h"
17+
# undef Py_CPYTHON_PYSTATS_H
10718
#else
108-
109-
#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0)
110-
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0)
111-
112-
#endif
113-
114-
#else
115-
116-
#define _Py_INCREF_STAT_INC() ((void)0)
117-
#define _Py_DECREF_STAT_INC() ((void)0)
118-
19+
# define _Py_INCREF_STAT_INC() ((void)0)
20+
# define _Py_DECREF_STAT_INC() ((void)0)
11921
#endif // !Py_STATS
12022

12123
#ifdef __cplusplus
12224
}
12325
#endif
124-
#endif /* !Py_PYSTATs_H */
26+
#endif // !Py_PYSTATS_H

0 commit comments

Comments
 (0)