From 355dee533bb34a571b9367820a63cccb668cf866 Mon Sep 17 00:00:00 2001 From: noptuno Date: Thu, 27 Apr 2023 20:29:30 -0400 Subject: Merging PR_218 openai_rev package with new streamlit chat app --- .../python3.9/site-packages/pympler/__init__.py | 1 + .../lib/python3.9/site-packages/pympler/asizeof.py | 2840 +++++++++++++++ venv/lib/python3.9/site-packages/pympler/charts.py | 62 + .../site-packages/pympler/classtracker.py | 590 +++ .../site-packages/pympler/classtracker_stats.py | 780 ++++ .../site-packages/pympler/garbagegraph.py | 80 + .../python3.9/site-packages/pympler/mprofile.py | 97 + venv/lib/python3.9/site-packages/pympler/muppy.py | 275 ++ venv/lib/python3.9/site-packages/pympler/panels.py | 115 + .../lib/python3.9/site-packages/pympler/process.py | 238 ++ .../python3.9/site-packages/pympler/refbrowser.py | 451 +++ .../python3.9/site-packages/pympler/refgraph.py | 350 ++ .../pympler/static/jquery.sparkline.min.js | 5 + .../lib/python3.9/site-packages/pympler/summary.py | 321 ++ .../pympler/templates/asized_referents.tpl | 9 + .../site-packages/pympler/templates/footer.tpl | 6 + .../site-packages/pympler/templates/garbage.tpl | 33 + .../pympler/templates/garbage_index.tpl | 41 + .../site-packages/pympler/templates/header.tpl | 36 + .../site-packages/pympler/templates/index.tpl | 26 + .../pympler/templates/jquery.flot.min.js | 8 + .../pympler/templates/jquery.flot.stack.min.js | 7 + .../pympler/templates/jquery.flot.tooltip.min.js | 12 + .../pympler/templates/memory_panel.html | 58 + .../site-packages/pympler/templates/process.tpl | 89 + .../site-packages/pympler/templates/referents.tpl | 13 + .../site-packages/pympler/templates/stacktrace.tpl | 30 + .../site-packages/pympler/templates/style.css | 992 +++++ .../site-packages/pympler/templates/tracker.tpl | 127 + .../pympler/templates/tracker_class.tpl | 79 + .../lib/python3.9/site-packages/pympler/tracker.py | 267 ++ .../site-packages/pympler/util/__init__.py | 0 .../python3.9/site-packages/pympler/util/bottle.py | 3771 ++++++++++++++++++++ .../python3.9/site-packages/pympler/util/compat.py | 23 + .../site-packages/pympler/util/stringutils.py | 77 + venv/lib/python3.9/site-packages/pympler/web.py | 346 ++ 36 files changed, 12255 insertions(+) create mode 100644 venv/lib/python3.9/site-packages/pympler/__init__.py create mode 100644 venv/lib/python3.9/site-packages/pympler/asizeof.py create mode 100644 venv/lib/python3.9/site-packages/pympler/charts.py create mode 100644 venv/lib/python3.9/site-packages/pympler/classtracker.py create mode 100644 venv/lib/python3.9/site-packages/pympler/classtracker_stats.py create mode 100644 venv/lib/python3.9/site-packages/pympler/garbagegraph.py create mode 100644 venv/lib/python3.9/site-packages/pympler/mprofile.py create mode 100644 venv/lib/python3.9/site-packages/pympler/muppy.py create mode 100644 venv/lib/python3.9/site-packages/pympler/panels.py create mode 100644 venv/lib/python3.9/site-packages/pympler/process.py create mode 100644 venv/lib/python3.9/site-packages/pympler/refbrowser.py create mode 100644 venv/lib/python3.9/site-packages/pympler/refgraph.py create mode 100644 venv/lib/python3.9/site-packages/pympler/static/jquery.sparkline.min.js create mode 100644 venv/lib/python3.9/site-packages/pympler/summary.py create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/asized_referents.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/footer.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/garbage.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/garbage_index.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/header.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/index.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/jquery.flot.min.js create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/jquery.flot.stack.min.js create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/jquery.flot.tooltip.min.js create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/memory_panel.html create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/process.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/referents.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/stacktrace.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/style.css create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/tracker.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/templates/tracker_class.tpl create mode 100644 venv/lib/python3.9/site-packages/pympler/tracker.py create mode 100644 venv/lib/python3.9/site-packages/pympler/util/__init__.py create mode 100644 venv/lib/python3.9/site-packages/pympler/util/bottle.py create mode 100644 venv/lib/python3.9/site-packages/pympler/util/compat.py create mode 100644 venv/lib/python3.9/site-packages/pympler/util/stringutils.py create mode 100644 venv/lib/python3.9/site-packages/pympler/web.py (limited to 'venv/lib/python3.9/site-packages/pympler') diff --git a/venv/lib/python3.9/site-packages/pympler/__init__.py b/venv/lib/python3.9/site-packages/pympler/__init__.py new file mode 100644 index 00000000..cd7ca498 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/__init__.py @@ -0,0 +1 @@ +__version__ = '1.0.1' diff --git a/venv/lib/python3.9/site-packages/pympler/asizeof.py b/venv/lib/python3.9/site-packages/pympler/asizeof.py new file mode 100644 index 00000000..f9e87c96 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/asizeof.py @@ -0,0 +1,2840 @@ +#!/usr/bin/env python + +# Copyright, license and disclaimer are at the very end of this file. + +# This is the latest, enhanced version of the asizeof.py recipes at +# +# + +# Note, objects like ``namedtuples``, ``closure``, and NumPy data +# ``arange``, ``array``, ``matrix``, etc. are only handled by recent +# versions of this module. Sizing of ``__slots__`` has been incorrect +# in versions before this one. Also, property ``Asizer.duplicate`` gave +# incorrect values before this release. Several other properties +# have been added to the ``Asizer`` class and the ``print_summary`` +# method has been updated. + +''' +This module exposes 9 functions and 2 classes to obtain lengths and +sizes of Python objects (for Python 3.5 or later). + +Earlier versions of this module supported Python versions down to +Python 2.2. If you are using Python 3.5 or older, please consider +downgrading Pympler. + +**Public Functions** [#unsafe]_ + + Function **asizeof** calculates the combined (approximate) size + in bytes of one or several Python objects. + + Function **asizesof** returns a tuple containing the (approximate) + size in bytes for each given Python object separately. + + Function **asized** returns for each object an instance of class + **Asized** containing all the size information of the object and + a tuple with the referents [#refs]_. + + Functions **basicsize** and **itemsize** return the *basic-* + respectively *itemsize* of the given object, both in bytes. For + objects as ``array.array``, ``numpy.array``, ``numpy.matrix``, + etc. where the item size varies depending on the instance-specific + data type, function **itemsize** returns that item size. + + Function **flatsize** returns the *flat size* of a Python object + in bytes defined as the *basic size* plus the *item size* times + the *length* of the given object. + + Function **leng** returns the *length* of an object, like standard + function ``len`` but extended for several types. E.g. the **leng** + of a multi-precision int (or long) is the number of ``digits`` + [#digit]_. The length of most *mutable* sequence objects includes + an estimate of the over-allocation and therefore, the **leng** value + may differ from the standard ``len`` result. For objects like + ``array.array``, ``numpy.array``, ``numpy.matrix``, etc. function + **leng** returns the proper number of items. + + Function **refs** returns (a generator for) the referents [#refs]_ + of the given object. + + Certain classes are known to be sub-classes of or to behave as + ``dict`` objects. Function **adict** can be used to register + other class objects to be treated like ``dict``. + +**Public Classes** [#unsafe]_ + + Class **Asizer** may be used to accumulate the results of several + **asizeof** or **asizesof** calls. After creating an **Asizer** + instance, use methods **asizeof** and **asizesof** as needed to + size any number of additional objects. + + Call methods **exclude_refs** and/or **exclude_types** to exclude + references to respectively instances or types of certain objects. + + Use one of the **print\\_...** methods to report the statistics. + + An instance of class **Asized** is returned for each object sized + by the **asized** function or method. + +**Duplicate Objects** + + Any duplicate, given objects are sized only once and the size + is included in the accumulated total only once. But functions + **asizesof** and **asized** will return a size value respectively + an **Asized** instance for each given object, including duplicates. + +**Definitions** [#arb]_ + + The *length* of an objects like ``dict``, ``list``, ``set``, + ``str``, ``tuple``, etc. is defined as the number of items held + in or allocated by the object. Held items are *references* to + other objects, called the *referents*. + + The *size* of an object is defined as the sum of the *flat size* + of the object plus the sizes of any referents [#refs]_. Referents + are visited recursively up to the specified detail level. However, + the size of objects referenced multiple times is included only once + in the total *size*. + + The *flat size* of an object is defined as the *basic size* of the + object plus the *item size* times the number of allocated *items*, + *references* to referents. The *flat size* does include the size + for the *references* to the referents, but not the size of the + referents themselves. + + The *flat size* returned by function *flatsize* equals the result + of function *asizeof* with options *code=True*, *ignored=False*, + *limit=0* and option *align* set to the same value. + + The accurate *flat size* for an object is obtained from function + ``sys.getsizeof()`` where available. Otherwise, the *length* and + *size* of sequence objects as ``dicts``, ``lists``, ``sets``, etc. + is based on an estimate for the number of allocated items. As a + result, the reported *length* and *size* may differ substantially + from the actual *length* and *size*. + + The *basic* and *item size* are obtained from the ``__basicsize__`` + respectively ``__itemsize__`` attributes of the (type of the) + object. Where necessary (e.g. sequence objects), a zero + ``__itemsize__`` is replaced by the size of a corresponding C type. + + The overhead for Python's garbage collector (GC) is included in + the *basic size* of (GC managed) objects as well as the space + needed for ``refcounts`` (used only in certain Python builds). + + Optionally, size values can be aligned to any power-of-2 multiple. + +**Size of (byte)code** + + The *(byte)code size* of objects like classes, functions, methods, + modules, etc. can be included by setting option *code=True*. + + Iterators are handled like sequences: iterated object(s) are sized + like *referents* [#refs]_, but only up to the specified level or + recursion *limit* (and only if function ``gc.get_referents()`` + returns the referent object of iterators). + + Generators are sized as *(byte)code* only, but the generated + objects are never sized. + +**Old- and New-style Classes** + + All old- and new-style ``class``, instance and ``type`` objects are + handled uniformly such that (a) instance objects are distinguished + from class objects and (b) instances of different old-style classes + can be dealt with separately. + + Class and type objects are represented as ```` + respectively ```` where the ``*`` indicates an old-style + class and the ``... def`` suffix marks the *definition object*. + Instances of classes are shown as ```` without + the ``... def`` suffix. The ``*`` after the name indicates an + instance of an old-style class. + +**Ignored Objects** + + To avoid excessive sizes, several object types are ignored [#arb]_ + by default, e.g. built-in functions, built-in types and classes + [#bi]_, function globals and module referents. However, any + instances thereof and module objects will be sized when passed as + given objects. Ignored object types are included unless option + *ignored* is set accordingly. + + In addition, many ``__...__`` attributes of callable objects are + ignored [#arb]_, except crucial ones, e.g. class attributes ``__dict__``, + ``__doc__``, ``__name__`` and ``__slots__``. For more details, see + the type-specific ``_..._refs()`` and ``_len_...()`` functions below. + +.. rubric:: Footnotes +.. [#unsafe] The functions and classes in this module are not thread-safe. + +.. [#refs] The *referents* of an object are the objects referenced *by* + that object. For example, the *referents* of a ``list`` are the + objects held in the ``list``, the *referents* of a ``dict`` are + the key and value objects in the ``dict``, etc. + +.. [#arb] These definitions and other assumptions are rather arbitrary + and may need corrections or adjustments. + +.. [#digit] See Python source file ``.../Include/longinterp.h`` for the + C ``typedef`` of ``digit`` used in multi-precision int (or long) + objects. The C ``sizeof(digit)`` in bytes can be obtained in + Python from the int (or long) ``__itemsize__`` attribute. + Function **leng** determines the number of ``digits`` of an int + (or long) object. + +.. [#bi] ``Type``s and ``class``es are considered built-in if the + ``__module__`` of the type or class is listed in the private + ``_builtin_modules``. +''' # PYCHOK escape +import sys +if sys.version_info < (3, 6, 0): + raise NotImplementedError('%s requires Python 3.6 or newer' % ('asizeof',)) + +from typing import Callable, Dict, List, Optional, Tuple, Union + +# all imports listed explicitly to help PyChecker +from inspect import (isbuiltin, isclass, iscode, isframe, isfunction, + ismethod, ismodule, stack) +from math import log +from os import curdir, linesep +from struct import calcsize # type/class Struct only in Python 2.5+ +import types as Types +import warnings +import weakref as Weakref + +__all__ = ['adict', 'asized', 'asizeof', 'asizesof', + 'Asized', 'Asizer', # classes + 'basicsize', 'flatsize', 'itemsize', 'leng', 'refs'] +__version__ = '21.08.09' + +# Any classes or types in modules listed in _builtin_modules are +# considered built-in and ignored by default, as built-in functions +_builtin_mods = [int.__module__, 'types', Exception.__module__] # 'weakref' +if __name__ != '__main__': # treat this very module as built-in + _builtin_mods.append(__name__) +_builtin_modules = tuple(_builtin_mods) + +# Sizes of some primitive C types +# XXX len(pack(T, 0)) == Struct(T).size == calcsize(T) +_sizeof_Cbyte = calcsize('c') # sizeof(unsigned char) +_sizeof_Clong = calcsize('l') # sizeof(long) +_sizeof_Cvoidp = calcsize('P') # sizeof(void*) + +# sizeof(long) != sizeof(ssize_t) on LLP64 +if _sizeof_Clong < _sizeof_Cvoidp: # pragma: no coverage + _z_P_L = 'P' +else: + _z_P_L = 'L' + + +def _calcsize(fmt): + '''Like struct.calcsize() but handling 'z' for Py_ssize_t. + ''' + return calcsize(fmt.replace('z', _z_P_L)) + + +# Defaults for some basic sizes with 'z' for C Py_ssize_t +_sizeof_CPyCodeObject = _calcsize('Pz10P5i0P') # sizeof(PyCodeObject) +_sizeof_CPyFrameObject = _calcsize('Pzz13P63i0P') # sizeof(PyFrameObject) +_sizeof_CPyModuleObject = _calcsize('PzP0P') # sizeof(PyModuleObject) + +# Defaults for some item sizes with 'z' for C Py_ssize_t +_sizeof_CPyDictEntry = _calcsize('z2P') # sizeof(PyDictEntry) +_sizeof_Csetentry = _calcsize('lP') # sizeof(setentry) + +_sizeof_Cdigit = int.__itemsize__ +if _sizeof_Cdigit < 2: # pragma: no coverage + raise AssertionError('sizeof(%s) bad: %d' % ('digit', _sizeof_Cdigit)) + +_builtins2 = (range,) + +# Get character size for internal unicode representation in Python < 3.3 +u = '\0'.encode('utf-8') +_sizeof_Cunicode = len(u) +del u + +try: # Size of GC header, sizeof(PyGC_Head) + import _testcapi as t + _sizeof_CPyGC_Head = t.SIZEOF_PYGC_HEAD # new in Python 2.6 +except (ImportError, AttributeError): # sizeof(PyGC_Head) + # alignment should be to sizeof(long double) but there + # is no way to obtain that value, assume twice double + t = calcsize('2d') - 1 + _sizeof_CPyGC_Head = (_calcsize('2Pz') + t) & ~t +del t + +# Size of refcounts (Python debug build only) +if hasattr(sys, 'gettotalrefcount'): # pragma: no coverage + _sizeof_Crefcounts = _calcsize('2z') +else: + _sizeof_Crefcounts = 0 + +from abc import ABCMeta + +# Some flags from .../Include/object.h +_Py_TPFLAGS_HEAPTYPE = 1 << 9 # Py_TPFLAGS_HEAPTYPE +_Py_TPFLAGS_HAVE_GC = 1 << 14 # Py_TPFLAGS_HAVE_GC + +_Type_type = type(type) # == type and new-style class type + + +# Compatibility functions for more uniform +# behavior across Python version 2.2 thu 3+ + +def _items(obj): # dict only + '''Return iter-/generator, preferably. + ''' + o = getattr(obj, 'iteritems', obj.items) + if _callable(o): + return o() + else: + return o or () + + +def _keys(obj): # dict only + '''Return iter-/generator, preferably. + ''' + o = getattr(obj, 'iterkeys', obj.keys) + if _callable(o): + return o() + else: + return o or () + + +def _values(obj): # dict only + '''Return iter-/generator, preferably. + ''' + o = getattr(obj, 'itervalues', obj.values) + if _callable(o): + return o() + else: + return o or () + + +try: # callable() builtin + _callable = callable +except NameError: # callable() removed in Python 3+ + def _callable(obj): + '''Substitute for callable().''' + return hasattr(obj, '__call__') + +# 'cell' is holding data used in closures +c = (lambda unused: (lambda: unused))(None) +_cell_type = type(c.__closure__[0]) # type: ignore +del c + +from gc import get_objects as _getobjects # containers only? + +if sys.platform == 'ios': # Apple iOS + _gc_getobjects = _getobjects + + def _getobjects(): # PYCHOK expected + # avoid Pythonista3/Python 3+ crash + return tuple(o for o in _gc_getobjects() if not _isNULL(o)) + +from gc import get_referents as _getreferents + +# sys.getsizeof() new in Python 2.6 +_getsizeof = sys.getsizeof # overridden below +_getsizeof_excls = () # types not sys.getsizeof'd + +from sys import intern as _intern + + +# Private functions + +def _basicsize(t, base=0, heap=False, obj=None): + '''Get non-zero basicsize of type, + including the header sizes. + ''' + s = max(getattr(t, '__basicsize__', 0), base) + # include gc header size + if t != _Type_type: + h = getattr(t, '__flags__', 0) & _Py_TPFLAGS_HAVE_GC + elif heap: # type, allocated on heap + h = True + else: # None has no __flags__ attr + h = getattr(obj, '__flags__', 0) & _Py_TPFLAGS_HEAPTYPE + if h: + s += _sizeof_CPyGC_Head + # include reference counters + return s + _sizeof_Crefcounts + + +def _classof(obj, dflt=None): + '''Return the object's class object. + ''' + return getattr(obj, '__class__', dflt) + + +def _derive_typedef(typ): + '''Return single, existing super type typedef or None. + ''' + v = [v for v in _values(_typedefs) if _issubclass(typ, v.type)] + if len(v) == 1: + return v[0] + return None + + +def _dir2(obj, pref='', excl=(), slots=None, itor=''): + '''Return an attribute name, object 2-tuple for certain + attributes or for the ``__slots__`` attributes of the + given object, but not both. Any iterator referent + objects are returned with the given name if the + latter is non-empty. + ''' + if slots: # __slots__ attrs + if hasattr(obj, slots): + # collect all inherited __slots__ attrs + # from list, tuple, or dict __slots__, + # while removing any duplicate attrs + s = {} + for c in type(obj).mro(): + for a in getattr(c, slots, ()): + if a.startswith('__'): + a = '_' + c.__name__ + a + if hasattr(obj, a): + s.setdefault(a, getattr(obj, a)) + # assume __slots__ tuple-like is holding the values + # yield slots, _Slots(s) # _keys(s) ... REMOVED, + # see _Slots.__doc__ further below + for t in _items(s): + yield t # attr name, value + elif itor: # iterator referents + for o in obj: # iter(obj) + yield itor, o + else: # regular attrs + for a in dir(obj): + if a.startswith(pref) and hasattr(obj, a) and a not in excl: + yield a, getattr(obj, a) + + +def _getsizeof_excls_add(typ): + '''Add another type to the tuple of types to be + excluded from sys.getsizeof due to errors. + ''' + global _getsizeof_excls + if typ and typ not in _getsizeof_excls: + _getsizeof_excls += (typ,) + + +def _infer_dict(obj): + '''Return True for likely dict object via duck typing. + ''' + for attrs in (('items', 'keys', 'values'), # 'update', + ('iteritems', 'iterkeys', 'itervalues')): + attrs += '__len__', 'get', 'has_key' + if all(_callable(getattr(obj, a, None)) for a in attrs): + return True + return False + + +def _isbuiltin2(obj): + '''Return True for builtins like Python 2. + ''' + # range is no longer a built-in in Python 3+ + return isbuiltin(obj) or obj in _builtins2 + + +def _iscell(obj): + '''Return True if obj is a cell as used in a closure. + ''' + return isinstance(obj, _cell_type) + + +def _isdictclass(obj): + '''Return True for known dict objects. + ''' + c = _classof(obj) + return c and c.__name__ in _dict_classes.get(c.__module__, ()) + + +def _isframe(obj): + '''Return True for a stack frame object. + ''' + try: # safe isframe(), see pympler.muppy + return isframe(obj) + except ReferenceError: + return False + + +def _isnamedtuple(obj): + '''Named tuples are identified via duck typing: + + ''' + return isinstance(obj, tuple) and hasattr(obj, '_fields') + + +def _isNULL(obj): + '''Prevent asizeof(all=True, ...) crash. + + Sizing gc.get_objects() crashes in Pythonista3 with + Python 3.5.1 on iOS due to 1-tuple (,) object, + see . + ''' + return isinstance(obj, tuple) and len(obj) == 1 \ + and repr(obj) == '(,)' + + +def _issubclass(sub, sup): + '''Safe issubclass(). + ''' + if sup is not object: + try: + return issubclass(sub, sup) + except TypeError: + pass + return False + + +def _itemsize(t, item=0): + '''Get non-zero itemsize of type. + ''' + # replace zero value with default + return getattr(t, '__itemsize__', 0) or item + + +def _kwdstr(**kwds): + '''Keyword arguments as a string. + ''' + return ', '.join(sorted('%s=%r' % kv for kv in _items(kwds))) + + +def _lengstr(obj): + '''Object length as a string. + ''' + n = leng(obj) + if n is None: # no len + r = '' + elif n > _len(obj): # extended + r = ' leng %d!' % n + else: + r = ' leng %d' % n + return r + + +def _moduleof(obj, dflt=''): + '''Return the object's module name. + ''' + return getattr(obj, '__module__', dflt) + + +def _nameof(obj, dflt=''): + '''Return the name of an object. + ''' + return getattr(obj, '__name__', dflt) + + +def _objs_opts_x(objs, all=None, **opts): + '''Return given or 'all' objects + and the remaining options. + ''' + if objs: # given objects + t = objs + x = False + elif all in (False, None): + t = () + x = True + elif all is True: # 'all' objects + t = _getobjects() + x = True + else: + raise ValueError('invalid option: %s=%r' % ('all', all)) + return t, opts, x + + +def _p100(part, total, prec=1): + '''Return percentage as string. + ''' + r = float(total) + if r: + r = part * 100.0 / r + return '%.*f%%' % (prec, r) + return 'n/a' + + +def _plural(num): + '''Return 's' if plural. + ''' + if num == 1: + s = '' + else: + s = 's' + return s + + +def _power2(n): + '''Find the next power of 2. + ''' + p2 = 16 + while n > p2: + p2 += p2 + return p2 + + +def _prepr(obj, clip=0): + '''Prettify and clip long repr() string. + ''' + return _repr(obj, clip=clip).strip('<>').replace("'", '') # remove <''> + + +def _printf(fmt, *args, **print3options): + '''Formatted print to sys.stdout or given stream. + + *print3options* -- some keyword arguments, like Python 3+ print. + ''' + if print3options: # like Python 3+ + f = print3options.get('file', None) or sys.stdout + if args: + f.write(fmt % args) + else: + f.write(fmt) + f.write(print3options.get('end', linesep)) + if print3options.get('flush', False): + f.flush() + elif args: + print(fmt % args) + else: + print(fmt) + + +def _refs(obj, named, *attrs, **kwds): + '''Return specific attribute objects of an object. + ''' + if named: + _N = _NamedRef + else: + def _N(_, o): + return o + + for a in attrs: # cf. inspect.getmembers() + if hasattr(obj, a): + yield _N(a, getattr(obj, a)) + if kwds: # kwds are _dir2() args + for a, o in _dir2(obj, **kwds): + yield _N(a, o) + + +def _repr(obj, clip=80): + '''Clip long repr() string. + ''' + try: # safe repr() + r = repr(obj).replace(linesep, '\\n') + except Exception: + r = 'N/A' + if len(r) > clip > 0: + h = (clip // 2) - 2 + if h > 0: + r = r[:h] + '....' + r[-h:] + return r + + +def _SI(size, K=1024, i='i'): + '''Return size as SI string. + ''' + if 1 < K <= size: + f = float(size) + for si in iter('KMGPTE'): + f /= K + if f < K: + return ' or %.1f %s%sB' % (f, si, i) + return '' + + +def _SI2(size, **kwds): + '''Return size as regular plus SI string. + ''' + return str(size) + _SI(size, **kwds) + + +# Type-specific referents functions + +def _cell_refs(obj, named): + try: # handle 'empty' cells + o = obj.cell_contents + if named: + o = _NamedRef('cell_contents', o) + yield o + except (AttributeError, ValueError): + pass + + +def _class_refs(obj, named): + '''Return specific referents of a class object. + ''' + return _refs(obj, named, '__class__', '__doc__', '__mro__', + '__name__', '__slots__', '__weakref__', + '__dict__') # __dict__ last + + +def _co_refs(obj, named): + '''Return specific referents of a code object. + ''' + return _refs(obj, named, pref='co_') + + +def _dict_refs(obj, named): + '''Return key and value objects of a dict/proxy. + ''' + try: + if named: + for k, v in _items(obj): + s = str(k) + yield _NamedRef('[K] ' + s, k) + yield _NamedRef('[V] ' + s + ': ' + _repr(v), v) + else: + for k, v in _items(obj): + yield k + yield v + except (KeyError, ReferenceError, TypeError) as x: + warnings.warn("Iterating '%s': %r" % (_classof(obj), x)) + + +def _enum_refs(obj, named): + '''Return specific referents of an enumerate object. + ''' + return _refs(obj, named, '__doc__') + + +def _exc_refs(obj, named): + '''Return specific referents of an Exception object. + ''' + # .message raises DeprecationWarning in Python 2.6 + return _refs(obj, named, 'args', 'filename', 'lineno', 'msg', 'text') # , 'message', 'mixed' + + +def _file_refs(obj, named): + '''Return specific referents of a file object. + ''' + return _refs(obj, named, 'mode', 'name') + + +def _frame_refs(obj, named): + '''Return specific referents of a frame object. + ''' + return _refs(obj, named, pref='f_') + + +def _func_refs(obj, named): + '''Return specific referents of a function or lambda object. + ''' + return _refs(obj, named, '__doc__', '__name__', '__code__', '__closure__', + pref='func_', excl=('func_globals',)) + + +def _gen_refs(obj, named): + '''Return the referent(s) of a generator (expression) object. + ''' + # only some gi_frame attrs + f = getattr(obj, 'gi_frame', None) + return _refs(f, named, 'f_locals', 'f_code') +# do not yield any items to keep generator intact +# for r in _refs(f, named, 'f_locals', 'f_code'): +# yield r +# for r in obj: +# yield r + + +def _im_refs(obj, named): + '''Return specific referents of a method object. + ''' + return _refs(obj, named, '__doc__', '__name__', '__code__', pref='im_') + + +def _inst_refs(obj, named): + '''Return specific referents of a class instance. + ''' + return _refs(obj, named, '__dict__', '__class__', slots='__slots__') + + +def _iter_refs(obj, named): + '''Return the referent(s) of an iterator object. + ''' + r = _getreferents(obj) # special case + return _refs(r, named, itor=_nameof(obj) or 'iteref') + + +def _module_refs(obj, named): + '''Return specific referents of a module object. + ''' + # ignore this very module + if obj.__name__ == __name__: + return () + # module is essentially a dict + return _dict_refs(obj.__dict__, named) + + +def _namedtuple_refs(obj, named): + '''Return specific referents of obj-as-sequence and slots but exclude dict. + ''' + for r in _refs(obj, named, '__class__', slots='__slots__'): + yield r + for r in obj: + yield r + + +def _prop_refs(obj, named): + '''Return specific referents of a property object. + ''' + return _refs(obj, named, '__doc__', pref='f') + + +def _seq_refs(obj, unused): # named unused for PyChecker + '''Return specific referents of a frozen/set, list, tuple and xrange object. + ''' + return obj # XXX for r in obj: yield r + + +def _stat_refs(obj, named): + '''Return referents of a os.stat object. + ''' + return _refs(obj, named, pref='st_') + + +def _statvfs_refs(obj, named): + '''Return referents of a os.statvfs object. + ''' + return _refs(obj, named, pref='f_') + + +def _tb_refs(obj, named): + '''Return specific referents of a traceback object. + ''' + return _refs(obj, named, pref='tb_') + + +def _type_refs(obj, named): + '''Return specific referents of a type object. + ''' + return _refs(obj, named, '__doc__', '__mro__', '__name__', + '__slots__', '__weakref__', '__dict__') + + +def _weak_refs(obj, unused): # named unused for PyChecker + '''Return weakly referent object. + ''' + try: # ignore 'key' of KeyedRef + return (obj(),) + except Exception: # XXX ReferenceError + return () + + +_all_refs = (None, _cell_refs, _class_refs, _co_refs, _dict_refs, _enum_refs, + _exc_refs, _file_refs, _frame_refs, _func_refs, _gen_refs, + _im_refs, _inst_refs, _iter_refs, _module_refs, _namedtuple_refs, + _prop_refs, _seq_refs, _stat_refs, _statvfs_refs, _tb_refs, + _type_refs, _weak_refs) + + +# Type-specific length functions + +def _len(obj): + '''Safe len(). + ''' + try: + return len(obj) + except TypeError: # no len() + return 0 + + +def _len_bytearray(obj): + '''Bytearray size. + ''' + return obj.__alloc__() + + +def _len_code(obj): # see .../Lib/test/test_sys.py + '''Length of code object (stack and variables only). + ''' + return (obj.co_stacksize + obj.co_nlocals + + _len(obj.co_freevars) + _len(obj.co_cellvars) - 1) + + +def _len_dict(obj): + '''Dict length in items (estimate). + ''' + n = len(obj) # active items + if n < 6: # ma_smalltable ... + n = 0 # ... in basicsize + else: # at least one unused + n = _power2(n + 1) + return n + + +def _len_frame(obj): + '''Length of a frame object. + ''' + c = getattr(obj, 'f_code', None) + if c: + n = _len_code(c) + else: + n = 0 + return n + + +_digit2p2 = 1 << (_sizeof_Cdigit << 3) +_digitmax = _digit2p2 - 1 # == (2 * PyLong_MASK + 1) +_digitlog = 1.0 / log(_digit2p2) + + +def _len_int(obj): + '''Length of multi-precision int (aka long) in digits. + ''' + if obj: + n, i = 1, abs(obj) + if i > _digitmax: + # no log(x[, base]) in Python 2.2 + n += int(log(i) * _digitlog) + else: # zero + n = 0 + return n + + +def _len_iter(obj): + '''Length (hint) of an iterator. + ''' + n = getattr(obj, '__length_hint__', None) + if n: + n = n() + else: # try len() + n = _len(obj) + return n + + +def _len_list(obj): + '''Length of list (estimate). + ''' + n = len(obj) + # estimate over-allocation + if n > 8: + n += 6 + (n >> 3) + elif n: + n += 4 + return n + + +def _len_module(obj): + '''Module length. + ''' + return _len(obj.__dict__) # _len(dir(obj)) + + +def _len_set(obj): + '''Length of frozen/set (estimate). + ''' + n = len(obj) + if n > 8: # assume half filled + n = _power2(n + n - 2) + elif n: # at least 8 + n = 8 + return n + + +def _len_slice(obj): + '''Slice length. + ''' + try: + return ((obj.stop - obj.start + 1) // obj.step) + except (AttributeError, TypeError): + return 0 + + +# REMOVED, see _Slots.__doc__ +# def _len_slots(obj): +# '''Slots length. +# ''' +# return len(obj) - 1 + + +def _len_struct(obj): + '''Struct length in bytes. + ''' + try: + return obj.size + except AttributeError: + return 0 + + +def _len_unicode(obj): + '''Unicode size. + ''' + return len(obj) + 1 + + +_all_lens = (None, _len, _len_bytearray, _len_code, _len_dict, + _len_frame, _len_int, _len_iter, _len_list, + _len_module, _len_set, _len_slice, _len_struct, + _len_unicode) # type: Tuple[Union[None, Callable], ...] # _len_array, _len_numpy, _len_slots + + +# More private functions and classes + +_old_style = '*' # marker +_new_style = '' # no marker + + +class _Claskey(object): + '''Wrapper for class objects. + ''' + __slots__ = ('_obj', '_sty') + + def __init__(self, obj, style): + self._obj = obj # XXX Weakref.ref(obj) + self._sty = style + + def __str__(self): + r = str(self._obj) + if r.endswith('>'): + r = '%s%s def>' % (r[:-1], self._sty) + elif self._sty is _old_style and not r.startswith('class '): + r = 'class %s%s def' % (r, self._sty) + else: + r = '%s%s def' % (r, self._sty) + return r + __repr__ = __str__ + + +# For most objects, the object type is used as the key in the +# _typedefs dict further below, except class and type objects +# and old-style instances. Those are wrapped with separate +# _Claskey or _Instkey instances to be able (1) to distinguish +# instances of different old-style classes by class, (2) to +# distinguish class (and type) instances from class (and type) +# definitions for new-style classes and (3) provide similar +# results for repr() and str() of new- and old-style classes +# and instances. + +_claskeys = {} # type: Dict[int, _Claskey] + + +def _claskey(obj, style): + '''Wrap an old- or new-style class object. + ''' + i = id(obj) + k = _claskeys.get(i, None) + if not k: + _claskeys[i] = k = _Claskey(obj, style) + return k + + +def _keytuple(obj): # PYCHOK expected + '''Return class and instance keys for a class. + ''' + if type(obj) is _Type_type: # isclass(obj): + return _claskey(obj, _new_style), obj + return None, None # not a class + + +def _objkey(obj): # PYCHOK expected + '''Return the key for any object. + ''' + k = type(obj) + if k is _Type_type: # isclass(obj): + k = _claskey(obj, _new_style) + return k + + +class _NamedRef(object): + '''Store referred object along + with the name of the referent. + ''' + __slots__ = ('name', 'ref') + + def __init__(self, name, ref): + self.name = name + self.ref = ref + + +# class _Slots(tuple): +# '''Wrapper class for __slots__ attribute at class definition. +# The instance-specific __slots__ attributes are stored in +# a "tuple-like" space inside the instance, see Luciano +# Ramalho, "Fluent Python", page 274+, O'Reilly, 2016 or +# at , then search for +# "Fluent Python" "Space Savings with the __slots__". +# ''' +# pass + + +# Kinds of _Typedefs +_i = _intern +_all_kinds = (_kind_static, _kind_dynamic, _kind_derived, _kind_ignored, _kind_inferred) = ( + _i('static'), _i('dynamic'), _i('derived'), _i('ignored'), _i('inferred')) +del _i + +_Not_vari = '' # non-variable item size + + +class _Typedef(object): + '''Type definition class. + ''' + __slots__ = { + 'base': 0, # basic size in bytes + 'item': 0, # item size in bytes + 'leng': None, # or _len_...() function + 'refs': None, # or _..._refs() function + 'both': None, # both data and code if True, code only if False + 'kind': None, # _kind_... value + 'type': None, # original type + 'vari': None} # item size attr name or _Not_vari + + def __init__(self, **kwds): + self.reset(**kwds) + + def __lt__(self, unused): # for Python 3+ + return True + + def __repr__(self): + return repr(self.args()) + + def __str__(self): + t = [str(self.base), str(self.item)] + for f in (self.leng, self.refs): + if f: + t.append(f.__name__) + else: + t.append('n/a') + if not self.both: + t.append('(code only)') + return ', '.join(t) + + def args(self): # as args tuple + '''Return all attributes as arguments tuple. + ''' + return (self.base, self.item, self.leng, self.refs, + self.both, self.kind, self.type) + + def dup(self, other=None, **kwds): + '''Duplicate attributes of dict or other typedef. + ''' + if other is None: + d = _dict_typedef.kwds() + else: + d = other.kwds() + d.update(kwds) + self.reset(**d) + + def flat(self, obj, mask=0): + '''Return the aligned flat size. + ''' + s = self.base + if self.leng and self.item > 0: # include items + s += self.leng(obj) * self.item + # workaround sys.getsizeof (and numpy?) bug ... some + # types are incorrectly sized in some Python versions + # (note, isinstance(obj, ()) == False) + if not isinstance(obj, _getsizeof_excls): + s = _getsizeof(obj, s) + if mask: # align + s = (s + mask) & ~mask + return s + + def format(self): + '''Return format dict. + ''' + i = self.item + if self.vari: + i = 'var' + c = n = '' + if not self.both: + c = ' (code only)' + if self.leng: + n = ' (%s)' % _nameof(self.leng) + return dict(base=self.base, item=i, leng=n, code=c, + kind=self.kind) + + def kwds(self): + '''Return all attributes as keywords dict. + ''' + return dict(base=self.base, both=self.both, + item=self.item, kind=self.kind, + leng=self.leng, refs=self.refs, + type=self.type, vari=self.vari) + + def save(self, t, base=0, heap=False): + '''Save this typedef plus its class typedef. + ''' + c, k = _keytuple(t) + if k and k not in _typedefs: # instance key + _typedefs[k] = self + if c and c not in _typedefs: # class key + if t.__module__ in _builtin_modules: + k = _kind_ignored # default + else: + k = self.kind + _typedefs[c] = _Typedef(base=_basicsize(type(t), base=base, heap=heap), + refs=_type_refs, + both=False, kind=k, type=t) + elif t not in _typedefs: + if not _isbuiltin2(t): # array, range, xrange in Python 2.x + s = ' '.join((self.vari, _moduleof(t), _nameof(t))) + s = '%r %s %s' % ((c, k), self.both, s.strip()) + raise KeyError('asizeof typedef %r bad: %s' % (self, s)) + + _typedefs[t] = _Typedef(base=_basicsize(t, base=base), + both=False, kind=_kind_ignored, type=t) + + def set(self, safe_len=False, **kwds): + '''Set one or more attributes. + ''' + if kwds: # double check + d = self.kwds() + d.update(kwds) + self.reset(**d) + if safe_len and self.item: + self.leng = _len + + def reset(self, base=0, item=0, leng=None, refs=None, + both=True, kind=None, type=None, vari=_Not_vari): + '''Reset all specified attributes. + ''' + if base < 0: + raise ValueError('invalid option: %s=%r' % ('base', base)) + else: + self.base = base + if item < 0: + raise ValueError('invalid option: %s=%r' % ('item', item)) + else: + self.item = item + if leng in _all_lens: # XXX or _callable(leng) + self.leng = leng + else: + raise ValueError('invalid option: %s=%r' % ('leng', leng)) + if refs in _all_refs: # XXX or _callable(refs) + self.refs = refs + else: + raise ValueError('invalid option: %s=%r' % ('refs', refs)) + if both in (False, True): + self.both = both + else: + raise ValueError('invalid option: %s=%r' % ('both', both)) + if kind in _all_kinds: + self.kind = kind + else: + raise ValueError('invalid option: %s=%r' % ('kind', kind)) + self.type = type + self.vari = vari or _Not_vari + if str(self.vari) != self.vari: + raise ValueError('invalid option: %s=%r' % ('vari', vari)) + + +_typedefs = {} # type: Dict[type, _Typedef] + + +def _typedef_both(t, base=0, item=0, leng=None, refs=None, + kind=_kind_static, heap=False, vari=_Not_vari): + '''Add new typedef for both data and code. + ''' + v = _Typedef(base=_basicsize(t, base=base), item=_itemsize(t, item), + refs=refs, leng=leng, + both=True, kind=kind, type=t, vari=vari) + v.save(t, base=base, heap=heap) + return v # for _dict_typedef + + +def _typedef_code(t, base=0, refs=None, kind=_kind_static, heap=False): + '''Add new typedef for code only. + ''' + v = _Typedef(base=_basicsize(t, base=base), + refs=refs, + both=False, kind=kind, type=t) + v.save(t, base=base, heap=heap) + return v # for _dict_typedef + + +# Static typedefs for data and code types +_typedef_both(complex) +_typedef_both(float) +_typedef_both(list, refs=_seq_refs, leng=_len_list, item=_sizeof_Cvoidp) # sizeof(PyObject*) +_typedef_both(tuple, refs=_seq_refs, leng=_len, item=_sizeof_Cvoidp) # sizeof(PyObject*) +_typedef_both(property, refs=_prop_refs) +_typedef_both(type(Ellipsis)) +_typedef_both(type(None)) + +# _Slots are "tuple-like", REMOVED see _Slots.__doc__ +# _typedef_both(_Slots, item=_sizeof_Cvoidp, +# leng=_len_slots, # length less one +# refs=None, # but no referents +# heap=True) # plus head + +# dict, dictproxy, dict_proxy and other dict-like types +_dict_typedef = _typedef_both(dict, item=_sizeof_CPyDictEntry, leng=_len_dict, refs=_dict_refs) +# XXX any class __dict__ is in Python 3+? +_typedef_both(type(_Typedef.__dict__), item=_sizeof_CPyDictEntry, leng=_len_dict, refs=_dict_refs) +# other dict-like classes and types may be derived or inferred, +# provided the module and class name is listed here (see functions +# adict, _isdictclass and _infer_dict for further details) +_dict_classes = {'UserDict': ('IterableUserDict', 'UserDict'), + 'weakref': ('WeakKeyDictionary', 'WeakValueDictionary')} +try: # is essentially a dict + _typedef_both(Types.ModuleType, base=_dict_typedef.base, + item=_dict_typedef.item + _sizeof_CPyModuleObject, + leng=_len_module, refs=_module_refs) +except AttributeError: # missing + pass + +# Newer or obsolete types +from array import array # array type + +def _array_kwds(obj): + if hasattr(obj, 'itemsize'): + v = 'itemsize' + else: + v = _Not_vari + # since item size varies by the array data type, set + # itemsize to 1 byte and use _len_array in bytes; note, + # function itemsize returns the actual size in bytes + # and function leng returns the length in number of items + return dict(leng=_len_array, item=_sizeof_Cbyte, vari=v) + +def _len_array(obj): + '''Array length (in bytes!). + ''' + return len(obj) * obj.itemsize + +_all_lens += (_len_array,) # type: ignore + +_typedef_both(array, **_array_kwds(array('d', []))) + +v = sys.version_info +_array_excl = (v[0] == 2 and v < (2, 7, 4)) or \ + (v[0] == 3 and v < (3, 2, 4)) +if _array_excl: # see function _typedef below + _getsizeof_excls_add(array) + +del v + +try: # bool has non-zero __itemsize__ in 3.0 + _typedef_both(bool) +except NameError: # missing + pass + +try: + _typedef_both(bytearray, item=_sizeof_Cbyte, leng=_len_bytearray) +except NameError: # bytearray new in 2.6, 3.0 + pass +try: + if type(bytes) is not type(str): # bytes is str in 2.6, bytes new in 2.6, 3.0 + _typedef_both(bytes, item=_sizeof_Cbyte, leng=_len) # bytes new in 2.6, 3.0 +except NameError: # missing + pass +# try: # XXX like bytes +# _typedef_both(str8, item=_sizeof_Cbyte, leng=_len) # str8 new in 2.6, 3.0 +# except NameError: # missing +# pass + +try: + _typedef_both(enumerate, refs=_enum_refs) +except NameError: # missing + pass + +try: # Exception is type in Python 3+ + _typedef_both(Exception, refs=_exc_refs) +except Exception: # missing + pass + +try: + _typedef_both(frozenset, item=_sizeof_Csetentry, leng=_len_set, refs=_seq_refs) +except NameError: # missing + pass +try: + _typedef_both(set, item=_sizeof_Csetentry, leng=_len_set, refs=_seq_refs) +except NameError: # missing + pass + +try: # not callable() + _typedef_both(Types.GetSetDescriptorType) +except AttributeError: # missing + pass + +_typedef_both(int, item=_sizeof_Cdigit, leng=_len_int) + +try: # not callable() + _typedef_both(Types.MemberDescriptorType) +except AttributeError: # missing + pass + +try: + _typedef_both(type(NotImplemented)) # == Types.NotImplementedType +except NameError: # missing + pass + +try: # MCCABE 14 + import numpy # NumPy array, matrix, etc. + + def _isnumpy(obj): + '''Return True for a NumPy arange, array, matrix, etc. instance. + ''' + try: + return isinstance(obj, _numpy_types) or (hasattr(obj, 'nbytes') and + _moduleof(_classof(obj)).startswith('numpy')) + except (AttributeError, OSError, ValueError): # on iOS/Pythonista + return False + + def _len_numpy(obj): + '''NumPy array, matrix, etc. length (in bytes!). + ''' + return obj.nbytes # == obj.size * obj.itemsize + + def _numpy_kwds(obj): + b = _getsizeof(obj, 96) - obj.nbytes # XXX 96..144 typical? + # since item size depends on the numpy data type, set + # itemsize to 1 byte and use _len_numpy in bytes; note, + # function itemsize returns the actual size in bytes, + # function alen returns the length in number of items + return dict(base=b, item=_sizeof_Cbyte, # not obj.itemsize + leng=_len_numpy, + refs=_numpy_refs, + vari='itemsize') + + def _numpy_refs(obj, named): + '''Return the .base object for NumPy slices, views, etc. + ''' + return _refs(obj, named, 'base') + + _all_lens += (_len_numpy,) + _all_refs += (_numpy_refs,) + + v = tuple(map(int, numpy.__version__.split('.')[:2])) + + if v < (1, 19): + t = (numpy.matrix(range(0)),) + else: # numpy.matrix deprecated in 1.19.3 + t = () + _numpy_types = () # type: Tuple[type, ...] + for d in (t + (numpy.array(range(0)), numpy.arange(0), + numpy.ma.masked_array([]), numpy.ndarray(0))): + t = type(d) + if t not in _numpy_types: + _numpy_types += (t,) + if _isnumpy(d): # double check + _typedef_both(t, **_numpy_kwds(d)) + else: + raise AssertionError('not %s: %r' % ('numpy', d)) + + # sizing numpy 1.13 arrays works fine, but 1.8 and older + # appears to suffer from sys.getsizeof() bug like array + _numpy_excl = v < (1, 9) + if _numpy_excl: # see function _typedef below + for t in _numpy_types: + _getsizeof_excls_add(t) + + del d, t, v +except ImportError: # no NumPy + _numpy_excl = numpy = None # type: ignore # see function _typedef below + + def _isnumpy(obj): # PYCHOK expected + '''Not applicable, no NumPy. + ''' + return False + +try: + _typedef_both(range) +except NameError: # missing + pass + +try: + _typedef_both(reversed, refs=_enum_refs) +except NameError: # missing + pass + +try: + _typedef_both(slice, item=_sizeof_Cvoidp, leng=_len_slice) # XXX worst-case itemsize? +except NameError: # missing + pass + +try: + from os import stat + _typedef_both(type(stat(curdir)), refs=_stat_refs) # stat_result +except ImportError: # missing + pass + +try: + from os import statvfs + _typedef_both(type(statvfs(curdir)), refs=_statvfs_refs, # statvfs_result + item=_sizeof_Cvoidp, leng=_len) +except ImportError: # missing + pass + +try: + from struct import Struct # only in Python 2.5 and 3.0 + _typedef_both(Struct, item=_sizeof_Cbyte, leng=_len_struct) # len in bytes +except ImportError: # missing + pass + +try: + _typedef_both(Types.TracebackType, refs=_tb_refs) +except AttributeError: # missing + pass + +_typedef_both(str, leng=_len_unicode, item=_sizeof_Cunicode) + +try: # + _typedef_both(Weakref.KeyedRef, refs=_weak_refs, heap=True) # plus head +except AttributeError: # missing + pass + +try: # + _typedef_both(Weakref.ProxyType) +except AttributeError: # missing + pass + +try: # + _typedef_both(Weakref.ReferenceType, refs=_weak_refs) +except AttributeError: # missing + pass + +# some other, callable types +_typedef_code(object, kind=_kind_ignored) +_typedef_code(super, kind=_kind_ignored) +_typedef_code(_Type_type, kind=_kind_ignored) + +try: + _typedef_code(classmethod, refs=_im_refs) +except NameError: + pass +try: + _typedef_code(staticmethod, refs=_im_refs) +except NameError: + pass +try: + _typedef_code(Types.MethodType, refs=_im_refs) +except NameError: + pass + +try: # generator (expression), no itemsize, no len(), not callable() + _typedef_both(Types.GeneratorType, refs=_gen_refs) +except AttributeError: # missing + pass + +try: # + _typedef_code(Weakref.CallableProxyType, refs=_weak_refs) +except AttributeError: # missing + pass + +# any type-specific iterators +s = [_items({}), _keys({}), _values({})] +try: # reversed list and tuples iterators + s.extend([reversed([]), reversed(())]) +except NameError: # missing + pass + +try: # callable-iterator + from re import finditer + s.append(finditer('', '')) +except ImportError: # missing + pass + +for t in _values(_typedefs): + if t.type and t.leng: + try: # create an (empty) instance + s.append(t.type()) + except TypeError: + pass +for t in s: + try: + i = iter(t) + _typedef_both(type(i), leng=_len_iter, refs=_iter_refs, item=0) # no itemsize! + except (KeyError, TypeError): # ignore non-iterables, duplicates, etc. + pass +del i, s, t + + +def _typedef(obj, derive=False, frames=False, infer=False): # MCCABE 25 + '''Create a new typedef for an object. + ''' + t = type(obj) + v = _Typedef(base=_basicsize(t, obj=obj), + kind=_kind_dynamic, type=t) +# _printf('new %r %r/%r %s', t, _basicsize(t), _itemsize(t), _repr(dir(obj))) + if ismodule(obj): # handle module like dict + v.dup(item=_dict_typedef.item + _sizeof_CPyModuleObject, + leng=_len_module, + refs=_module_refs) + elif _isframe(obj): + v.set(base=_basicsize(t, base=_sizeof_CPyFrameObject, obj=obj), + item=_itemsize(t), + leng=_len_frame, + refs=_frame_refs) + if not frames: # ignore frames + v.set(kind=_kind_ignored) + elif iscode(obj): + v.set(base=_basicsize(t, base=_sizeof_CPyCodeObject, obj=obj), + item=_sizeof_Cvoidp, + leng=_len_code, + refs=_co_refs, + both=False) # code only + elif _callable(obj): + if isclass(obj): # class or type + v.set(refs=_class_refs, + both=False) # code only + if _moduleof(obj) in _builtin_modules: + v.set(kind=_kind_ignored) + elif isbuiltin(obj): # function or method + v.set(both=False, # code only + kind=_kind_ignored) + elif isfunction(obj): + v.set(refs=_func_refs, + both=False) # code only + elif ismethod(obj): + v.set(refs=_im_refs, + both=False) # code only + elif isclass(t): # callable instance, e.g. SCons, + # handle like any other instance further below + v.set(item=_itemsize(t), safe_len=True, + refs=_inst_refs) # not code only! + else: + v.set(both=False) # code only + elif _issubclass(t, dict): + v.dup(kind=_kind_derived) + elif _isdictclass(obj) or (infer and _infer_dict(obj)): + v.dup(kind=_kind_inferred) + elif _iscell(obj): + v.set(item=_itemsize(t), refs=_cell_refs) + elif _isnamedtuple(obj): + v.set(refs=_namedtuple_refs) + elif numpy and _isnumpy(obj): # NumPy data + v.set(**_numpy_kwds(obj)) + if _numpy_excl: + _getsizeof_excls_add(t) + elif array and isinstance(obj, array): + v.set(**_array_kwds(obj)) + if _array_excl: + _getsizeof_excls_add(t) + elif _moduleof(obj) in _builtin_modules: + v.set(kind=_kind_ignored) + else: # assume an instance of some class + if derive: + p = _derive_typedef(t) + if p: # duplicate parent + v.dup(other=p, kind=_kind_derived) + return v + if _issubclass(t, Exception): + v.set(item=_itemsize(t), safe_len=True, + refs=_exc_refs, + kind=_kind_derived) + elif isinstance(obj, Exception): + v.set(item=_itemsize(t), safe_len=True, + refs=_exc_refs) + else: + v.set(item=_itemsize(t), safe_len=True, + refs=_inst_refs) + return v + + +class _Prof(object): + '''Internal type profile class. + ''' + high = 0 # largest size + number = 0 # number of (unique) objects + objref = None # largest obj (weakref) + total = 0 # total size + weak = False # objref is weakref(obj) + + def __cmp__(self, other): + if self.total < other.total: + return -1 + elif self.total > other.total: + return +1 + elif self.number < other.number: + return -1 + elif self.number > other.number: + return +1 + return 0 + + def __lt__(self, other): # for Python 3+ + return self.__cmp__(other) < 0 + + def format(self, clip=0, grand=None): + '''Return format dict. + ''' + if self.number > 1: # avg., plural + a, p = int(self.total / self.number), 's' + else: + a, p = self.total, '' + o = self.objref + if self.weak: + o = o() + t = _SI2(self.total) + if grand: + t += ' (%s)' % _p100(self.total, grand, prec=0) + return dict(avg=_SI2(a), high=_SI2(self.high), + lengstr=_lengstr(o), obj=_repr(o, clip=clip), + plural=p, total=t) + + def update(self, obj, size): + '''Update this profile. + ''' + self.number += 1 + self.total += size + if self.high < size: # largest + self.high = size + try: # prefer using weak ref + self.objref, self.weak = Weakref.ref(obj), True + except TypeError: + self.objref, self.weak = obj, False + + +class _Rank(object): + '''Internal largest object class. + ''' + __slots__ = { + 'deep': 0, # recursion depth + 'id': 0, # obj id + 'key': None, # Typedef + 'objref': None, # obj or Weakref.ref(obj) + 'pid': 0, # parent obj id + 'size': 0, # size in bytes + 'weak': False} # objref is Weakref.ref + + def __init__(self, key, obj, size, deep, pid): + self.deep = deep + self.id = id(obj) + self.key = key + try: # prefer using weak ref + self.objref, self.weak = Weakref.ref(obj), True + except TypeError: + self.objref, self.weak = obj, False + self.pid = pid + self.size = size + + def format(self, clip=0, id2x={}): + '''Return string. + ''' + o = self.objref + if self.weak: + o = o() + if self.deep > 0: + d = ' (at %s)' % (self.deep,) + else: + d = '' + if self.pid: + p = ', pix %s' % (id2x.get(self.pid, '?'),) + else: + p = '' + return '%s: %s%s, ix %d%s%s' % (_prepr(self.key, clip=clip), + _repr(o, clip=clip), _lengstr(o), id2x[self.id], d, p) + + +class _Seen(dict): + '''Internal obj visits counter. + ''' + def again(self, key): + try: + s = self[key] + 1 + except KeyError: + s = 1 + if s > 0: + self[key] = s + + +# Public classes + +class Asized(object): + '''Stores the results of an **asized** object in the following + 4 attributes: + + *size* -- total size of the object (including referents) + + *flat* -- flat size of the object (in bytes) + + *name* -- name or ``repr`` of the object + + *refs* -- tuple containing an **Asized** instance for each referent + ''' + __slots__ = ('flat', 'name', 'refs', 'size') + + def __init__(self, size, flat, refs=(), name=None): + self.size = size # total size + self.flat = flat # flat size + self.name = name # name, repr or None + self.refs = tuple(refs) + + def __str__(self): + return 'size %r, flat %r, refs[%d], name %r' % ( + self.size, self.flat, len(self.refs), self.name) + + def format(self, format='%(name)s size=%(size)d flat=%(flat)d', + depth=-1, order_by='size', indent=''): + '''Format the size information of the object and of all + sized referents as a string. + + *format* -- Specifies the format per instance (with 'name', + 'size' and 'flat' as interpolation parameters) + + *depth* -- Recursion level up to which the referents are + printed (use -1 for unlimited) + + *order_by* -- Control sort order of referents, valid choices + are 'name', 'size' and 'flat' + + *indent* -- Optional indentation (default '') + ''' + t = indent + (format % dict(size=self.size, flat=self.flat, + name=self.name)) + if depth and self.refs: + rs = sorted(self.refs, key=lambda x: getattr(x, order_by), + reverse=order_by in ('size', 'flat')) + rs = [r.format(format=format, depth=depth-1, order_by=order_by, + indent=indent+' ') for r in rs] + t = '\n'.join([t] + rs) + return t + + def get(self, name, dflt=None): + '''Return the named referent (or *dflt* if not found). + ''' + for ref in self.refs: + if name == ref.name: + return ref + return dflt + + +class Asizer(object): + '''Sizer state and options to accumulate sizes. + ''' + _above_ = 1024 # rank only objs of size 1K+ + _align_ = 8 + _clip_ = 80 + _code_ = False + _cutoff_ = 0 # in percent + _derive_ = False + _detail_ = 0 # for Asized only + _frames_ = False + _infer_ = False + _limit_ = 100 + _stats_ = 0 + + _depth = 0 # deepest recursion + _excl_d = None # {} + _ign_d = _kind_ignored + _incl = '' # or ' (incl. code)' + _mask = 7 # see _align_ + _missed = 0 # due to errors + _profile = False # no profiling + _profs = None # {} + _ranked = 0 + _ranks = [] # type: List[_Rank] # sorted by decreasing size + _seen = None # {} + _stream = None # I/O stream for printing + _total = 0 # total size + + def __init__(self, **opts): + '''New **Asizer** accumulator. + + See this module documentation for more details. + See method **reset** for all available options and defaults. + ''' + self._excl_d = {} + self.reset(**opts) + + def _c100(self, stats): + '''Cutoff as percentage (for backward compatibility) + ''' + s = int(stats) + c = int((stats - s) * 100.0 + 0.5) or self.cutoff + return s, c + + def _clear(self): + '''Clear state. + ''' + self._depth = 0 # recursion depth reached + self._incl = '' # or ' (incl. code)' + self._missed = 0 # due to errors + self._profile = False + self._profs = {} + self._ranked = 0 + self._ranks = [] + self._seen = _Seen() + self._total = 0 # total size + for k in _keys(self._excl_d): + self._excl_d[k] = 0 + # don't size, profile or rank private, possibly large objs + m = sys.modules[__name__] + self.exclude_objs(self, self._excl_d, self._profs, self._ranks, + self._seen, m, m.__dict__, m.__doc__, + _typedefs) + + def _nameof(self, obj): + '''Return the object's name. + ''' + return _nameof(obj, '') or self._repr(obj) + + def _prepr(self, obj): + '''Like **prepr()**. + ''' + return _prepr(obj, clip=self._clip_) + + def _printf(self, fmt, *args, **print3options): + '''Print to sys.stdout or the configured stream if any is + specified and if the file keyword argument is not already + set in the **print3options** for this specific call. + ''' + if self._stream and not print3options.get('file', None): + if args: + fmt = fmt % args + _printf(fmt, file=self._stream, **print3options) + else: + _printf(fmt, *args, **print3options) + + def _prof(self, key): + '''Get _Prof object. + ''' + p = self._profs.get(key, None) + if not p: + self._profs[key] = p = _Prof() + self.exclude_objs(p) # XXX superfluous? + return p + + def _rank(self, key, obj, size, deep, pid): + '''Rank 100 largest objects by size. + ''' + rs = self._ranks + # bisect, see + i, j = 0, len(rs) + while i < j: + m = (i + j) // 2 + if size < rs[m].size: + i = m + 1 + else: + j = m + if i < 100: + r = _Rank(key, obj, size, deep, pid) + rs.insert(i, r) + self.exclude_objs(r) # XXX superfluous? + while len(rs) > 100: + rs.pop() + # self._ranks[:] = rs[:100] + self._ranked += 1 + + def _repr(self, obj): + '''Like ``repr()``. + ''' + return _repr(obj, clip=self._clip_) + + def _sizer(self, obj, pid, deep, sized): # MCCABE 19 + '''Size an object, recursively. + ''' + s, f, i = 0, 0, id(obj) + if i not in self._seen: + self._seen[i] = 1 + elif deep or self._seen[i]: + # skip obj if seen before + # or if ref of a given obj + self._seen.again(i) + if sized: + s = sized(s, f, name=self._nameof(obj)) + self.exclude_objs(s) + return s # zero + else: # deep == seen[i] == 0 + self._seen.again(i) + try: + k, rs = _objkey(obj), [] + if k in self._excl_d: + self._excl_d[k] += 1 + else: + v = _typedefs.get(k, None) + if not v: # new typedef + _typedefs[k] = v = _typedef(obj, derive=self._derive_, + frames=self._frames_, + infer=self._infer_) + if (v.both or self._code_) and v.kind is not self._ign_d: + s = f = v.flat(obj, self._mask) # flat size + if self._profile: + # profile based on *flat* size + self._prof(k).update(obj, s) + # recurse, but not for nested modules + if v.refs and deep < self._limit_ \ + and not (deep and ismodule(obj)): + # add sizes of referents + z, d = self._sizer, deep + 1 + if sized and deep < self._detail_: + # use named referents + self.exclude_objs(rs) + for o in v.refs(obj, True): + if isinstance(o, _NamedRef): + r = z(o.ref, i, d, sized) + r.name = o.name + else: + r = z(o, i, d, sized) + r.name = self._nameof(o) + rs.append(r) + s += r.size + else: # just size and accumulate + for o in v.refs(obj, False): + s += z(o, i, d, None) + # deepest recursion reached + if self._depth < d: + self._depth = d + if self._stats_ and s > self._above_ > 0: + # rank based on *total* size + self._rank(k, obj, s, deep, pid) + except RuntimeError: # XXX RecursionLimitExceeded: + self._missed += 1 + if not deep: + self._total += s # accumulate + if sized: + s = sized(s, f, name=self._nameof(obj), refs=rs) + self.exclude_objs(s) + return s + + def _sizes(self, objs, sized=None): + '''Return the size or an **Asized** instance for each + given object plus the total size. The total includes + the size of duplicates only once. + ''' + self.exclude_refs(*objs) # skip refs to objs + s, t = {}, [] + self.exclude_objs(s, t) + for o in objs: + i = id(o) + if i in s: # duplicate + self._seen.again(i) + else: + s[i] = self._sizer(o, 0, 0, sized) + t.append(s[i]) + return tuple(t) + + @property + def above(self): + '''Get the large object size threshold (int). + ''' + return self._above_ + + @property + def align(self): + '''Get the size alignment (int). + ''' + return self._align_ + + def asized(self, *objs, **opts): + '''Size each object and return an **Asized** instance with + size information and referents up to the given detail + level (and with modified options, see method **set**). + + If only one object is given, the return value is the + **Asized** instance for that object. The **Asized** size + of duplicate and ignored objects will be zero. + ''' + if opts: + self.set(**opts) + t = self._sizes(objs, Asized) + if len(t) == 1: + t = t[0] + return t + + def asizeof(self, *objs, **opts): + '''Return the combined size of the given objects + (with modified options, see method **set**). + ''' + if opts: + self.set(**opts) + self.exclude_refs(*objs) # skip refs to objs + return sum(self._sizer(o, 0, 0, None) for o in objs) + + def asizesof(self, *objs, **opts): + '''Return the individual sizes of the given objects + (with modified options, see method **set**). + + The size of duplicate and ignored objects will be zero. + ''' + if opts: + self.set(**opts) + return self._sizes(objs, None) + + @property + def clip(self): + '''Get the clipped string length (int). + ''' + return self._clip_ + + @property + def code(self): + '''Size (byte) code (bool). + ''' + return self._code_ + + @property + def cutoff(self): + '''Stats cutoff (int). + ''' + return self._cutoff_ + + @property + def derive(self): + '''Derive types (bool). + ''' + return self._derive_ + + @property + def detail(self): + '''Get the detail level for **Asized** refs (int). + ''' + return self._detail_ + + @property + def duplicate(self): + '''Get the number of duplicate objects seen so far (int). + ''' + return sum(1 for v in _values(self._seen) if v > 1) # == len + + def exclude_objs(self, *objs): + '''Exclude the specified objects from sizing, profiling and ranking. + ''' + for o in objs: + self._seen.setdefault(id(o), -1) + + def exclude_refs(self, *objs): + '''Exclude any references to the specified objects from sizing. + + While any references to the given objects are excluded, the + objects will be sized if specified as positional arguments + in subsequent calls to methods **asizeof** and **asizesof**. + ''' + for o in objs: + self._seen.setdefault(id(o), 0) + + def exclude_types(self, *objs): + '''Exclude the specified object instances and types from sizing. + + All instances and types of the given objects are excluded, + even objects specified as positional arguments in subsequent + calls to methods **asizeof** and **asizesof**. + ''' + for o in objs: + for t in _keytuple(o): + if t and t not in self._excl_d: + self._excl_d[t] = 0 + + @property + def excluded(self): + '''Get the types being excluded (tuple). + ''' + return tuple(_keys(self._excl_d)) + + @property + def frames(self): + '''Ignore stack frames (bool). + ''' + return self._frames_ + + @property + def ignored(self): + '''Ignore certain types (bool). + ''' + return True if self._ign_d else False + + @property + def infer(self): + '''Infer types (bool). + ''' + return self._infer_ + + @property + def limit(self): + '''Get the recursion limit (int). + ''' + return self._limit_ + + @property + def missed(self): + '''Get the number of objects missed due to errors (int). + ''' + return self._missed + + def print_largest(self, w=0, cutoff=0, **print3options): + '''Print the largest objects. + + The available options and defaults are: + + *w=0* -- indentation for each line + + *cutoff=100* -- number of largest objects to print + + *print3options* -- some keyword arguments, like Python 3+ print + ''' + c = int(cutoff) if cutoff else self._cutoff_ + n = min(len(self._ranks), max(c, 0)) + s = self._above_ + if n > 0 and s > 0: + self._printf('%s%*d largest object%s (of %d over %d bytes%s)', linesep, + w, n, _plural(n), self._ranked, s, _SI(s), **print3options) + id2x = dict((r.id, i) for i, r in enumerate(self._ranks)) + for r in self._ranks[:n]: + s, t = r.size, r.format(self._clip_, id2x) + self._printf('%*d bytes%s: %s', w, s, _SI(s), t, **print3options) + + def print_profiles(self, w=0, cutoff=0, **print3options): + '''Print the profiles above *cutoff* percentage. + + The available options and defaults are: + + *w=0* -- indentation for each line + + *cutoff=0* -- minimum percentage printed + + *print3options* -- some keyword arguments, like Python 3+ print + ''' + # get the profiles with non-zero size or count + t = [(v, k) for k, v in _items(self._profs) if v.total > 0 or v.number > 1] + if (len(self._profs) - len(t)) < 9: # just show all + t = [(v, k) for k, v in _items(self._profs)] + if t: + s = '' + if self._total: + s = ' (% of grand total)' + c = int(cutoff) if cutoff else self._cutoff_ + C = int(c * 0.01 * self._total) + else: + C = c = 0 + self._printf('%s%*d profile%s: total%s, average, and largest flat size%s: largest object', + linesep, w, len(t), _plural(len(t)), s, self._incl, **print3options) + r = len(t) + for v, k in sorted(t, reverse=True): + s = 'object%(plural)s: %(total)s, %(avg)s, %(high)s: %(obj)s%(lengstr)s' % v.format(self._clip_, self._total) + self._printf('%*d %s %s', w, v.number, self._prepr(k), s, **print3options) + r -= 1 + if r > 1 and v.total < C: + self._printf('%+*d profiles below cutoff (%.0f%%)', w, r, c) + break + z = len(self._profs) - len(t) + if z > 0: + self._printf('%+*d %r object%s', w, z, 'zero', _plural(z), **print3options) + + def print_stats(self, objs=(), opts={}, sized=(), sizes=(), stats=3, **print3options): + '''Prints the statistics. + + The available options and defaults are: + + *w=0* -- indentation for each line + + *objs=()* -- optional, list of objects + + *opts={}* -- optional, dict of options used + + *sized=()* -- optional, tuple of **Asized** instances returned + + *sizes=()* -- optional, tuple of sizes returned + + *stats=3* -- print stats, see function **asizeof** + + *print3options* -- some keyword arguments, like Python 3+ print + ''' + s = min(opts.get('stats', stats) or 0, self.stats) + if s > 0: # print stats + w = len(str(self.missed + self.seen + self.total)) + 1 + t = c = '' + o = _kwdstr(**opts) + if o and objs: + c = ', ' + # print header line(s) + if sized and objs: + n = len(objs) + if n > 1: + self._printf('%sasized(...%s%s) ...', linesep, c, o, **print3options) + for i in range(n): # no enumerate in Python 2.2.3 + self._printf('%*d: %s', w - 1, i, sized[i], **print3options) + else: + self._printf('%sasized(%s): %s', linesep, o, sized, **print3options) + elif sizes and objs: + self._printf('%sasizesof(...%s%s) ...', linesep, c, o, **print3options) + for z, o in zip(sizes, objs): + self._printf('%*d bytes%s%s: %s', w, z, _SI(z), self._incl, self._repr(o), **print3options) + else: + if objs: + t = self._repr(objs) + self._printf('%sasizeof(%s%s%s) ...', linesep, t, c, o, **print3options) + # print summary + self.print_summary(w=w, objs=objs, **print3options) + # for backward compatibility, cutoff from fractional stats + s, c = self._c100(s) + self.print_largest(w=w, cutoff=c if s < 2 else 10, **print3options) + if s > 1: # print profile + self.print_profiles(w=w, cutoff=c, **print3options) + if s > 2: # print typedefs + self.print_typedefs(w=w, **print3options) # PYCHOK .print_largest? + + def print_summary(self, w=0, objs=(), **print3options): + '''Print the summary statistics. + + The available options and defaults are: + + *w=0* -- indentation for each line + + *objs=()* -- optional, list of objects + + *print3options* -- some keyword arguments, like Python 3+ print + ''' + self._printf('%*d bytes%s%s', w, self._total, _SI(self._total), self._incl, **print3options) + if self._mask: + self._printf('%*d byte aligned', w, self._mask + 1, **print3options) + self._printf('%*d byte sizeof(void*)', w, _sizeof_Cvoidp, **print3options) + n = len(objs or ()) + self._printf('%*d object%s %s', w, n, _plural(n), 'given', **print3options) + n = self.sized + self._printf('%*d object%s %s', w, n, _plural(n), 'sized', **print3options) + if self._excl_d: + n = sum(_values(self._excl_d)) + self._printf('%*d object%s %s', w, n, _plural(n), 'excluded', **print3options) + n = self.seen + self._printf('%*d object%s %s', w, n, _plural(n), 'seen', **print3options) + n = self.ranked + if n > 0: + self._printf('%*d object%s %s', w, n, _plural(n), 'ranked', **print3options) + n = self.missed + self._printf('%*d object%s %s', w, n, _plural(n), 'missed', **print3options) + n = self.duplicate + self._printf('%*d duplicate%s', w, n, _plural(n), **print3options) + if self._depth > 0: + self._printf('%*d deepest recursion', w, self._depth, **print3options) + + def print_typedefs(self, w=0, **print3options): + '''Print the types and dict tables. + + The available options and defaults are: + + *w=0* -- indentation for each line + + *print3options* -- some keyword arguments, like Python 3+ print + ''' + for k in _all_kinds: + # XXX Python 3+ doesn't sort type objects + t = [(self._prepr(a), v) for a, v in _items(_typedefs) + if v.kind == k and (v.both or self._code_)] + if t: + self._printf('%s%*d %s type%s: basicsize, itemsize, _len_(), _refs()', + linesep, w, len(t), k, _plural(len(t)), **print3options) + for a, v in sorted(t): + self._printf('%*s %s: %s', w, '', a, v, **print3options) + # dict and dict-like classes + t = sum(len(v) for v in _values(_dict_classes)) + if t: + self._printf('%s%*d dict/-like classes:', linesep, w, t, **print3options) + for m, v in _items(_dict_classes): + self._printf('%*s %s: %s', w, '', m, self._prepr(v), **print3options) + + @property + def ranked(self): + '''Get the number objects ranked by size so far (int). + ''' + return self._ranked + + def reset(self, above=1024, align=8, clip=80, code=False, # PYCHOK too many args + cutoff=10, derive=False, detail=0, frames=False, ignored=True, + infer=False, limit=100, stats=0, stream=None, **extra): + '''Reset sizing options, state, etc. to defaults. + + The available options and default values are: + + *above=0* -- threshold for largest objects stats + + *align=8* -- size alignment + + *code=False* -- incl. (byte)code size + + *cutoff=10* -- limit large objects or profiles stats + + *derive=False* -- derive from super type + + *detail=0* -- **Asized** refs level + + *frames=False* -- ignore frame objects + + *ignored=True* -- ignore certain types + + *infer=False* -- try to infer types + + *limit=100* -- recursion limit + + *stats=0* -- print statistics, see function **asizeof** + + *stream=None* -- output stream for printing + + See function **asizeof** for a description of the options. + ''' + if extra: + t = _plural(len(extra)), _kwdstr(**extra) + raise KeyError('invalid option%s: %s' % t) + # options + self._above_ = above + self._align_ = align + self._clip_ = clip + self._code_ = code + self._cutoff_ = cutoff + self._derive_ = derive + self._detail_ = detail # for Asized only + self._frames_ = frames + self._infer_ = infer + self._limit_ = limit + self._stats_ = stats + self._stream = stream + if ignored: + self._ign_d = _kind_ignored + else: + self._ign_d = None + # clear state + self._clear() + self.set(align=align, code=code, cutoff=cutoff, stats=stats) + + @property + def seen(self): + '''Get the number objects seen so far (int). + ''' + return sum(v for v in _values(self._seen) if v > 0) + + def set(self, above=None, align=None, code=None, cutoff=None, + frames=None, detail=None, limit=None, stats=None): + '''Set some sizing options. See also **reset**. + + The available options are: + + *above* -- threshold for largest objects stats + + *align* -- size alignment + + *code* -- incl. (byte)code size + + *cutoff* -- limit large objects or profiles stats + + *detail* -- **Asized** refs level + + *frames* -- size or ignore frame objects + + *limit* -- recursion limit + + *stats* -- print statistics, see function **asizeof** + + Any options not set remain unchanged from the previous setting. + ''' + # adjust + if above is not None: + self._above_ = int(above) + if align is not None: + self._align_ = align + if align > 1: + self._mask = align - 1 + if (self._mask & align) != 0: + raise ValueError('invalid option: %s=%r' % ('align', align)) + else: + self._mask = 0 + if code is not None: + self._code_ = code + if code: # incl. (byte)code + self._incl = ' (incl. code)' + if detail is not None: + self._detail_ = detail + if frames is not None: + self._frames_ = frames + if limit is not None: + self._limit_ = limit + if stats is not None: + if stats < 0: + raise ValueError('invalid option: %s=%r' % ('stats', stats)) + # for backward compatibility, cutoff from fractional stats + s, c = self._c100(stats) + self._cutoff_ = int(cutoff) if cutoff else c + self._stats_ = s + self._profile = s > 1 # profile types + + @property + def sized(self): + '''Get the number objects sized so far (int). + ''' + return sum(1 for v in _values(self._seen) if v > 0) + + @property + def stats(self): + '''Get the stats and cutoff setting (float). + ''' + return self._stats_ # + (self._cutoff_ * 0.01) + + @property + def total(self): + '''Get the total size (in bytes) accumulated so far. + ''' + return self._total + + +# Public functions + +def adict(*classes): + '''Install one or more classes to be handled as dict. + ''' + a = True + for c in classes: + # if class is dict-like, add class + # name to _dict_classes[module] + if isclass(c) and _infer_dict(c): + t = _dict_classes.get(c.__module__, ()) + if c.__name__ not in t: # extend tuple + _dict_classes[c.__module__] = t + (c.__name__,) + else: # not a dict-like class + a = False + return a # all installed if True + + +_asizer = Asizer() + + +def asized(*objs, **opts): + '''Return a tuple containing an **Asized** instance for each + object passed as positional argument. + + The available options and defaults are: + + *above=0* -- threshold for largest objects stats + + *align=8* -- size alignment + + *code=False* -- incl. (byte)code size + + *cutoff=10* -- limit large objects or profiles stats + + *derive=False* -- derive from super type + + *detail=0* -- Asized refs level + + *frames=False* -- ignore stack frame objects + + *ignored=True* -- ignore certain types + + *infer=False* -- try to infer types + + *limit=100* -- recursion limit + + *stats=0* -- print statistics + + If only one object is given, the return value is the **Asized** + instance for that object. Otherwise, the length of the returned + tuple matches the number of given objects. + + The **Asized** size of duplicate and ignored objects will be zero. + + Set *detail* to the desired referents level and *limit* to the + maximum recursion depth. + + See function **asizeof** for descriptions of the other options. + ''' + _asizer.reset(**opts) + if objs: + t = _asizer.asized(*objs) + _asizer.print_stats(objs, opts=opts, sized=t) # show opts as _kwdstr + _asizer._clear() + else: + t = () + return t + + +def asizeof(*objs, **opts): + '''Return the combined size (in bytes) of all objects passed + as positional arguments. + + The available options and defaults are: + + *above=0* -- threshold for largest objects stats + + *align=8* -- size alignment + + *clip=80* -- clip ``repr()`` strings + + *code=False* -- incl. (byte)code size + + *cutoff=10* -- limit large objects or profiles stats + + *derive=False* -- derive from super type + + *frames=False* -- ignore stack frame objects + + *ignored=True* -- ignore certain types + + *infer=False* -- try to infer types + + *limit=100* -- recursion limit + + *stats=0* -- print statistics + + Set *align* to a power of 2 to align sizes. Any value less + than 2 avoids size alignment. + + If *all* is True and if no positional arguments are supplied. + size all current gc objects, including module, global and stack + frame objects. + + A positive *clip* value truncates all repr() strings to at + most *clip* characters. + + The (byte)code size of callable objects like functions, + methods, classes, etc. is included only if *code* is True. + + If *derive* is True, new types are handled like an existing + (super) type provided there is one and only of those. + + By default certain base types like object, super, etc. are + ignored. Set *ignored* to False to include those. + + If *infer* is True, new types are inferred from attributes + (only implemented for dict types on callable attributes + as get, has_key, items, keys and values). + + Set *limit* to a positive value to accumulate the sizes of + the referents of each object, recursively up to the limit. + Using *limit=0* returns the sum of the flat sizes of the + given objects. High *limit* values may cause runtime errors + and miss objects for sizing. + + A positive value for *stats* prints up to 9 statistics, (1) + a summary of the number of objects sized and seen and a list + of the largests objects with size over *above* bytes, (2) a + simple profile of the sized objects by type and (3+) up to 6 + tables showing the static, dynamic, derived, ignored, inferred + and dict types used, found respectively installed. + The fractional part of the *stats* value (x 100) is the number + of largest objects shown for (*stats*1.+) or the cutoff + percentage for simple profiles for (*stats*=2.+). For example, + *stats=1.10* shows the summary and the 10 largest objects, + also the default. + + See this module documentation for the definition of flat size. + ''' + t, p, x = _objs_opts_x(objs, **opts) + _asizer.reset(**p) + if t: + if x: # don't size, profile or rank _getobjects tuple + _asizer.exclude_objs(t) + s = _asizer.asizeof(*t) + _asizer.print_stats(objs=t, opts=opts) # show opts as _kwdstr + _asizer._clear() + else: + s = 0 + return s + + +def asizesof(*objs, **opts): + '''Return a tuple containing the size (in bytes) of all objects + passed as positional arguments. + + The available options and defaults are: + + *above=1024* -- threshold for largest objects stats + + *align=8* -- size alignment + + *clip=80* -- clip ``repr()`` strings + + *code=False* -- incl. (byte)code size + + *cutoff=10* -- limit large objects or profiles stats + + *derive=False* -- derive from super type + + *frames=False* -- ignore stack frame objects + + *ignored=True* -- ignore certain types + + *infer=False* -- try to infer types + + *limit=100* -- recursion limit + + *stats=0* -- print statistics + + See function **asizeof** for a description of the options. + + The length of the returned tuple equals the number of given + objects. + + The size of duplicate and ignored objects will be zero. + ''' + _asizer.reset(**opts) + if objs: + t = _asizer.asizesof(*objs) + _asizer.print_stats(objs, opts=opts, sizes=t) # show opts as _kwdstr + _asizer._clear() + else: + t = () + return t + + +def _typedefof(obj, save=False, **opts): + '''Get the typedef for an object. + ''' + k = _objkey(obj) + v = _typedefs.get(k, None) + if not v: # new typedef + v = _typedef(obj, **opts) + if save: + _typedefs[k] = v + return v + + +def basicsize(obj, **opts): + '''Return the basic size of an object (in bytes). + + The available options and defaults are: + + *derive=False* -- derive type from super type + + *infer=False* -- try to infer types + + *save=False* -- save the object's type definition if new + + See this module documentation for the definition of *basic size*. + ''' + b = t = _typedefof(obj, **opts) + if t: + b = t.base + return b + + +def flatsize(obj, align=0, **opts): + '''Return the flat size of an object (in bytes), optionally aligned + to the given power of 2. + + See function **basicsize** for a description of other available options. + + See this module documentation for the definition of *flat size*. + ''' + f = t = _typedefof(obj, **opts) + if t: + if align > 1: + m = align - 1 + if (align & m) != 0: + raise ValueError('invalid option: %s=%r' % ('align', align)) + else: + m = 0 + f = t.flat(obj, mask=m) + return f + + +def itemsize(obj, **opts): + '''Return the item size of an object (in bytes). + + See function **basicsize** for a description of the available options. + + See this module documentation for the definition of *item size*. + ''' + i = t = _typedefof(obj, **opts) + if t: + i, v = t.item, t.vari + if v and i == _sizeof_Cbyte: + i = getattr(obj, v, i) + return i + + +def leng(obj, **opts): + '''Return the length of an object (in items). + + See function **basicsize** for a description of the available options. + ''' + n = t = _typedefof(obj, **opts) + if t: + n = t.leng + if n and _callable(n): + i, v, n = t.item, t.vari, n(obj) + if v and i == _sizeof_Cbyte: + i = getattr(obj, v, i) + if i > _sizeof_Cbyte: + n = n // i + return n + + +def named_refs(obj, **opts): + '''Return all named **referents** of an object (re-using + functionality from **asizeof**). + + Does not return un-named *referents*, e.g. objects in a list. + + See function **basicsize** for a description of the available options. + ''' + rs = [] + v = _typedefof(obj, **opts) + if v: + v = v.refs + if v and _callable(v): + for r in v(obj, True): + try: + rs.append((r.name, r.ref)) + except AttributeError: + pass + return rs + + +def refs(obj, **opts): + '''Return (a generator for) specific *referents* of an object. + + See function **basicsize** for a description of the available options. + ''' + v = _typedefof(obj, **opts) + if v: + v = v.refs + if v and _callable(v): + v = v(obj, False) + return v + + +if __name__ == '__main__': + + if '-v' in sys.argv: + import platform + print('%s %s (Python %s %s)' % (__file__, __version__, + sys.version.split()[0], + platform.architecture()[0])) + + elif '-types' in sys.argv: # print static _typedefs + n = len(_typedefs) + w = len(str(n)) * ' ' + _printf('%s%d type definitions: %s and %s, kind ... %s', linesep, + n, 'basic-', 'itemsize (leng)', '-type[def]s') + for k, td in sorted((_prepr(k), td) for k, td in _items(_typedefs)): + desc = '%(base)s and %(item)s%(leng)s, %(kind)s%(code)s' % td.format() + _printf('%s %s: %s', w, k, desc) + + else: + import gc + collect = False + if '-gc' in sys.argv: + collect = True + gc.collect() + + frames = '-frames' in sys.argv + + # just an example + asizeof(all=True, frames=frames, stats=1, above=1024) # print summary + 10 largest + + if collect: + print('gc.collect() %d' % (gc.collect(),)) + +# License from the initial version of this source file follows: + +# -------------------------------------------------------------------- +# Copyright (c) 2002-2019 -- ProphICy Semiconductor, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# - Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# - Neither the name of ProphICy Semiconductor, Inc. nor the names +# of its contributors may be used to endorse or promote products +# derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# -------------------------------------------------------------------- diff --git a/venv/lib/python3.9/site-packages/pympler/charts.py b/venv/lib/python3.9/site-packages/pympler/charts.py new file mode 100644 index 00000000..050cc027 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/charts.py @@ -0,0 +1,62 @@ +""" +Generate charts from gathered data. + +Requires **matplotlib**. +""" + +from pympler.classtracker_stats import Stats + +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + + def tracker_timespace(filename: str, stats: Stats) -> None: + """ + Create a time-space chart from a ``Stats`` instance. + """ + classlist = list(stats.index.keys()) + classlist.sort() + + for snapshot in stats.snapshots: + stats.annotate_snapshot(snapshot) + + timestamps = [fp.timestamp for fp in stats.snapshots] + offsets = [0] * len(stats.snapshots) + poly_labels = [] + polys = [] + for clsname in classlist: + pct = [fp.classes[clsname]['pct'] for fp in stats.snapshots + if fp.classes and clsname in fp.classes] + if max(pct) > 3.0: + sizes = [fp.classes[clsname]['sum'] for fp in stats.snapshots + if fp.classes and clsname in fp.classes] + sizes = [float(x) / (1024 * 1024) for x in sizes] + sizes = [offset + size for offset, size in zip(offsets, sizes)] + poly = matplotlib.mlab.poly_between(timestamps, offsets, sizes) + polys.append((poly, {'label': clsname})) + poly_labels.append(clsname) + offsets = sizes + + fig = plt.figure(figsize=(10, 4)) + axis = fig.add_subplot(111) + + axis.set_title("Snapshot Memory") + axis.set_xlabel("Execution Time [s]") + axis.set_ylabel("Virtual Memory [MiB]") + + totals = [float(x.asizeof_total) / (1024 * 1024) + for x in stats.snapshots] + axis.plot(timestamps, totals, 'r--', label='Total') + tracked = [float(x.tracked_total) / (1024 * 1024) + for x in stats.snapshots] + axis.plot(timestamps, tracked, 'b--', label='Tracked total') + + for (args, kwds) in polys: + axis.fill(*args, **kwds) + axis.legend(loc=2) # TODO fill legend + fig.savefig(filename) + +except ImportError: + def tracker_timespace(filename: str, stats: Stats) -> None: + pass diff --git a/venv/lib/python3.9/site-packages/pympler/classtracker.py b/venv/lib/python3.9/site-packages/pympler/classtracker.py new file mode 100644 index 00000000..b187e4b3 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/classtracker.py @@ -0,0 +1,590 @@ +""" +The `ClassTracker` is a facility delivering insight into the memory +distribution of a Python program. It can introspect memory consumption of +certain classes and objects. Facilities are provided to track and size +individual objects or all instances of certain classes. Tracked objects are +sized recursively to provide an overview of memory distribution between the +different tracked objects. +""" + +from typing import Any, Callable, Dict, IO, List, Optional, Tuple + +from collections import defaultdict +from functools import partial +from inspect import stack, isclass +from threading import Thread, Lock +from time import sleep, time +from weakref import ref as weakref_ref + +from pympler.classtracker_stats import ConsoleStats +from pympler.util.stringutils import safe_repr + +import pympler.asizeof as asizeof +import pympler.process + + +__all__ = ["ClassTracker"] + +# Fixpoint for program start relative time stamp. +_local_start = time() + + +class _ClassObserver(object): + """ + Stores options for tracked classes. + The observer also keeps the original constructor of the observed class. + """ + __slots__ = ('init', 'name', 'detail', 'keep', 'trace') + + def __init__(self, init: Callable, name: str, detail: int, keep: bool, + trace: bool): + self.init = init + self.name = name + self.detail = detail + self.keep = keep + self.trace = trace + + def modify(self, name: str, detail: int, keep: bool, trace: bool) -> None: + self.name = name + self.detail = detail + self.keep = keep + self.trace = trace + + +def _get_time() -> float: + """ + Get a timestamp relative to the program start time. + """ + return time() - _local_start + + +class TrackedObject(object): + """ + Stores size and lifetime information of a tracked object. A weak reference + is attached to monitor the object without preventing its deletion. + """ + __slots__ = ("ref", "id", "repr", "name", "birth", "death", "trace", + "snapshots", "_resolution_level", "__dict__") + + def __init__(self, instance: Any, name: str, resolution_level: int = 0, + trace: bool = False, on_delete: Optional[Callable] = None): + """ + Create a weak reference for 'instance' to observe an object but which + won't prevent its deletion (which is monitored by the finalize + callback). The size of the object is recorded in 'snapshots' as + (timestamp, size) tuples. + """ + self.ref = weakref_ref(instance, self.finalize) + self.id = id(instance) + self.repr = '' + self.name = name + self.birth = _get_time() + self.death = None # type: Optional[float] + self._resolution_level = resolution_level + self.trace = None # type: Optional[List[Tuple]] + + if trace: + self._save_trace() + + initial_size = asizeof.basicsize(instance) or 0 + size = asizeof.Asized(initial_size, initial_size) + self.snapshots = [(self.birth, size)] + self.on_delete = on_delete + + def __getstate__(self) -> Dict: + """ + Make the object serializable for dump_stats. Read the available slots + and store the values in a dictionary. Derived values (stored in the + dict) are not pickled as those can be reconstructed based on the other + data. References cannot be serialized, ignore 'ref' as well. + """ + state = {} + for name in getattr(TrackedObject, '__slots__', ()): + if hasattr(self, name) and name not in ['ref', '__dict__']: + state[name] = getattr(self, name) + return state + + def __setstate__(self, state: Dict) -> None: + """ + Restore the state from pickled data. Needed because a slotted class is + used. + """ + for key, value in list(state.items()): + setattr(self, key, value) + + def _save_trace(self) -> None: + """ + Save current stack trace as formatted string. + """ + stack_trace = stack() + try: + self.trace = [] + for frm in stack_trace[5:]: # eliminate our own overhead + self.trace.insert(0, frm[1:]) + finally: + del stack_trace + + def track_size(self, ts: float, sizer: asizeof.Asizer) -> None: + """ + Store timestamp and current size for later evaluation. + The 'sizer' is a stateful sizing facility that excludes other tracked + objects. + """ + obj = self.ref() + self.snapshots.append( + (ts, sizer.asized(obj, detail=self._resolution_level)) + ) + if obj is not None: + self.repr = safe_repr(obj, clip=128) + + def get_max_size(self) -> int: + """ + Get the maximum of all sampled sizes. + """ + return max([s.size for (_, s) in self.snapshots]) + + def get_size_at_time(self, timestamp: float) -> int: + """ + Get the size of the object at a specific time (snapshot). + If the object was not alive/sized at that instant, return 0. + """ + size = 0 + for (t, s) in self.snapshots: + if t == timestamp: + size = s.size + return size + + def set_resolution_level(self, resolution_level: int) -> None: + """ + Set resolution level to a new value. The next size estimation will + respect the new value. This is useful to set different levels for + different instances of tracked classes. + """ + self._resolution_level = resolution_level + + def finalize(self, ref: weakref_ref) -> None: + """ + Mark the reference as dead and remember the timestamp. It would be + great if we could measure the pre-destruction size. Unfortunately, the + object is gone by the time the weakref callback is called. However, + weakref callbacks are useful to be informed when tracked objects died + without the need of destructors. + + If the object is destroyed at the end of the program execution, it's + not possible to import modules anymore. Hence, the finalize callback + just does nothing (self.death stays None). + """ + try: + self.death = _get_time() + if self.on_delete: + self.on_delete() + except Exception: # pragma: no cover + pass + + +def track_object_creation(time_series: List[Tuple[float, int]]) -> None: + num_instances = time_series[-1][1] if time_series else 0 + time_series.append((_get_time(), num_instances+1)) + + +def track_object_deletion(time_series: List[Tuple[float, int]]) -> None: + num_instances = time_series[-1][1] + time_series.append((_get_time(), num_instances-1)) + + +class PeriodicThread(Thread): + """ + Thread object to take snapshots periodically. + """ + + def __init__(self, tracker: 'ClassTracker', interval: float, *args: Any, + **kwargs: Any): + """ + Create thread with given interval and associated with the given + tracker. + """ + self.interval = interval + self.tracker = tracker + self.stop = False + super(PeriodicThread, self).__init__(*args, **kwargs) + + def run(self) -> None: + """ + Loop until a stop signal is set. + """ + self.stop = False + while not self.stop: + self.tracker.create_snapshot() + sleep(self.interval) + + +class Snapshot(object): + """Sample sizes of objects and the process at an instant.""" + + def __init__(self, timestamp: float, description: str = '') -> None: + """Initialize process-wide size information.""" + self.tracked_total = 0 + self.asizeof_total = 0 + self.overhead = 0 + self.timestamp = timestamp + self.system_total = pympler.process.ProcessMemoryInfo() + self.desc = description + self.classes = None # type: Optional[Dict[str, Dict[str, Any]]] + + @property + def total(self) -> int: + """ + Return the total (virtual) size of the process in bytes. If process + information is not available, get the best number available, even if it + is a poor approximation of reality. + """ + if self.system_total.available: + return self.system_total.vsz + elif self.asizeof_total: # pragma: no cover + return self.asizeof_total + else: # pragma: no cover + return self.tracked_total + + @property + def label(self) -> str: + """Return timestamped label for this snapshot, or a raw timestamp.""" + if not self.desc: + return "%.3fs" % self.timestamp + return "%s (%.3fs)" % (self.desc, self.timestamp) + + +class ClassTracker(object): + + def __init__(self, stream: Optional[IO] = None): + """ + Creates a new `ClassTracker` object. + + :param stream: Output stream to use when printing statistics via + ``stats``. + """ + # Dictionaries of TrackedObject objects associated with the actual + # objects that are tracked. 'index' uses the class name as the key and + # associates a list of tracked objects. It contains all TrackedObject + # instances, including those of dead objects. + self.index = defaultdict(list) # type: Dict[str, List[TrackedObject]] + + # 'objects' uses the id (address) as the key and associates the tracked + # object with it. TrackedObject's referring to dead objects are + # replaced lazily, i.e. when the id is recycled by another tracked + # object. + self.objects = {} # type: Dict[int, Any] + + # List of `Snapshot` objects. + self.snapshots = [] # type: List[Snapshot] + + # Time series of instance count for each tracked class. + self.history = defaultdict(list) \ + # type: Dict[str, List[Tuple[float, int]]] + + # Keep objects alive by holding a strong reference. + self._keepalive = [] # type: List[Any] + + # Dictionary of class observers identified by classname. + self._observers = {} # type: Dict[type, _ClassObserver] + + # Thread object responsible for background monitoring + self._periodic_thread = None # type: Optional[PeriodicThread] + + self._stream = stream + + @property + def stats(self) -> ConsoleStats: + """ + Return a ``ConsoleStats`` instance initialized with the current state + of the class tracker. + """ + return ConsoleStats(tracker=self, stream=self._stream) + + def _tracker(self, _observer_: _ClassObserver, _self_: Any, *args: Any, + **kwds: Any) -> None: + """ + Injected constructor for tracked classes. + Call the actual constructor of the object and track the object. Attach + to the object before calling the constructor to track the object with + the parameters of the most specialized class. + """ + self.track_object(_self_, + name=_observer_.name, + resolution_level=_observer_.detail, + keep=_observer_.keep, + trace=_observer_.trace) + _observer_.init(_self_, *args, **kwds) + + def _inject_constructor(self, cls: type, func: Callable, name: str, + resolution_level: int, keep: bool, trace: bool, + ) -> None: + """ + Modifying Methods in Place - after the recipe 15.7 in the Python + Cookbook by Ken Seehof. The original constructors may be restored + later. + """ + try: + constructor = cls.__init__ # type: ignore + except AttributeError: + def constructor(self: Any, *_args: Any, **_kwargs: Any) -> None: + pass + + # Possible name clash between keyword arguments of the tracked class' + # constructor and the curried arguments of the injected constructor. + # Therefore, the additional argument has a 'magic' name to make it less + # likely that an argument name clash occurs. + observer = _ClassObserver(constructor, + name, + resolution_level, + keep, + trace) + self._observers[cls] = observer + + def new_constructor(*args: Any, **kwargs: Any) -> None: + return func(observer, *args, **kwargs) + + cls.__init__ = new_constructor # type: ignore + + def _is_tracked(self, cls: type) -> bool: + """ + Determine if the class is tracked. + """ + return cls in self._observers + + def _track_modify(self, cls: type, name: str, detail: int, keep: bool, + trace: bool) -> None: + """ + Modify settings of a tracked class + """ + self._observers[cls].modify(name, detail, keep, trace) + + def _restore_constructor(self, cls: type) -> None: + """ + Restore the original constructor, lose track of class. + """ + cls.__init__ = self._observers[cls].init # type: ignore + del self._observers[cls] + + def track_change(self, instance: Any, resolution_level: int = 0) -> None: + """ + Change tracking options for the already tracked object 'instance'. + If instance is not tracked, a KeyError will be raised. + """ + tobj = self.objects[id(instance)] + tobj.set_resolution_level(resolution_level) + + def track_object(self, instance: Any, name: Optional[str] = None, + resolution_level: int = 0, keep: bool = False, + trace: bool = False) -> None: + """ + Track object 'instance' and sample size and lifetime information. Not + all objects can be tracked; trackable objects are class instances and + other objects that can be weakly referenced. When an object cannot be + tracked, a `TypeError` is raised. + + :param resolution_level: The recursion depth up to which referents are + sized individually. Resolution level 0 (default) treats the object + as an opaque entity, 1 sizes all direct referents individually, 2 + also sizes the referents of the referents and so forth. + :param keep: Prevent the object's deletion by keeping a (strong) + reference to the object. + """ + + # Check if object is already tracked. This happens if track_object is + # called multiple times for the same object or if an object inherits + # from multiple tracked classes. In the latter case, the most + # specialized class wins. To detect id recycling, the weak reference + # is checked. If it is 'None' a tracked object is dead and another one + # takes the same 'id'. + if id(instance) in self.objects and \ + self.objects[id(instance)].ref() is not None: + return + + name = name if name else instance.__class__.__name__ + + track_object_creation(self.history[name]) + on_delete = partial(track_object_deletion, self.history[name]) + + tobj = TrackedObject(instance, + name, + resolution_level=resolution_level, + trace=trace, + on_delete=on_delete) + + self.index[name].append(tobj) + self.objects[id(instance)] = tobj + + if keep: + self._keepalive.append(instance) + + def track_class(self, cls: type, name: Optional[str] = None, + resolution_level: int = 0, keep: bool = False, + trace: bool = False) -> None: + """ + Track all objects of the class `cls`. Objects of that type that already + exist are *not* tracked. If `track_class` is called for a class already + tracked, the tracking parameters are modified. Instantiation traces can + be generated by setting `trace` to True. + A constructor is injected to begin instance tracking on creation + of the object. The constructor calls `track_object` internally. + + :param cls: class to be tracked, may be an old-style or a new-style + class + :param name: reference the class by a name, default is the + concatenation of module and class name + :param resolution_level: The recursion depth up to which referents are + sized individually. Resolution level 0 (default) treats the object + as an opaque entity, 1 sizes all direct referents individually, 2 + also sizes the referents of the referents and so forth. + :param keep: Prevent the object's deletion by keeping a (strong) + reference to the object. + :param trace: Save instantiation stack trace for each instance + """ + if not isclass(cls): + raise TypeError("only class objects can be tracked") + if name is None: + name = cls.__module__ + '.' + cls.__name__ + if self._is_tracked(cls): + self._track_modify(cls, name, resolution_level, keep, trace) + else: + self._inject_constructor(cls, self._tracker, name, + resolution_level, keep, trace) + + def detach_class(self, cls: type) -> None: + """ + Stop tracking class 'cls'. Any new objects of that type are not + tracked anymore. Existing objects are still tracked. + """ + self._restore_constructor(cls) + + def detach_all_classes(self) -> None: + """ + Detach from all tracked classes. + """ + classes = list(self._observers.keys()) + for cls in classes: + self.detach_class(cls) + + def detach_all(self) -> None: + """ + Detach from all tracked classes and objects. + Restore the original constructors and cleanse the tracking lists. + """ + self.detach_all_classes() + self.objects.clear() + self.index.clear() + self._keepalive[:] = [] + + def clear(self) -> None: + """ + Clear all gathered data and detach from all tracked objects/classes. + """ + self.detach_all() + self.snapshots[:] = [] + + def close(self) -> None: + """ + Detach from tracked classes by removing injected constructors. Makes it + possible to use ClassTracker in `contextlib.closing` to safely remove + profiling hooks when the tracker goes out of scope:: + + import contextlib + with contextlib.closing(ClassTracker()) as tracker: + tracker.track_class(Foo) + + """ + self.detach_all_classes() + +# +# Background Monitoring +# + + def start_periodic_snapshots(self, interval: float = 1.0) -> None: + """ + Start a thread which takes snapshots periodically. The `interval` + specifies the time in seconds the thread waits between taking + snapshots. The thread is started as a daemon allowing the program to + exit. If periodic snapshots are already active, the interval is + updated. + """ + if not self._periodic_thread: + self._periodic_thread = PeriodicThread(self, interval, + name='BackgroundMonitor') + self._periodic_thread.setDaemon(True) + self._periodic_thread.start() + else: + self._periodic_thread.interval = interval + + def stop_periodic_snapshots(self) -> None: + """ + Post a stop signal to the thread that takes the periodic snapshots. The + function waits for the thread to terminate which can take some time + depending on the configured interval. + """ + if self._periodic_thread and self._periodic_thread.is_alive(): + self._periodic_thread.stop = True + self._periodic_thread.join() + self._periodic_thread = None + +# +# Snapshots +# + + snapshot_lock = Lock() + + def create_snapshot(self, description: str = '', + compute_total: bool = False) -> None: + """ + Collect current per instance statistics and saves total amount of + memory associated with the Python process. + + If `compute_total` is `True`, the total consumption of all objects + known to *asizeof* is computed. The latter might be very slow if many + objects are mapped into memory at the time the snapshot is taken. + Therefore, `compute_total` is set to `False` by default. + + The overhead of the `ClassTracker` structure is also computed. + + Snapshots can be taken asynchronously. The function is protected with a + lock to prevent race conditions. + """ + + try: + # TODO: It is not clear what happens when memory is allocated or + # released while this function is executed but it will likely lead + # to inconsistencies. Either pause all other threads or don't size + # individual objects in asynchronous mode. + self.snapshot_lock.acquire() + + timestamp = _get_time() + + sizer = asizeof.Asizer() + objs = [tobj.ref() for tobj in list(self.objects.values())] + sizer.exclude_refs(*objs) + + # The objects need to be sized in a deterministic order. Sort the + # objects by its creation date which should at least work for + # non-parallel execution. The "proper" fix would be to handle + # shared data separately. + tracked_objects = list(self.objects.values()) + tracked_objects.sort(key=lambda x: x.birth) + for tobj in tracked_objects: + tobj.track_size(timestamp, sizer) + + snapshot = Snapshot(timestamp, str(description)) + snapshot.tracked_total = sizer.total + if compute_total: + snapshot.asizeof_total = asizeof.asizeof(all=True, code=True) + + # Compute overhead of all structures, use sizer to exclude tracked + # objects(!) + snapshot.overhead = 0 + if snapshot.tracked_total: + snapshot.overhead = sizer.asizeof(self) + if snapshot.asizeof_total: + snapshot.asizeof_total -= snapshot.overhead + + self.snapshots.append(snapshot) + + finally: + self.snapshot_lock.release() diff --git a/venv/lib/python3.9/site-packages/pympler/classtracker_stats.py b/venv/lib/python3.9/site-packages/pympler/classtracker_stats.py new file mode 100644 index 00000000..191f1fba --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/classtracker_stats.py @@ -0,0 +1,780 @@ +""" +Provide saving, loading and presenting gathered `ClassTracker` statistics. +""" + +from typing import ( + Any, Dict, IO, Iterable, List, Optional, Tuple, TYPE_CHECKING, Union +) + +import os +import pickle +import sys +from copy import deepcopy +from pympler.util.stringutils import trunc, pp, pp_timestamp + +from pympler.asizeof import Asized + +if TYPE_CHECKING: + from .classtracker import TrackedObject, ClassTracker, Snapshot + + +__all__ = ["Stats", "ConsoleStats", "HtmlStats"] + + +def _ref2key(ref: Asized) -> str: + return ref.name.split(':')[0] + + +def _merge_asized(base: Asized, other: Asized, level: int = 0) -> None: + """ + Merge **Asized** instances `base` and `other` into `base`. + """ + base.size += other.size + base.flat += other.flat + if level > 0: + base.name = _ref2key(base) + # Add refs from other to base. Any new refs are appended. + base.refs = list(base.refs) # we may need to append items + refs = {} + for ref in base.refs: + refs[_ref2key(ref)] = ref + for ref in other.refs: + key = _ref2key(ref) + if key in refs: + _merge_asized(refs[key], ref, level=level + 1) + else: + # Don't modify existing Asized instances => deepcopy + base.refs.append(deepcopy(ref)) + base.refs[-1].name = key + + +def _merge_objects(tref: float, merged: Asized, obj: 'TrackedObject') -> None: + """ + Merge the snapshot size information of multiple tracked objects. The + tracked object `obj` is scanned for size information at time `tref`. + The sizes are merged into **Asized** instance `merged`. + """ + size = None + for (timestamp, tsize) in obj.snapshots: + if timestamp == tref: + size = tsize + if size: + _merge_asized(merged, size) + + +def _format_trace(trace: List[Tuple]) -> str: + """ + Convert the (stripped) stack-trace to a nice readable format. The stack + trace `trace` is a list of frame records as returned by + **inspect.stack** but without the frame objects. + Returns a string. + """ + lines = [] + for fname, lineno, func, src, _ in trace: + if src: + for line in src: + lines.append(' ' + line.strip() + '\n') + lines.append(' %s:%4d in %s\n' % (fname, lineno, func)) + return ''.join(lines) + + +class Stats(object): + """ + Presents the memory statistics gathered by a `ClassTracker` based on user + preferences. + """ + + def __init__(self, tracker: 'Optional[ClassTracker]' = None, + filename: Optional[str] = None, + stream: Optional[IO] = None): + """ + Initialize the data log structures either from a `ClassTracker` + instance (argument `tracker`) or a previously dumped file (argument + `filename`). + + :param tracker: ClassTracker instance + :param filename: filename of previously dumped statistics + :param stream: where to print statistics, defaults to ``sys.stdout`` + """ + if stream: + self.stream = stream + else: + self.stream = sys.stdout + self.tracker = tracker + self.index = {} # type: Dict[str, List[TrackedObject]] + self.snapshots = [] # type: List[Snapshot] + if tracker: + self.index = tracker.index + self.snapshots = tracker.snapshots + self.history = tracker.history + self.sorted = [] # type: List[TrackedObject] + if filename: + self.load_stats(filename) + + def load_stats(self, fdump: Union[str, IO[bytes]]) -> None: + """ + Load the data from a dump file. + The argument `fdump` can be either a filename or an open file object + that requires read access. + """ + if isinstance(fdump, str): + fdump = open(fdump, 'rb') + self.index = pickle.load(fdump) + self.snapshots = pickle.load(fdump) + self.sorted = [] + + def dump_stats(self, fdump: Union[str, IO[bytes]], close: bool = True + ) -> None: + """ + Dump the logged data to a file. + The argument `file` can be either a filename or an open file object + that requires write access. `close` controls if the file is closed + before leaving this method (the default behaviour). + """ + if self.tracker: + self.tracker.stop_periodic_snapshots() + + if isinstance(fdump, str): + fdump = open(fdump, 'wb') + pickle.dump(self.index, fdump, protocol=pickle.HIGHEST_PROTOCOL) + pickle.dump(self.snapshots, fdump, protocol=pickle.HIGHEST_PROTOCOL) + if close: + fdump.close() + + def _init_sort(self) -> None: + """ + Prepare the data to be sorted. + If not yet sorted, import all tracked objects from the tracked index. + Extend the tracking information by implicit information to make + sorting easier (DSU pattern). + """ + if not self.sorted: + # Identify the snapshot that tracked the largest amount of memory. + tmax = None + maxsize = 0 + for snapshot in self.snapshots: + if snapshot.tracked_total > maxsize: + tmax = snapshot.timestamp + for key in list(self.index.keys()): + for tobj in self.index[key]: + tobj.classname = key # type: ignore + tobj.size = tobj.get_max_size() # type: ignore + tobj.tsize = tobj.get_size_at_time(tmax) # type: ignore + self.sorted.extend(self.index[key]) + + def sort_stats(self, *args: str) -> 'Stats': + """ + Sort the tracked objects according to the supplied criteria. The + argument is a string identifying the basis of a sort (example: 'size' + or 'classname'). When more than one key is provided, then additional + keys are used as secondary criteria when there is equality in all keys + selected before them. For example, ``sort_stats('name', 'size')`` will + sort all the entries according to their class name, and resolve all + ties (identical class names) by sorting by size. The criteria are + fields in the tracked object instances. Results are stored in the + ``self.sorted`` list which is used by ``Stats.print_stats()`` and other + methods. The fields available for sorting are: + + 'classname' + the name with which the class was registered + 'name' + the classname + 'birth' + creation timestamp + 'death' + destruction timestamp + 'size' + the maximum measured size of the object + 'tsize' + the measured size during the largest snapshot + 'repr' + string representation of the object + + Note that sorts on size are in descending order (placing most memory + consuming items first), whereas name, repr, and creation time searches + are in ascending order (alphabetical). + + The function returns self to allow calling functions on the result:: + + stats.sort_stats('size').reverse_order().print_stats() + """ + + criteria = ('classname', 'tsize', 'birth', 'death', + 'name', 'repr', 'size') + + if not set(criteria).issuperset(set(args)): + raise ValueError("Invalid sort criteria") + + if not args: + args = criteria + + def args_to_tuple(obj: 'TrackedObject') -> Tuple[str, ...]: + keys: List[str] = [] + for attr in args: + attribute = getattr(obj, attr, '') + if attr in ('tsize', 'size'): + attribute = -int(attribute) + keys.append(attribute) + return tuple(keys) + + self._init_sort() + self.sorted.sort(key=args_to_tuple) + + return self + + def reverse_order(self) -> 'Stats': + """ + Reverse the order of the tracked instance index `self.sorted`. + """ + self._init_sort() + self.sorted.reverse() + return self + + def annotate(self) -> None: + """ + Annotate all snapshots with class-based summaries. + """ + for snapshot in self.snapshots: + self.annotate_snapshot(snapshot) + + def annotate_snapshot(self, snapshot: 'Snapshot' + ) -> Dict[str, Dict[str, Any]]: + """ + Store additional statistical data in snapshot. + """ + if snapshot.classes is not None: + return snapshot.classes + + snapshot.classes = {} + + for classname in list(self.index.keys()): + total = 0 + active = 0 + merged = Asized(0, 0) + for tobj in self.index[classname]: + _merge_objects(snapshot.timestamp, merged, tobj) + total += tobj.get_size_at_time(snapshot.timestamp) + if (tobj.birth < snapshot.timestamp and + (tobj.death is None or + tobj.death > snapshot.timestamp)): + active += 1 + + try: + pct = total * 100.0 / snapshot.total + except ZeroDivisionError: # pragma: no cover + pct = 0 + try: + avg = total / active + except ZeroDivisionError: + avg = 0 + + snapshot.classes[classname] = dict(sum=total, + avg=avg, + pct=pct, + active=active) + snapshot.classes[classname]['merged'] = merged + + return snapshot.classes + + @property + def tracked_classes(self) -> List[str]: + """Return a list of all tracked classes occurring in any snapshot.""" + return sorted(list(self.index.keys())) + + +class ConsoleStats(Stats): + """ + Presentation layer for `Stats` to be used in text-based consoles. + """ + + def _print_refs(self, refs: Iterable[Asized], total: int, + prefix: str = ' ', level: int = 1, minsize: int = 0, + minpct: float = 0.1) -> None: + """ + Print individual referents recursively. + """ + lrefs = list(refs) + lrefs.sort(key=lambda x: x.size) + lrefs.reverse() + for ref in lrefs: + if ref.size > minsize and (ref.size * 100.0 / total) > minpct: + self.stream.write('%-50s %-14s %3d%% [%d]\n' % ( + trunc(prefix + str(ref.name), 50), + pp(ref.size), + int(ref.size * 100.0 / total), + level + )) + self._print_refs(ref.refs, total, prefix=prefix + ' ', + level=level + 1) + + def print_object(self, tobj: 'TrackedObject') -> None: + """ + Print the gathered information of object `tobj` in human-readable + format. + """ + if tobj.death: + self.stream.write('%-32s ( free ) %-35s\n' % ( + trunc(tobj.name, 32, left=True), trunc(tobj.repr, 35))) + else: + self.stream.write('%-32s 0x%08x %-35s\n' % ( + trunc(tobj.name, 32, left=True), + + tobj.id, + trunc(tobj.repr, 35) + )) + if tobj.trace: + self.stream.write(_format_trace(tobj.trace)) + for (timestamp, size) in tobj.snapshots: + self.stream.write(' %-30s %s\n' % ( + pp_timestamp(timestamp), pp(size.size) + )) + self._print_refs(size.refs, size.size) + if tobj.death is not None: + self.stream.write(' %-30s finalize\n' % ( + pp_timestamp(tobj.death), + )) + + def print_stats(self, clsname: Optional[str] = None, limit: float = 1.0 + ) -> None: + """ + Write tracked objects to stdout. The output can be filtered and + pruned. Only objects are printed whose classname contain the substring + supplied by the `clsname` argument. The output can be pruned by + passing a `limit` value. + + :param clsname: Only print objects whose classname contain the given + substring. + :param limit: If `limit` is a float smaller than one, only the supplied + percentage of the total tracked data is printed. If `limit` is + bigger than one, this number of tracked objects are printed. + Tracked objects are first filtered, and then pruned (if specified). + """ + if self.tracker: + self.tracker.stop_periodic_snapshots() + + if not self.sorted: + self.sort_stats() + + _sorted = self.sorted + + if clsname: + _sorted = [ + to for to in _sorted + if clsname in to.classname # type: ignore + ] + + if limit < 1.0: + limit = max(1, int(len(self.sorted) * limit)) + _sorted = _sorted[:int(limit)] + + # Emit per-instance data + for tobj in _sorted: + self.print_object(tobj) + + def print_summary(self) -> None: + """ + Print per-class summary for each snapshot. + """ + # Emit class summaries for each snapshot + classlist = self.tracked_classes + + fobj = self.stream + + fobj.write('---- SUMMARY ' + '-' * 66 + '\n') + for snapshot in self.snapshots: + classes = self.annotate_snapshot(snapshot) + fobj.write('%-35s %11s %12s %12s %5s\n' % ( + trunc(snapshot.desc, 35), + 'active', + pp(snapshot.asizeof_total), + 'average', + 'pct' + )) + for classname in classlist: + info = classes[classname] + fobj.write(' %-33s %11d %12s %12s %4d%%\n' % ( + trunc(classname, 33), + info['active'], + pp(info['sum']), + pp(info['avg']), + info['pct'] + )) + fobj.write('-' * 79 + '\n') + + +class HtmlStats(Stats): + """ + Output the `ClassTracker` statistics as HTML pages and graphs. + """ + + style = """ + """ + + nopylab_msg = """
Could not generate %s chart! + Install Matplotlib + to generate charts.
\n""" + + chart_tag = '\n' + header = "%s%s\n" + tableheader = '\n' + tablefooter = '
\n' + footer = '\n' + + refrow = """ + %(name)s + %(size)s + %(pct)3.1f%%""" + + def _print_refs(self, fobj: IO, refs: Iterable[Asized], total: int, + level: int = 1, minsize: int = 0, minpct: float = 0.1 + ) -> None: + """ + Print individual referents recursively. + """ + lrefs = list(refs) + lrefs.sort(key=lambda x: x.size) + lrefs.reverse() + if level == 1: + fobj.write('\n') + for ref in lrefs: + if ref.size > minsize and (ref.size * 100.0 / total) > minpct: + data = dict(level=level, + name=trunc(str(ref.name), 128), + size=pp(ref.size), + pct=ref.size * 100.0 / total) + fobj.write(self.refrow % data) + self._print_refs(fobj, ref.refs, total, level=level + 1) + if level == 1: + fobj.write("
\n") + + class_summary = """

%(cnt)d instances of %(cls)s were registered. The + average size is %(avg)s, the minimal size is %(min)s, the maximum size + is %(max)s.

\n""" + class_snapshot = '''

Snapshot: %(name)s, %(total)s occupied by instances + of class %(cls)s

\n''' + + def print_class_details(self, fname: str, classname: str) -> None: + """ + Print detailed statistics and instances for the class `classname`. All + data will be written to the file `fname`. + """ + fobj = open(fname, "w") + fobj.write(self.header % (classname, self.style)) + + fobj.write("

%s

\n" % (classname)) + + sizes = [tobj.get_max_size() for tobj in self.index[classname]] + total = 0 + for s in sizes: + total += s + data = {'cnt': len(self.index[classname]), 'cls': classname} + data['avg'] = pp(total / len(sizes)) + data['max'] = pp(max(sizes)) + data['min'] = pp(min(sizes)) + fobj.write(self.class_summary % data) + + fobj.write(self.charts[classname]) + + fobj.write("

Coalesced Referents per Snapshot

\n") + for snapshot in self.snapshots: + if snapshot.classes and classname in snapshot.classes: + merged = snapshot.classes[classname]['merged'] + fobj.write(self.class_snapshot % { + 'name': snapshot.desc, + 'cls': classname, + 'total': pp(merged.size), + }) + if merged.refs: + self._print_refs(fobj, merged.refs, merged.size) + else: + fobj.write('

No per-referent sizes recorded.

\n') + + fobj.write("

Instances

\n") + for tobj in self.index[classname]: + fobj.write('\n') + fobj.write('' + + '\n' % + (tobj.name, tobj.id)) + if tobj.repr: + fobj.write("" + + "\n" % tobj.repr) + fobj.write("\n" % + (pp_timestamp(tobj.birth), pp_timestamp(tobj.death))) + if tobj.trace: + trace = "
%s
" % (_format_trace(tobj.trace)) + fobj.write("\n" % + trace) + for (timestamp, size) in tobj.snapshots: + fobj.write("" % pp_timestamp(timestamp)) + if not size.refs: + fobj.write("\n" % pp(size.size)) + else: + fobj.write("\n") + fobj.write("
Instance%s at 0x%08x
Representation%s 
Lifetime%s - %s
Instantiation%s
%s%s
%s" % pp(size.size)) + self._print_refs(fobj, size.refs, size.size) + fobj.write("
\n") + + fobj.write(self.footer) + fobj.close() + + snapshot_cls_header = """ + Class + Instance # + Total + Average size + Share\n""" + + snapshot_cls = """ + %(cls)s + %(active)d + %(sum)s + %(avg)s + %(pct)3.2f%%\n""" + + snapshot_summary = """

Total virtual memory assigned to the program + at that time was %(sys)s, which includes %(overhead)s profiling + overhead. The ClassTracker tracked %(tracked)s in total. The measurable + objects including code objects but excluding overhead have a total size + of %(asizeof)s.

\n""" + + def relative_path(self, filepath: str, basepath: Optional[str] = None + ) -> str: + """ + Convert the filepath path to a relative path against basepath. By + default basepath is self.basedir. + """ + if basepath is None: + basepath = self.basedir + if not basepath: + return filepath + if filepath.startswith(basepath): + filepath = filepath[len(basepath):] + if filepath and filepath[0] == os.sep: + filepath = filepath[1:] + return filepath + + def create_title_page(self, filename: str, title: str = '') -> None: + """ + Output the title page. + """ + fobj = open(filename, "w") + fobj.write(self.header % (title, self.style)) + + fobj.write("

%s

\n" % title) + fobj.write("

Memory distribution over time

\n") + fobj.write(self.charts['snapshots']) + + fobj.write("

Snapshots statistics

\n") + fobj.write('\n') + + classlist = list(self.index.keys()) + classlist.sort() + + for snapshot in self.snapshots: + fobj.write('\n') + + fobj.write("
\n') + fobj.write('\n') + fobj.write("

%s snapshot at %s

\n" % ( + snapshot.desc or 'Untitled', + pp_timestamp(snapshot.timestamp) + )) + + data = {} + data['sys'] = pp(snapshot.system_total.vsz) + data['tracked'] = pp(snapshot.tracked_total) + data['asizeof'] = pp(snapshot.asizeof_total) + data['overhead'] = pp(getattr(snapshot, 'overhead', 0)) + + fobj.write(self.snapshot_summary % data) + + if snapshot.tracked_total: + fobj.write(self.snapshot_cls_header) + for classname in classlist: + if snapshot.classes: + info = snapshot.classes[classname].copy() + path = self.relative_path(self.links[classname]) + info['cls'] = '%s' % (path, classname) + info['sum'] = pp(info['sum']) + info['avg'] = pp(info['avg']) + fobj.write(self.snapshot_cls % info) + fobj.write('
') + fobj.write('		\n') + if snapshot.tracked_total: + fobj.write(self.charts[snapshot]) + fobj.write('
\n") + fobj.write(self.footer) + fobj.close() + + def create_lifetime_chart(self, classname: str, filename: str = '') -> str: + """ + Create chart that depicts the lifetime of the instance registered with + `classname`. The output is written to `filename`. + """ + try: + from pylab import figure, title, xlabel, ylabel, plot, savefig + except ImportError: + return HtmlStats.nopylab_msg % (classname + " lifetime") + + cnt = [] + for tobj in self.index[classname]: + cnt.append([tobj.birth, 1]) + if tobj.death: + cnt.append([tobj.death, -1]) + cnt.sort() + for i in range(1, len(cnt)): + cnt[i][1] += cnt[i - 1][1] + + x = [t for [t, c] in cnt] + y = [c for [t, c] in cnt] + + figure() + xlabel("Execution time [s]") + ylabel("Instance #") + title("%s instances" % classname) + plot(x, y, 'o') + savefig(filename) + + return self.chart_tag % (os.path.basename(filename)) + + def create_snapshot_chart(self, filename: str = '') -> str: + """ + Create chart that depicts the memory allocation over time apportioned + to the tracked classes. + """ + try: + from pylab import (figure, title, xlabel, ylabel, plot, fill, + legend, savefig) + import matplotlib.mlab as mlab + except ImportError: + return self.nopylab_msg % ("memory allocation") + + classlist = self.tracked_classes + + times = [snapshot.timestamp for snapshot in self.snapshots] + base = [0.0] * len(self.snapshots) + poly_labels = [] + polys = [] + for cn in classlist: + pct = [snapshot.classes[cn]['pct'] for snapshot in self.snapshots + if snapshot.classes is not None] + if pct and max(pct) > 3.0: + sz = [float(fp.classes[cn]['sum']) / (1024 * 1024) + for fp in self.snapshots + if fp.classes is not None] + sz = [sx + sy for sx, sy in zip(base, sz)] + xp, yp = mlab.poly_between(times, base, sz) + polys.append(((xp, yp), {'label': cn})) + poly_labels.append(cn) + base = sz + + figure() + title("Snapshot Memory") + xlabel("Execution Time [s]") + ylabel("Virtual Memory [MiB]") + + sizes = [float(fp.asizeof_total) / (1024 * 1024) + for fp in self.snapshots] + plot(times, sizes, 'r--', label='Total') + sizes = [float(fp.tracked_total) / (1024 * 1024) + for fp in self.snapshots] + plot(times, sizes, 'b--', label='Tracked total') + + for (args, kwds) in polys: + fill(*args, **kwds) + legend(loc=2) + savefig(filename) + + return self.chart_tag % (self.relative_path(filename)) + + def create_pie_chart(self, snapshot: 'Snapshot', filename: str = '') -> str: + """ + Create a pie chart that depicts the distribution of the allocated + memory for a given `snapshot`. The chart is saved to `filename`. + """ + try: + from pylab import figure, title, pie, axes, savefig + from pylab import sum as pylab_sum + except ImportError: + return self.nopylab_msg % ("pie_chart") + + # Don't bother illustrating a pie without pieces. + if not snapshot.tracked_total or snapshot.classes is None: + return '' + + classlist = [] + sizelist = [] + for k, v in list(snapshot.classes.items()): + if v['pct'] > 3.0: + classlist.append(k) + sizelist.append(v['sum']) + sizelist.insert(0, snapshot.asizeof_total - pylab_sum(sizelist)) + classlist.insert(0, 'Other') + + title("Snapshot (%s) Memory Distribution" % (snapshot.desc)) + figure(figsize=(8, 8)) + axes([0.1, 0.1, 0.8, 0.8]) + pie(sizelist, labels=classlist) + savefig(filename, dpi=50) + + return self.chart_tag % (self.relative_path(filename)) + + def create_html(self, fname: str, title: str = "ClassTracker Statistics" + ) -> None: + """ + Create HTML page `fname` and additional files in a directory derived + from `fname`. + """ + # Create a folder to store the charts and additional HTML files. + self.basedir = os.path.dirname(os.path.abspath(fname)) + self.filesdir = os.path.splitext(fname)[0] + '_files' + if not os.path.isdir(self.filesdir): + os.mkdir(self.filesdir) + self.filesdir = os.path.abspath(self.filesdir) + self.links = {} # type: Dict[str, str] + + # Annotate all snapshots in advance + self.annotate() + + # Create charts. The tags to show the images are returned and stored in + # the self.charts dictionary. This allows to return alternative text if + # the chart creation framework is not available. + self.charts = {} # type: Dict[Union[str, Snapshot], str] + fn = os.path.join(self.filesdir, 'timespace.png') + self.charts['snapshots'] = self.create_snapshot_chart(fn) + + for fp, idx in zip(self.snapshots, list(range(len(self.snapshots)))): + fn = os.path.join(self.filesdir, 'fp%d.png' % (idx)) + self.charts[fp] = self.create_pie_chart(fp, fn) + + for cn in list(self.index.keys()): + fn = os.path.join(self.filesdir, cn.replace('.', '_') + '-lt.png') + self.charts[cn] = self.create_lifetime_chart(cn, fn) + + # Create HTML pages first for each class and then the index page. + for cn in list(self.index.keys()): + fn = os.path.join(self.filesdir, cn.replace('.', '_') + '.html') + self.links[cn] = fn + self.print_class_details(fn, cn) + + self.create_title_page(fname, title=title) diff --git a/venv/lib/python3.9/site-packages/pympler/garbagegraph.py b/venv/lib/python3.9/site-packages/pympler/garbagegraph.py new file mode 100644 index 00000000..fdd6cb74 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/garbagegraph.py @@ -0,0 +1,80 @@ + +from pympler.refgraph import ReferenceGraph +from pympler.util.stringutils import trunc, pp + +import sys +import gc + +__all__ = ['GarbageGraph', 'start_debug_garbage', 'end_debug_garbage'] + + +class GarbageGraph(ReferenceGraph): + """ + The ``GarbageGraph`` is a ``ReferenceGraph`` that illustrates the objects + building reference cycles. The garbage collector is switched to debug mode + (all identified garbage is stored in `gc.garbage`) and the garbage + collector is invoked. The collected objects are then illustrated in a + directed graph. + + Large graphs can be reduced to the actual cycles by passing ``reduce=True`` + to the constructor. + + It is recommended to disable the garbage collector when using the + ``GarbageGraph``. + + >>> from pympler.garbagegraph import GarbageGraph, start_debug_garbage + >>> start_debug_garbage() + >>> l = [] + >>> l.append(l) + >>> del l + >>> gb = GarbageGraph() + >>> gb.render('garbage.eps') + True + """ + def __init__(self, reduce=False, collectable=True): + """ + Initialize the GarbageGraph with the objects identified by the garbage + collector. If `collectable` is true, every reference cycle is recorded. + Otherwise only uncollectable objects are reported. + """ + if collectable: + gc.set_debug(gc.DEBUG_SAVEALL) + else: + gc.set_debug(0) + gc.collect() + + ReferenceGraph.__init__(self, gc.garbage, reduce) + + def print_stats(self, stream=None): + """ + Log annotated garbage objects to console or file. + + :param stream: open file, uses sys.stdout if not given + """ + if not stream: # pragma: no cover + stream = sys.stdout + self.metadata.sort(key=lambda x: -x.size) + stream.write('%-10s %8s %-12s %-46s\n' % ('id', 'size', 'type', + 'representation')) + for g in self.metadata: + stream.write('0x%08x %8d %-12s %-46s\n' % (g.id, g.size, + trunc(g.type, 12), + trunc(g.str, 46))) + stream.write('Garbage: %8d collected objects (%s in cycles): %12s\n' % + (self.count, self.num_in_cycles, pp(self.total_size))) + + +def start_debug_garbage(): + """ + Turn off garbage collector to analyze *collectable* reference cycles. + """ + gc.collect() + gc.disable() + + +def end_debug_garbage(): + """ + Turn garbage collection on and disable debug output. + """ + gc.set_debug(0) + gc.enable() diff --git a/venv/lib/python3.9/site-packages/pympler/mprofile.py b/venv/lib/python3.9/site-packages/pympler/mprofile.py new file mode 100644 index 00000000..936d6f76 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/mprofile.py @@ -0,0 +1,97 @@ +""" +Memory usage profiler for Python. + +""" +import inspect +import sys + +from pympler import muppy + + +class MProfiler(object): + """A memory usage profiler class. + + Memory data for each function is stored as a 3-element list in the + dictionary self.memories. The index is always a codepoint (see below). + The following are the definitions of the members: + + [0] = The number of times this function was called + [1] = Minimum memory consumption when this function was measured. + [2] = Maximum memory consumption when this function was measured. + + A codepoint is a list of 3-tuple of the type + (filename, functionname, linenumber). You can omit either element, which + will cause the profiling to be triggered if any of the other criteria + match. E.g. + - (None, foo, None), will profile any foo function, + - (bar, foo, None) will profile only the foo function from the bar file, + - (bar, foo, 17) will profile only line 17 of the foo function defined + in the file bar. + + Additionally, you can define on what events you want the profiling be + triggered. Possible events are defined in + http://docs.python.org/lib/debugger-hooks.html. + + If you do not define either codepoints or events, the profiler will + record the memory usage in at every codepoint and event. + + """ + + def __init__(self, codepoints=None, events=None): + """ + keyword arguments: + codepoints -- a list of points in code to monitor (defaults to all + codepoints) + events -- a list of events to monitor (defaults to all events) + """ + self.memories = {} + self.codepoints = codepoints + self.events = events + + def codepoint_included(self, codepoint): + """Check if codepoint matches any of the defined codepoints.""" + if self.codepoints is None: + return True + for cp in self.codepoints: + mismatch = False + for i in range(len(cp)): + if (cp[i] is not None) and (cp[i] != codepoint[i]): + mismatch = True + break + if not mismatch: + return True + return False + + def profile(self, frame, event, arg): # arg req to match signature + """Profiling method used to profile matching codepoints and events.""" + if (self.events is None) or (event in self.events): + frame_info = inspect.getframeinfo(frame) + cp = (frame_info[0], frame_info[2], frame_info[1]) + if self.codepoint_included(cp): + objects = muppy.get_objects() + size = muppy.get_size(objects) + if cp not in self.memories: + self.memories[cp] = [0, 0, 0, 0] + self.memories[cp][0] = 1 + self.memories[cp][1] = size + self.memories[cp][2] = size + else: + self.memories[cp][0] += 1 + if self.memories[cp][1] > size: + self.memories[cp][1] = size + if self.memories[cp][2] < size: + self.memories[cp][2] = size + + def run(self, cmd): + sys.setprofile(self.profile) + try: + exec(cmd) + finally: + sys.setprofile(None) + return self + + +if __name__ == "__main__": + p = MProfiler() + p.run("print('hello')") + print(p.memories) diff --git a/venv/lib/python3.9/site-packages/pympler/muppy.py b/venv/lib/python3.9/site-packages/pympler/muppy.py new file mode 100644 index 00000000..57000ff8 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/muppy.py @@ -0,0 +1,275 @@ +from typing import Any, Callable, Dict, List, Optional, Set, Tuple + +import gc + +from pympler import summary +from pympler.util import compat + +from inspect import isframe, stack + +from sys import getsizeof + +from pympler.asizeof import _Py_TPFLAGS_HAVE_GC + + +def ignore_object(obj: Any) -> bool: + try: + return isframe(obj) + except ReferenceError: + return True + + +def get_objects(remove_dups: bool = True, include_frames: bool = False + ) -> List[Any]: + """Return a list of all known objects excluding frame objects. + + If (outer) frame objects shall be included, pass `include_frames=True`. In + order to prevent building reference cycles, the current frame object (of + the caller of get_objects) is ignored. This will not prevent creating + reference cycles if the object list is passed up the call-stack. Therefore, + frame objects are not included by default. + + Keyword arguments: + remove_dups -- if True, all duplicate objects will be removed. + include_frames -- if True, includes frame objects. + """ + gc.collect() + + # Do not initialize local variables before calling gc.get_objects or those + # will be included in the list. Furthermore, ignore frame objects to + # prevent reference cycles. + tmp = gc.get_objects() + tmp = [o for o in tmp if not ignore_object(o)] + + res = [] + for o in tmp: + # gc.get_objects returns only container objects, but we also want + # the objects referenced by them + refs = get_referents(o) + for ref in refs: + if not gc.is_tracked(ref): + # we already got the container objects, now we only add + # non-container objects + res.append(ref) + res.extend(tmp) + if remove_dups: + res = _remove_duplicates(res) + + if include_frames: + for sf in stack()[2:]: + res.append(sf[0]) + return res + + +def get_size(objects: List[Any]) -> int: + """Compute the total size of all elements in objects.""" + res = 0 + for o in objects: + try: + res += getsizeof(o) + except AttributeError: + print("IGNORING: type=%s; o=%s" % (str(type(o)), str(o))) + return res + + +def get_diff(left: List[Any], right: List[Any]) -> Dict[str, List[Any]]: + """Get the difference of both lists. + + The result will be a dict with this form {'+': [], '-': []}. + Items listed in '+' exist only in the right list, + items listed in '-' exist only in the left list. + + """ + res = {'+': [], '-': []} # type: Dict[str, List[Any]] + + def partition(objects: List[Any]) -> Dict[type, List[Any]]: + """Partition the passed object list.""" + res = {} # type: Dict[type, List[Any]] + for o in objects: + t = type(o) + if type(o) not in res: + res[t] = [] + res[t].append(o) + return res + + def get_not_included(foo: List[Any], bar: Dict[type, List[Any]] + ) -> List[Any]: + """Compare objects from foo with objects defined in the values of + bar (set of partitions). + Returns a list of all objects included in list, but not dict values. + """ + res = [] # type: List[Any] + for o in foo: + if not compat.object_in_list(type(o), bar): + res.append(o) + elif not compat.object_in_list(o, bar[type(o)]): + res.append(o) + return res + + # Create partitions of both lists. This will reduce the time required for + # the comparison + left_objects = partition(left) + right_objects = partition(right) + # and then do the diff + res['+'] = get_not_included(right, left_objects) + res['-'] = get_not_included(left, right_objects) + return res + + +def sort(objects: List[Any]) -> List[Any]: + """Sort objects by size in bytes.""" + objects = sorted(objects, key=getsizeof) + return objects + + +def filter(objects: List[Any], Type: Optional[type] = None, min: int = -1, + max: int = -1) -> List[Any]: + """Filter objects. + + The filter can be by type, minimum size, and/or maximum size. + + Keyword arguments: + Type -- object type to filter by + min -- minimum object size + max -- maximum object size + + """ + res = [] # type: List[Any] + if min > max and max > -1: + raise ValueError("minimum must be smaller than maximum") + + if Type is not None: + objects = [o for o in objects if isinstance(o, Type)] + if min > -1: + objects = [o for o in objects if getsizeof(o) > min] + if max > -1: + objects = [o for o in objects if getsizeof(o) < max] + return objects + + +def get_referents(object: Any, level: int = 1) -> List[Any]: + """Get all referents of an object up to a certain level. + + The referents will not be returned in a specific order and + will not contain duplicate objects. Duplicate objects will be removed. + + Keyword arguments: + level -- level of indirection to which referents considered. + + This function is recursive. + + """ + res = gc.get_referents(object) + level -= 1 + if level > 0: + for o in res: + res.extend(get_referents(o, level)) + res = _remove_duplicates(res) + return res + + +def _get_usage(function: Callable, *args: Any) -> Optional[List]: + """Test if more memory is used after the function has been called. + + The function will be invoked twice and only the second measurement will be + considered. Thus, memory used in initialisation (e.g. loading modules) + will not be included in the result. The goal is to identify memory leaks + caused by functions which use more and more memory. + + Any arguments next to the function will be passed on to the function + on invocation. + + Note that this function is currently experimental, because it is not + tested thoroughly and performs poorly. + + """ + # The usage of a function is calculated by creating one summary of all + # objects before the function is invoked and afterwards. These summaries + # are compared and the diff is returned. + # This function works in a 2-steps process. Before the actual function is + # invoked an empty dummy function is measurement to identify the overhead + # involved in the measuring process. This overhead then is subtracted from + # the measurement performed on the passed function. The result reflects the + # actual usage of a function call. + # Also, a measurement is performed twice, allowing the adjustment to + # initializing things, e.g. modules + + res = None + + def _get_summaries(function: Callable, *args: Any) -> Tuple: + """Get a 2-tuple containing one summary from before, and one summary + from after the function has been invoked. + + """ + s_before = summary.summarize(get_objects()) + function(*args) + s_after = summary.summarize(get_objects()) + return (s_before, s_after) + + def _get_usage(function: Callable, *args: Any) -> List: + """Get the usage of a function call. + This function is to be used only internally. The 'real' get_usage + function is a wrapper around _get_usage, but the workload is done + here. + + """ + # init before calling + (s_before, s_after) = _get_summaries(function, *args) + # ignore all objects used for the measurement + ignore = [] + if s_before != s_after: + ignore.append(s_before) + for row in s_before: + # ignore refs from summary and frame (loop) + if len(gc.get_referrers(row)) == 2: + ignore.append(row) + for item in row: + # ignore refs from summary and frame (loop) + if len(gc.get_referrers(item)) == 2: + ignore.append(item) + for o in ignore: + s_after = summary._subtract(s_after, o) + res = summary.get_diff(s_before, s_after) + return summary._sweep(res) + + # calibrate; twice for initialization + def noop() -> None: + pass + offset = _get_usage(noop) + offset = _get_usage(noop) + # perform operation twice to handle objects possibly used in + # initialisation + tmp = _get_usage(function, *args) + tmp = _get_usage(function, *args) + tmp = summary.get_diff(offset, tmp) + tmp = summary._sweep(tmp) + if len(tmp) != 0: + res = tmp + return res + + +def _is_containerobject(o: Any) -> bool: + """Is the passed object a container object.""" + return bool(getattr(type(o), '__flags__', 0) & _Py_TPFLAGS_HAVE_GC) + + +def _remove_duplicates(objects: List[Any]) -> List[Any]: + """Remove duplicate objects. + + Inspired by http://www.peterbe.com/plog/uniqifiers-benchmark + + """ + seen = set() # type: Set[int] + result = [] + for item in objects: + marker = id(item) + if marker in seen: + continue + seen.add(marker) + result.append(item) + return result + + +def print_summary() -> None: + """Print a summary of all known objects.""" + summary.print_(summary.summarize(get_objects())) diff --git a/venv/lib/python3.9/site-packages/pympler/panels.py b/venv/lib/python3.9/site-packages/pympler/panels.py new file mode 100644 index 00000000..3bb6217a --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/panels.py @@ -0,0 +1,115 @@ +""" +Expose a memory-profiling panel to the Django Debug toolbar. + +Shows process memory information (virtual size, resident set size) and model +instances for the current request. + +Requires Django and Django Debug toolbar: + +https://github.com/jazzband/django-debug-toolbar + +Pympler adds a memory panel as a third party addon (not included in the +django-debug-toolbar). It can be added by overriding the `DEBUG_TOOLBAR_PANELS` +setting in the Django project settings:: + + DEBUG_TOOLBAR_PANELS = ( + 'debug_toolbar.panels.timer.TimerDebugPanel', + 'pympler.panels.MemoryPanel', + ) + +Pympler also needs to be added to the `INSTALLED_APPS` in the Django settings:: + + INSTALLED_APPS = INSTALLED_APPS + ('debug_toolbar', 'pympler') +""" + +from pympler.classtracker import ClassTracker +from pympler.process import ProcessMemoryInfo +from pympler.util.stringutils import pp + +try: + from debug_toolbar.panels import Panel + from django.apps import apps + from django.template import Context, Template + from django.template.loader import render_to_string + from django.http.request import HttpRequest + from django.http.response import HttpResponse +except ImportError: + class Panel(object): # type: ignore + pass + + class Template(object): # type: ignore + pass + + class Context(object): # type: ignore + pass + + class HttpRequest(object): # type: ignore + pass + + class HttpResponse(object): # type: ignore + pass + + +class MemoryPanel(Panel): + + name = 'pympler' + + title = 'Memory' + + template = 'memory_panel.html' + + classes = [Context, Template] + + def process_request(self, request: HttpRequest) -> HttpResponse: + self._tracker = ClassTracker() + for cls in apps.get_models() + self.classes: + self._tracker.track_class(cls) + self._tracker.create_snapshot('before') + self.record_stats({'before': ProcessMemoryInfo()}) + response = super(MemoryPanel, self).process_request(request) + self.record_stats({'after': ProcessMemoryInfo()}) + self._tracker.create_snapshot('after') + stats = self._tracker.stats + stats.annotate() + self.record_stats({'stats': stats}) + return response + + def enable_instrumentation(self) -> None: + self._tracker = ClassTracker() + for cls in apps.get_models() + self.classes: + self._tracker.track_class(cls) + + def disable_instrumentation(self) -> None: + self._tracker.detach_all_classes() + + def nav_subtitle(self) -> str: + context = self.get_stats() + before = context['before'] + after = context['after'] + rss = after.rss + delta = rss - before.rss + delta = ('(+%s)' % pp(delta)) if delta > 0 else '' + return "%s %s" % (pp(rss), delta) + + @property + def content(self) -> str: + context = self.get_stats() + before = context['before'] + after = context['after'] + stats = context['stats'] + rows = [('Resident set size', after.rss), + ('Virtual size', after.vsz), + ] + rows.extend(after - before) + rows = [(key, pp(value)) for key, value in rows] + rows.extend(after.os_specific) + + classes = [] + snapshot = stats.snapshots[-1] + for model in stats.tracked_classes: + history = [cnt for _, cnt in stats.history[model]] + size = snapshot.classes.get(model, {}).get('sum', 0) + if history and history[-1] > 0: + classes.append((model, history, pp(size))) + context.update({'rows': rows, 'classes': classes}) + return render_to_string(self.template, context) diff --git a/venv/lib/python3.9/site-packages/pympler/process.py b/venv/lib/python3.9/site-packages/pympler/process.py new file mode 100644 index 00000000..b3a31528 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/process.py @@ -0,0 +1,238 @@ +""" +This module queries process memory allocation metrics from the operating +system. It provides a platform independent layer to get the amount of virtual +and physical memory allocated to the Python process. + +Different mechanisms are implemented: Either the process stat file is read +(Linux), the `ps` command is executed (BSD/OSX/Solaris) or the resource module +is queried (Unix fallback). On Windows try to use the win32 module if +available. If all fails, return 0 for each attribute. + +Windows without the win32 module is not supported. + + >>> from pympler.process import ProcessMemoryInfo + >>> pmi = ProcessMemoryInfo() + >>> print ("Virtual size [Byte]: " + str(pmi.vsz)) # doctest: +ELLIPSIS + Virtual size [Byte]: ... +""" + +from typing import Iterable, List, Tuple + +import logging +import threading + +from mmap import PAGESIZE # type: ignore +from os import getpid +from subprocess import Popen, PIPE + +from pympler.util.stringutils import pp + + +class _ProcessMemoryInfo(object): + """Stores information about various process-level memory metrics. The + virtual size is stored in attribute `vsz`, the physical memory allocated to + the process in `rss`, and the number of (major) pagefaults in `pagefaults`. + On Linux, `data_segment`, `code_segment`, `shared_segment` and + `stack_segment` contain the number of Bytes allocated for the respective + segments. This is an abstract base class which needs to be overridden by + operating system specific implementations. This is done when importing the + module. + """ + + pagesize = PAGESIZE + + def __init__(self) -> None: + self.pid = getpid() + self.rss = 0 + self.vsz = 0 + self.pagefaults = 0 + self.os_specific = [] # type: List[Tuple[str, str]] + + self.data_segment = 0 + self.code_segment = 0 + self.shared_segment = 0 + self.stack_segment = 0 + + self.available = self.update() + + def __repr__(self) -> str: + return "<%s vsz=%d rss=%d>" % (self.__class__.__name__, + self.vsz, self.rss) + + def update(self) -> bool: + """ + Refresh the information using platform instruments. Returns true if + this operation yields useful values on the current platform. + """ + return False # pragma: no cover + + def __sub__(self, other: '_ProcessMemoryInfo') -> Iterable[Tuple[str, int]]: + diff = [('Resident set size (delta)', self.rss - other.rss), + ('Virtual size (delta)', self.vsz - other.vsz), + ] + return diff + + +ProcessMemoryInfo = _ProcessMemoryInfo # type: type + + +def is_available() -> bool: + """ + Convenience function to check if the current platform is supported by this + module. + """ + return ProcessMemoryInfo().update() + + +class _ProcessMemoryInfoPS(_ProcessMemoryInfo): + + def update(self) -> bool: + """ + Get virtual and resident size of current process via 'ps'. + This should work for MacOS X, Solaris, Linux. Returns true if it was + successful. + """ + try: + p = Popen(['/bin/ps', '-p%s' % self.pid, '-o', 'rss,vsz'], + stdout=PIPE, stderr=PIPE) + except OSError: # pragma: no cover + pass + else: + s = p.communicate()[0].split() + if p.returncode == 0 and len(s) >= 2: # pragma: no branch + self.vsz = int(s[-1]) * 1024 + self.rss = int(s[-2]) * 1024 + return True + return False # pragma: no cover + + +class _ProcessMemoryInfoProc(_ProcessMemoryInfo): + + key_map = { + 'VmPeak': 'Peak virtual memory size', + 'VmSize': 'Virtual memory size', + 'VmLck': 'Locked memory size', + 'VmHWM': 'Peak resident set size', + 'VmRSS': 'Resident set size', + 'VmStk': 'Size of stack segment', + 'VmData': 'Size of data segment', + 'VmExe': 'Size of code segment', + 'VmLib': 'Shared library code size', + 'VmPTE': 'Page table entries size', + } + + def update(self) -> bool: + """ + Get virtual size of current process by reading the process' stat file. + This should work for Linux. + """ + try: + stat = open('/proc/self/stat') + status = open('/proc/self/status') + except IOError: # pragma: no cover + return False + else: + stats = stat.read().split() + self.vsz = int(stats[22]) + self.rss = int(stats[23]) * self.pagesize + self.pagefaults = int(stats[11]) + + for entry in status.readlines(): + try: + key, value = entry.split(':', 1) + except ValueError: + continue + value = value.strip() + + def size_in_bytes(x: str) -> int: + return int(x.split()[0]) * 1024 + + if key == 'VmData': + self.data_segment = size_in_bytes(value) + elif key == 'VmExe': + self.code_segment = size_in_bytes(value) + elif key == 'VmLib': + self.shared_segment = size_in_bytes(value) + elif key == 'VmStk': + self.stack_segment = size_in_bytes(value) + key = self.key_map.get(key, '') + if key: + self.os_specific.append((key, pp(size_in_bytes(value)))) + + stat.close() + status.close() + return True + + +try: + from resource import getrusage, RUSAGE_SELF + + class _ProcessMemoryInfoResource(_ProcessMemoryInfo): + def update(self) -> bool: + """ + Get memory metrics of current process through `getrusage`. Only + available on Unix, on Linux most of the fields are not set, + and on BSD units are used that are not very helpful, see: + + http://www.perlmonks.org/?node_id=626693 + + Furthermore, getrusage only provides accumulated statistics (e.g. + max rss vs current rss). + """ + usage = getrusage(RUSAGE_SELF) + self.rss = usage.ru_maxrss * 1024 + self.data_segment = usage.ru_idrss * 1024 # TODO: ticks? + self.shared_segment = usage.ru_ixrss * 1024 # TODO: ticks? + self.stack_segment = usage.ru_isrss * 1024 # TODO: ticks? + self.vsz = (self.data_segment + self.shared_segment + + self.stack_segment) + + self.pagefaults = usage.ru_majflt + return self.rss != 0 + + if _ProcessMemoryInfoProc().update(): # pragma: no branch + ProcessMemoryInfo = _ProcessMemoryInfoProc + elif _ProcessMemoryInfoPS().update(): # pragma: no cover + ProcessMemoryInfo = _ProcessMemoryInfoPS + elif _ProcessMemoryInfoResource().update(): # pragma: no cover + ProcessMemoryInfo = _ProcessMemoryInfoResource + +except ImportError: + try: + # Requires pywin32 + from win32process import GetProcessMemoryInfo + from win32api import GetCurrentProcess, GlobalMemoryStatusEx + except ImportError: + logging.warn("Please install pywin32 when using pympler on Windows.") + else: + class _ProcessMemoryInfoWin32(_ProcessMemoryInfo): + def update(self) -> bool: + process_handle = GetCurrentProcess() + meminfo = GetProcessMemoryInfo(process_handle) + memstatus = GlobalMemoryStatusEx() + self.vsz = (memstatus['TotalVirtual'] - + memstatus['AvailVirtual']) + self.rss = meminfo['WorkingSetSize'] + self.pagefaults = meminfo['PageFaultCount'] + return True + + ProcessMemoryInfo = _ProcessMemoryInfoWin32 + + +class ThreadInfo(object): + """Collect information about an active thread.""" + + def __init__(self, thread: threading.Thread): + self.ident = thread.ident + self.name = thread.name + self.daemon = thread.daemon + + +def get_current_threads() -> Iterable[ThreadInfo]: + """Get a list of `ThreadInfo` objects.""" + return [ThreadInfo(thread) for thread in threading.enumerate()] + + +def get_current_thread_id() -> int: + """Get the ID of the current thread.""" + return threading.get_ident() diff --git a/venv/lib/python3.9/site-packages/pympler/refbrowser.py b/venv/lib/python3.9/site-packages/pympler/refbrowser.py new file mode 100644 index 00000000..ae8966fd --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/refbrowser.py @@ -0,0 +1,451 @@ +"""Tree-like exploration of object referrers. + +This module provides a base implementation for tree-like referrers browsing. +The two non-interactive classes ConsoleBrowser and FileBrowser output a tree +to the console or a file. One graphical user interface for referrers browsing +is provided as well. Further types can be subclassed. + +All types share a similar initialisation. That is, you provide a root object +and may specify further settings such as the initial depth of the tree or an +output function. +Afterwards you can print the tree which will be arranged based on your previous +settings. + +The interactive browser is based on a TreeWidget implemented in IDLE. It is +available only if you have Tcl/Tk installed. If you try to instantiate the +interactive browser without having Tkinter installed, an ImportError will be +raised. + +""" +import gc +import inspect +import sys + +from pympler import muppy +from pympler import summary + +from pympler.util.compat import tkinter + + +class _Node(object): + """A node as it is used in the tree structure. + + Each node contains the object it represents and a list of children. + Children can be other nodes or arbitrary other objects. Any object + in a tree which is not of the type _Node is considered a leaf. + + """ + def __init__(self, o, str_func=None): + """You have to define the object this node represents. Also you can + define an output function which will be used to represent this node. + If no function is defined, the default str representation is used. + + keyword arguments + str_func -- output function + + """ + self.o = o + self.children = [] + self.str_func = str_func + + def __str__(self): + """Override str(self.o) if str_func is defined.""" + if self.str_func is not None: + return self.str_func(self.o) + else: + return str(self.o) + + +class RefBrowser(object): + """Base class to other RefBrowser implementations. + + This base class provides means to extract a tree from a given root object + and holds information on already known objects (to avoid repetition + if requested). + + """ + + def __init__(self, rootobject, maxdepth=3, str_func=summary._repr, + repeat=True, stream=None): + """You have to provide the root object used in the refbrowser. + + keyword arguments + maxdepth -- maximum depth of the initial tree + str_func -- function used when calling str(node) + repeat -- should nodes appear repeatedly in the tree, or should be + referred to existing nodes + stream -- output stream (used in derived classes) + + """ + self.root = rootobject + self.maxdepth = maxdepth + self.str_func = str_func + self.repeat = repeat + self.stream = stream + # objects which should be ignored while building the tree + # e.g. the current frame + self.ignore = [] + # set of object ids which are already included + self.already_included = set() + self.ignore.append(self.already_included) + + def get_tree(self): + """Get a tree of referrers of the root object.""" + self.ignore.append(inspect.currentframe()) + return self._get_tree(self.root, self.maxdepth) + + def _get_tree(self, root, maxdepth): + """Workhorse of the get_tree implementation. + + This is a recursive method which is why we have a wrapper method. + root is the current root object of the tree which should be returned. + Note that root is not of the type _Node. + maxdepth defines how much further down the from the root the tree + should be build. + + """ + objects = gc.get_referrers(root) + res = _Node(root, self.str_func) + self.already_included.add(id(root)) + if maxdepth == 0: + return res + self.ignore.append(inspect.currentframe()) + self.ignore.append(objects) + for o in objects: + # Ignore dict of _Node and RefBrowser objects + if isinstance(o, dict): + if any(isinstance(ref, (_Node, RefBrowser)) + for ref in gc.get_referrers(o)): + continue + _id = id(o) + if not self.repeat and (_id in self.already_included): + s = self.str_func(o) + res.children.append("%s (already included, id %s)" % + (s, _id)) + continue + if (not isinstance(o, _Node)) and (o not in self.ignore): + res.children.append(self._get_tree(o, maxdepth - 1)) + return res + + +class StreamBrowser(RefBrowser): + """RefBrowser implementation which prints the tree to the console. + + If you don't like the looks, you can change it a little bit. + The class attributes 'hline', 'vline', 'cross', and 'space' can be + modified to your needs. + + """ + hline = '-' + vline = '|' + cross = '+' + space = ' ' + + def print_tree(self, tree=None): + """ Print referrers tree to console. + + keyword arguments + tree -- if not None, the passed tree will be printed. Otherwise it is + based on the rootobject. + + """ + if tree is None: + tree = self.get_tree() + self._print(tree, '', '') + + def _print(self, tree, prefix, carryon): + """Compute and print a new line of the tree. + + This is a recursive function. + + arguments + tree -- tree to print + prefix -- prefix to the current line to print + carryon -- prefix which is used to carry on the vertical lines + + """ + level = prefix.count(self.cross) + prefix.count(self.vline) + len_children = 0 + if isinstance(tree, _Node): + len_children = len(tree.children) + + # add vertex + prefix += str(tree) + # and as many spaces as the vertex is long + carryon += self.space * len(str(tree)) + if (level == self.maxdepth) or (not isinstance(tree, _Node)) or\ + (len_children == 0): + self.stream.write(prefix + '\n') + return + else: + # add in between connections + prefix += self.hline + carryon += self.space + # if there is more than one branch, add a cross + if len(tree.children) > 1: + prefix += self.cross + carryon += self.vline + prefix += self.hline + carryon += self.space + + if len_children > 0: + # print the first branch (on the same line) + self._print(tree.children[0], prefix, carryon) + for b in range(1, len_children): + # the carryon becomes the prefix for all following children + prefix = carryon[:-2] + self.cross + self.hline + # remove the vlines for any children of last branch + if b == (len_children - 1): + carryon = carryon[:-2] + 2 * self.space + self._print(tree.children[b], prefix, carryon) + # leave a free line before the next branch + if b == (len_children - 1): + if len(carryon.strip(' ')) == 0: + return + self.stream.write(carryon[:-2].rstrip() + '\n') + + +class ConsoleBrowser(StreamBrowser): + """RefBrowser that prints to the console (stdout).""" + + def __init__(self, *args, **kwargs): + super(ConsoleBrowser, self).__init__(*args, **kwargs) + if not self.stream: + self.stream = sys.stdout + + +class FileBrowser(StreamBrowser): + """RefBrowser implementation which prints the tree to a file.""" + + def print_tree(self, filename, tree=None): + """ Print referrers tree to file (in text format). + + keyword arguments + tree -- if not None, the passed tree will be printed. + + """ + old_stream = self.stream + self.stream = open(filename, 'w') + try: + super(FileBrowser, self).print_tree(tree=tree) + finally: + self.stream.close() + self.stream = old_stream + + +# Code for interactive browser (GUI) +# ================================== + +# The interactive browser requires Tkinter which is not always available. To +# avoid an import error when loading the module, we encapsulate most of the +# code in the following try-except-block. The InteractiveBrowser itself +# remains outside this block. If you try to instantiate it without having +# Tkinter installed, the import error will be raised. +try: + if sys.version_info < (3, 5, 2): + from idlelib import TreeWidget as _TreeWidget + else: + from idlelib import tree as _TreeWidget + + class _TreeNode(_TreeWidget.TreeNode): + """TreeNode used by the InteractiveBrowser. + + Not to be confused with _Node. This one is used in the GUI + context. + + """ + def reload_referrers(self): + """Reload all referrers for this _TreeNode.""" + self.item.node = self.item.reftree._get_tree(self.item.node.o, 1) + self.item._clear_children() + self.expand() + self.update() + + def print_object(self): + """Print object which this _TreeNode represents to console.""" + print(self.item.node.o) + + def drawtext(self): + """Override drawtext from _TreeWidget.TreeNode. + + This seems to be a good place to add the popup menu. + + """ + _TreeWidget.TreeNode.drawtext(self) + # create a menu + menu = tkinter.Menu(self.canvas, tearoff=0) + menu.add_command(label="reload referrers", + command=self.reload_referrers) + menu.add_command(label="print", command=self.print_object) + menu.add_separator() + menu.add_command(label="expand", command=self.expand) + menu.add_separator() + # the popup only disappears when to click on it + menu.add_command(label="Close Popup Menu") + + def do_popup(event): + menu.post(event.x_root, event.y_root) + + self.label.bind("", do_popup) + # override, i.e. disable the editing of items + + # disable editing of TreeNodes + def edit(self, event=None): + pass # see comment above + + def edit_finish(self, event=None): + pass # see comment above + + def edit_cancel(self, event=None): + pass # see comment above + + class _ReferrerTreeItem(_TreeWidget.TreeItem, tkinter.Label): + """Tree item wrapper around _Node object.""" + + def __init__(self, parentwindow, node, reftree): # constr calls + """You need to provide the parent window, the node this TreeItem + represents, as well as the tree (_Node) which the node + belongs to. + + """ + _TreeWidget.TreeItem.__init__(self) + tkinter.Label.__init__(self, parentwindow) + self.node = node + self.parentwindow = parentwindow + self.reftree = reftree + + def _clear_children(self): + """Clear children list from any TreeNode instances. + + Normally these objects are not required for memory profiling, as + they are part of the profiler. + + """ + new_children = [] + for child in self.node.children: + if not isinstance(child, _TreeNode): + new_children.append(child) + self.node.children = new_children + + def GetText(self): + return str(self.node) + + def GetIconName(self): + """Different icon when object cannot be expanded, i.e. has no + referrers. + + """ + if not self.IsExpandable(): + return "python" + + def IsExpandable(self): + """An object is expandable when it is a node which has children and + is a container object. + + """ + if not isinstance(self.node, _Node): + return False + else: + if len(self.node.children) > 0: + return True + else: + return muppy._is_containerobject(self.node.o) + + def GetSubList(self): + """This method is the point where further referrers are computed. + + Thus, the computation is done on-demand and only when needed. + + """ + sublist = [] + + children = self.node.children + if (len(children) == 0) and\ + (muppy._is_containerobject(self.node.o)): + self.node = self.reftree._get_tree(self.node.o, 1) + self._clear_children() + children = self.node.children + + for child in children: + item = _ReferrerTreeItem(self.parentwindow, child, + self.reftree) + sublist.append(item) + return sublist + +except ImportError: + _TreeWidget = None + + +def gui_default_str_function(o): + """Default str function for InteractiveBrowser.""" + return summary._repr(o) + '(id=%s)' % id(o) + + +class InteractiveBrowser(RefBrowser): + """Interactive referrers browser. + + The interactive browser is based on a TreeWidget implemented in IDLE. It is + available only if you have Tcl/Tk installed. If you try to instantiate the + interactive browser without having Tkinter installed, an ImportError will + be raised. + + """ + def __init__(self, rootobject, maxdepth=3, + str_func=gui_default_str_function, repeat=True): + """You have to provide the root object used in the refbrowser. + + keyword arguments + maxdepth -- maximum depth of the initial tree + str_func -- function used when calling str(node) + repeat -- should nodes appear repeatedly in the tree, or should be + referred to existing nodes + + """ + if tkinter is None: + raise ImportError( + "InteractiveBrowser requires Tkinter to be installed.") + RefBrowser.__init__(self, rootobject, maxdepth, str_func, repeat) + + def main(self, standalone=False): + """Create interactive browser window. + + keyword arguments + standalone -- Set to true, if the browser is not attached to other + windows + + """ + window = tkinter.Tk() + sc = _TreeWidget.ScrolledCanvas(window, bg="white", + highlightthickness=0, takefocus=1) + sc.frame.pack(expand=1, fill="both") + item = _ReferrerTreeItem(window, self.get_tree(), self) + node = _TreeNode(sc.canvas, None, item) + node.expand() + if standalone: + window.mainloop() + + +# list to hold to referrers +superlist = [] +root = "root" +for i in range(3): + tmp = [root] + superlist.append(tmp) + + +def foo(o): + return str(type(o)) + + +def print_sample(): + cb = ConsoleBrowser(root, str_func=foo) + cb.print_tree() + + +def write_sample(): + fb = FileBrowser(root, str_func=foo) + fb.print_tree('sample.txt') + + +if __name__ == "__main__": + write_sample() diff --git a/venv/lib/python3.9/site-packages/pympler/refgraph.py b/venv/lib/python3.9/site-packages/pympler/refgraph.py new file mode 100644 index 00000000..7f1f28bd --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/refgraph.py @@ -0,0 +1,350 @@ +""" +This module exposes utilities to illustrate objects and their references as +(directed) graphs. The current implementation requires 'graphviz' to be +installed. +""" + +from pympler.asizeof import Asizer, named_refs +from pympler.util.stringutils import safe_repr, trunc +from gc import get_referents +from subprocess import Popen, PIPE +from copy import copy +from sys import platform + +__all__ = ['ReferenceGraph'] + + +# Popen might lead to deadlocks when file descriptors are leaked to +# sub-processes on Linux. On Windows, however, close_fds=True leads to +# ValueError if stdin/stdout/stderr is piped: +# http://code.google.com/p/pympler/issues/detail?id=28#c1 +popen_flags = {} +if platform not in ['win32']: # pragma: no branch + popen_flags['close_fds'] = True + + +class _MetaObject(object): + """ + The _MetaObject stores meta-information, like a string representation, + corresponding to each object passed to a ReferenceGraph. + """ + __slots__ = ('size', 'id', 'type', 'str', 'group', 'cycle') + + def __init__(self): + self.cycle = False + + +class _Edge(object): + """ + Describes a reference from one object `src` to another object `dst`. + """ + __slots__ = ('src', 'dst', 'label', 'group') + + def __init__(self, src, dst, label): + self.src = src + self.dst = dst + self.label = label + self.group = None + + def __repr__(self): + return "<%08x => %08x, '%s', %s>" % (self.src, self.dst, self.label, + self.group) + + def __hash__(self): + return (self.src, self.dst, self.label).__hash__() + + def __eq__(self, other): + return self.__hash__() == other.__hash__() + + +class ReferenceGraph(object): + """ + The ReferenceGraph illustrates the references between a collection of + objects by rendering a directed graph. That requires that 'graphviz' is + installed. + + >>> from pympler.refgraph import ReferenceGraph + >>> a = 42 + >>> b = 'spam' + >>> c = {a: b} + >>> gb = ReferenceGraph([a,b,c]) + >>> gb.render('spam.eps') + True + """ + def __init__(self, objects, reduce=False): + """ + Initialize the ReferenceGraph with a collection of `objects`. + """ + self.objects = list(objects) + self.count = len(self.objects) + self.num_in_cycles = 'N/A' + self.edges = None + + if reduce: + self.num_in_cycles = self._reduce_to_cycles() + self._reduced = self # TODO: weakref? + else: + self._reduced = None + + self._get_edges() + self._annotate_objects() + + def _eliminate_leafs(self, graph): + """ + Eliminate leaf objects - that are objects not referencing any other + objects in the list `graph`. Returns the list of objects without the + objects identified as leafs. + """ + result = [] + idset = set([id(x) for x in graph]) + for n in graph: + refset = set([id(x) for x in get_referents(n)]) + if refset.intersection(idset): + result.append(n) + return result + + def _reduce_to_cycles(self): + """ + Iteratively eliminate leafs to reduce the set of objects to only those + that build cycles. Return the number of objects involved in reference + cycles. If there are no cycles, `self.objects` will be an empty list + and this method returns 0. + """ + cycles = self.objects[:] + cnt = 0 + while cnt != len(cycles): + cnt = len(cycles) + cycles = self._eliminate_leafs(cycles) + self.objects = cycles + return len(self.objects) + + def reduce_to_cycles(self): + """ + Iteratively eliminate leafs to reduce the set of objects to only those + that build cycles. Return the reduced graph. If there are no cycles, + None is returned. + """ + if not self._reduced: + reduced = copy(self) + reduced.objects = self.objects[:] + reduced.metadata = [] + reduced.edges = [] + self.num_in_cycles = reduced._reduce_to_cycles() + reduced.num_in_cycles = self.num_in_cycles + if self.num_in_cycles: + reduced._get_edges() + reduced._annotate_objects() + for meta in reduced.metadata: + meta.cycle = True + else: + reduced = None + self._reduced = reduced + return self._reduced + + def _get_edges(self): + """ + Compute the edges for the reference graph. + The function returns a set of tuples (id(a), id(b), ref) if a + references b with the referent 'ref'. + """ + idset = set([id(x) for x in self.objects]) + self.edges = set([]) + for n in self.objects: + refset = set([id(x) for x in get_referents(n)]) + for ref in refset.intersection(idset): + label = '' + members = None + if isinstance(n, dict): + members = n.items() + if not members: + members = named_refs(n) + for (k, v) in members: + if id(v) == ref: + label = k + break + self.edges.add(_Edge(id(n), ref, label)) + + def _annotate_groups(self): + """ + Annotate the objects belonging to separate (non-connected) graphs with + individual indices. + """ + g = {} + for x in self.metadata: + g[x.id] = x + + idx = 0 + for x in self.metadata: + if not hasattr(x, 'group'): + x.group = idx + idx += 1 + neighbors = set() + for e in self.edges: + if e.src == x.id: + neighbors.add(e.dst) + if e.dst == x.id: + neighbors.add(e.src) + for nb in neighbors: + g[nb].group = min(x.group, getattr(g[nb], 'group', idx)) + + # Assign the edges to the respective groups. Both "ends" of the edge + # should share the same group so just use the first object's group. + for e in self.edges: + e.group = g[e.src].group + + self._max_group = idx + + def _filter_group(self, group): + """ + Eliminate all objects but those which belong to `group`. + ``self.objects``, ``self.metadata`` and ``self.edges`` are modified. + Returns `True` if the group is non-empty. Otherwise returns `False`. + """ + self.metadata = [x for x in self.metadata if x.group == group] + group_set = set([x.id for x in self.metadata]) + self.objects = [obj for obj in self.objects if id(obj) in group_set] + self.count = len(self.metadata) + if self.metadata == []: + return False + + self.edges = [e for e in self.edges if e.group == group] + + del self._max_group + + return True + + def split(self): + """ + Split the graph into sub-graphs. Only connected objects belong to the + same graph. `split` yields copies of the Graph object. Shallow copies + are used that only replicate the meta-information, but share the same + object list ``self.objects``. + + >>> from pympler.refgraph import ReferenceGraph + >>> a = 42 + >>> b = 'spam' + >>> c = {a: b} + >>> t = (1,2,3) + >>> rg = ReferenceGraph([a,b,c,t]) + >>> for subgraph in rg.split(): + ... print (subgraph.index) + 0 + 1 + """ + self._annotate_groups() + index = 0 + + for group in range(self._max_group): + subgraph = copy(self) + subgraph.metadata = self.metadata[:] + subgraph.edges = self.edges.copy() + + if subgraph._filter_group(group): + subgraph.total_size = sum([x.size for x in subgraph.metadata]) + subgraph.index = index + index += 1 + yield subgraph + + def split_and_sort(self): + """ + Split the graphs into sub graphs and return a list of all graphs sorted + by the number of nodes. The graph with most nodes is returned first. + """ + graphs = list(self.split()) + graphs.sort(key=lambda x: -len(x.metadata)) + for index, graph in enumerate(graphs): + graph.index = index + return graphs + + def _annotate_objects(self): + """ + Extract meta-data describing the stored objects. + """ + self.metadata = [] + sizer = Asizer() + sizes = sizer.asizesof(*self.objects) + self.total_size = sizer.total + for obj, sz in zip(self.objects, sizes): + md = _MetaObject() + md.size = sz + md.id = id(obj) + try: + md.type = obj.__class__.__name__ + except (AttributeError, ReferenceError): # pragma: no cover + md.type = type(obj).__name__ + md.str = safe_repr(obj, clip=128) + self.metadata.append(md) + + def _get_graphviz_data(self): + """ + Emit a graph representing the connections between the objects described + within the metadata list. The text representation can be transformed to + a graph with graphviz. Returns a string. + """ + s = [] + header = '// Process this file with graphviz\n' + s.append(header) + s.append('digraph G {\n') + s.append(' node [shape=box];\n') + for md in self.metadata: + label = trunc(md.str, 48).replace('"', "'") + extra = '' + if md.type == 'instancemethod': + extra = ', color=red' + elif md.type == 'frame': + extra = ', color=orange' + s.append(' "X%s" [ label = "%s\\n%s" %s ];\n' % + (hex(md.id)[1:], label, md.type, extra)) + for e in self.edges: + extra = '' + if e.label == '__dict__': + extra = ',weight=100' + s.append(' X%s -> X%s [label="%s"%s];\n' % + (hex(e.src)[1:], hex(e.dst)[1:], e.label, extra)) + + s.append('}\n') + return "".join(s) + + def render(self, filename, cmd='dot', format='ps', unflatten=False): + """ + Render the graph to `filename` using graphviz. The graphviz invocation + command may be overridden by specifying `cmd`. The `format` may be any + specifier recognized by the graph renderer ('-Txxx' command). The + graph can be preprocessed by the *unflatten* tool if the `unflatten` + parameter is True. If there are no objects to illustrate, the method + does not invoke graphviz and returns False. If the renderer returns + successfully (return code 0), True is returned. + + An `OSError` is raised if the graphviz tool cannot be found. + """ + if self.objects == []: + return False + + data = self._get_graphviz_data() + + options = ('-Nfontsize=10', + '-Efontsize=10', + '-Nstyle=filled', + '-Nfillcolor=#E5EDB8', + '-Ncolor=#CCCCCC') + cmdline = (cmd, '-T%s' % format, '-o', filename) + options + + if unflatten: + p1 = Popen(('unflatten', '-l7'), stdin=PIPE, stdout=PIPE, + **popen_flags) + p2 = Popen(cmdline, stdin=p1.stdout, **popen_flags) + p1.communicate(data.encode()) + p2.communicate() + return p2.returncode == 0 + else: + p = Popen(cmdline, stdin=PIPE, **popen_flags) + p.communicate(data.encode()) + return p.returncode == 0 + + def write_graph(self, filename): + """ + Write raw graph data which can be post-processed using graphviz. + """ + f = open(filename, 'w') + f.write(self._get_graphviz_data()) + f.close() diff --git a/venv/lib/python3.9/site-packages/pympler/static/jquery.sparkline.min.js b/venv/lib/python3.9/site-packages/pympler/static/jquery.sparkline.min.js new file mode 100644 index 00000000..74187063 --- /dev/null +++ b/venv/lib/python3.9/site-packages/pympler/static/jquery.sparkline.min.js @@ -0,0 +1,5 @@ +/* jquery.sparkline 2.1.1 - http://omnipotent.net/jquery.sparkline/ +** Licensed under the New BSD License - see above site for details */ + +(function(a){typeof define=="function"&&define.amd?define(["jquery"],a):a(jQuery)})(function(a){"use strict";var b={},c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,A,B,C,D,E,F,G,H,I=0;c=function(){return{common:{type:"line",lineColor:"#00f",fillColor:"#cdf",defaultPixelsPerValue:3,width:"auto",height:"auto",composite:!1,tagValuesAttribute:"values",tagOptionsPrefix:"spark",enableTagOptions:!1,enableHighlight:!0,highlightLighten:1.4,tooltipSkipNull:!0,tooltipPrefix:"",tooltipSuffix:"",disableHiddenCheck:!1,numberFormatter:!1,numberDigitGroupCount:3,numberDigitGroupSep:",",numberDecimalMark:".",disableTooltips:!1,disableInteraction:!1},line:{spotColor:"#f80",highlightSpotColor:"#5f5",highlightLineColor:"#f22",spotRadius:1.5,minSpotColor:"#f80",maxSpotColor:"#f80",lineWidth:1,normalRangeMin:undefined,normalRangeMax:undefined,normalRangeColor:"#ccc",drawNormalOnTop:!1,chartRangeMin:undefined,chartRangeMax:undefined,chartRangeMinX:undefined,chartRangeMaxX:undefined,tooltipFormat:new e('● {{prefix}}{{y}}{{suffix}}')},bar:{barColor:"#3366cc",negBarColor:"#f44",stackedBarColor:["#3366cc","#dc3912","#ff9900","#109618","#66aa00","#dd4477","#0099c6","#990099"],zeroColor:undefined,nullColor:undefined,zeroAxis:!0,barWidth:4,barSpacing:1,chartRangeMax:undefined,chartRangeMin:undefined,chartRangeClip:!1,colorMap:undefined,tooltipFormat:new e('● {{prefix}}{{value}}{{suffix}}')},tristate:{barWidth:4,barSpacing:1,posBarColor:"#6f6",negBarColor:"#f44",zeroBarColor:"#999",colorMap:{},tooltipFormat:new e('● {{value:map}}'),tooltipValueLookups:{map:{"-1":"Loss",0:"Draw",1:"Win"}}},discrete:{lineHeight:"auto",thresholdColor:undefined,thresholdValue:0,chartRangeMax:undefined,chartRangeMin:undefined,chartRangeClip:!1,tooltipFormat:new e("{{prefix}}{{value}}{{suffix}}")},bullet:{targetColor:"#f33",targetWidth:3,performanceColor:"#33f",rangeColors:["#d3dafe","#a8b6ff","#7f94ff"],base:undefined,tooltipFormat:new e("{{fieldkey:fields}} - {{value}}"),tooltipValueLookups:{fields:{r:"Range",p:"Performance",t:"Target"}}},pie:{offset:0,sliceColors:["#3366cc","#dc3912","#ff9900","#109618","#66aa00","#dd4477","#0099c6","#990099"],borderWidth:0,borderColor:"#000",tooltipFormat:new e('● {{value}} ({{percent.1}}%)')},box:{raw:!1,boxLineColor:"#000",boxFillColor:"#cdf",whiskerColor:"#000",outlierLineColor:"#333",outlierFillColor:"#fff",medianColor:"#f00",showOutliers:!0,outlierIQR:1.5,spotRadius:1.5,target:undefined,targetColor:"#4a2",chartRangeMax:undefined,chartRangeMin:undefined,tooltipFormat:new e("{{field:fields}}: {{value}}"),tooltipFormatFieldlistKey:"field",tooltipValueLookups:{fields:{lq:"Lower Quartile",med:"Median",uq:"Upper Quartile",lo:"Left Outlier",ro:"Right Outlier",lw:"Left Whisker",rw:"Right Whisker"}}}}},B='.jqstooltip { position: absolute;left: 0px;top: 0px;visibility: hidden;background: rgb(0, 0, 0) transparent;background-color: rgba(0,0,0,0.6);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr=#99000000, endColorstr=#99000000);-ms-filter: "progid:DXImageTransform.Microsoft.gradient(startColorstr=#99000000, endColorstr=#99000000)";color: white;font: 10px arial, san serif;text-align: left;white-space: nowrap;padding: 5px;border: 1px solid white;z-index: 10000;}.jqsfield { color: white;font: 10px arial, san serif;text-align: left;}',d=function(){var b,c;return b=function(){this.init.apply(this,arguments)},arguments.length>1?(arguments[0]?(b.prototype=a.extend(new arguments[0],arguments[arguments.length-1]),b._super=arguments[0].prototype):b.prototype=arguments[arguments.length-1],arguments.length>2&&(c=Array.prototype.slice.call(arguments,1,-1),c.unshift(b.prototype),a.extend.apply(a,c))):b.prototype=arguments[0],b.prototype.cls=b,b},a.SPFormatClass=e=d({fre:/\{\{([\w.]+?)(:(.+?))?\}\}/g,precre:/(\w+)\.(\d+)/,init:function(a,b){this.format=a,this.fclass=b},render:function(a,b,c){var d=this,e=a,f,g,h,i,j;return this.format.replace(this.fre,function(){var a;return g=arguments[1],h=arguments[3],f=d.precre.exec(g),f?(j=f[2],g=f[1]):j=!1,i=e[g],i===undefined?"":h&&b&&b[h]?(a=b[h],a.get?b[h].get(i)||i:b[h][i]||i):(k(i)&&(c.get("numberFormatter")?i=c.get("numberFormatter")(i):i=p(i,j,c.get("numberDigitGroupCount"),c.get("numberDigitGroupSep"),c.get("numberDecimalMark"))),i)})}}),a.spformat=function(a,b){return new e(a,b)},f=function(a,b,c){return ac?c:a},g=function(a,b){var c;return b===2?(c=Math.floor(a.length/2),a.length%2?a[c]:(a[c-1]+a[c])/2):a.length%2?(c=(a.length*b+b)/4,c%1?(a[Math.floor(c)]+a[Math.floor(c)-1])/2:a[c-1]):(c=(a.length*b+2)/4,c%1?(a[Math.floor(c)]+a[Math.floor(c)-1])/2:a[c-1])},h=function(a){var b;switch(a){case"undefined":a=undefined;break;case"null":a=null;break;case"true":a=!0;break;case"false":a=!1;break;default:b=parseFloat(a),a==b&&(a=b)}return a},i=function(a){var b,c=[];for(b=a.length;b--;)c[b]=h(a[b]);return c},j=function(a,b){var c,d,e=[];for(c=0,d=a.length;c0;h-=d)b.splice(h,0,e);return b.join("")},l=function(a,b,c){var d;for(d=b.length;d--;){if(c&&b[d]===null)continue;if(b[d]!==a)return!1}return!0},m=function(a){var b=0,c;for(c=a.length;c--;)b+=typeof a[c]=="number"?a[c]:0;return b},o=function(b){return a.isArray(b)?b:[b]},n=function(a){var b;document.createStyleSheet?document.createStyleSheet().cssText=a:(b=document.createElement("style"),b.type="text/css",document.getElementsByTagName("head")[0].appendChild(b),b[typeof document.body.style.WebkitAppearance=="string"?"innerText":"innerHTML"]=a)},a.fn.simpledraw=function(b,c,d,e){var f,g;if(d&&(f=this.data("_jqs_vcanvas")))return f;b===undefined&&(b=a(this).innerWidth()),c===undefined&&(c=a(this).innerHeight());if(a.fn.sparkline.hasCanvas)f=new F(b,c,this,e);else{if(!a.fn.sparkline.hasVML)return!1;f=new G(b,c,this)}return g=a(this).data("_jqs_mhandler"),g&&g.registerCanvas(f),f},a.fn.cleardraw=function(){var a=this.data("_jqs_vcanvas");a&&a.reset()},a.RangeMapClass=q=d({init:function(a){var b,c,d=[];for(b in a)a.hasOwnProperty(b)&&typeof b=="string"&&b.indexOf(":")>-1&&(c=b.split(":"),c[0]=c[0].length===0?-Infinity:parseFloat(c[0]),c[1]=c[1].length===0?Infinity:parseFloat(c[1]),c[2]=a[b],d.push(c));this.map=a,this.rangelist=d||!1},get:function(a){var b=this.rangelist,c,d,e;if((e=this.map[a])!==undefined)return e;if(b)for(c=b.length;c--;){d=b[c];if(d[0]<=a&&d[1]>=a)return d[2]}return undefined}}),a.range_map=function(a){return new q(a)},r=d({init:function(b,c){var d=a(b);this.$el=d,this.options=c,this.currentPageX=0,this.currentPageY=0,this.el=b,this.splist=[],this.tooltip=null,this.over=!1,this.displayTooltips=!c.get("disableTooltips"),this.highlightEnabled=!c.get("disableHighlight")},registerSparkline:function(a){this.splist.push(a),this.over&&this.updateDisplay()},registerCanvas:function(b){var c=a(b.canvas);this.canvas=b,this.$canvas=c,c.mouseenter(a.proxy(this.mouseenter,this)),c.mouseleave(a.proxy(this.mouseleave,this)),c.click(a.proxy(this.mouseclick,this))},reset:function(a){this.splist=[],this.tooltip&&a&&(this.tooltip.remove(),this.tooltip=undefined)},mouseclick:function(b){var c=a.Event("sparklineClick");c.originalEvent=b,c.sparklines=this.splist,this.$el.trigger(c)},mouseenter:function(b){a(document.body).unbind("mousemove.jqs"),a(document.body).bind("mousemove.jqs",a.proxy(this.mousemove,this)),this.over=!0,this.currentPageX=b.pageX,this.currentPageY=b.pageY,this.currentEl=b.target,!this.tooltip&&this.displayTooltips&&(this.tooltip=new s(this.options),this.tooltip.updatePosition(b.pageX,b.pageY)),this.updateDisplay()},mouseleave:function(){a(document.body).unbind("mousemove.jqs");var b=this.splist,c=b.length,d=!1,e,f;this.over=!1,this.currentEl=null,this.tooltip&&(this.tooltip.remove(),this.tooltip=null);for(f=0;f