"""Undocumented Module""" __all__ = ['FakeObject', '_createGarbage', 'GarbageReport', 'GarbageLogger'] from direct.directnotify.DirectNotifyGlobal import directNotify from direct.showbase.PythonUtil import safeRepr, fastRepr, printListEnumGen, printNumberedTypesGen from direct.showbase.PythonUtil import AlphabetCounter from direct.showbase.Job import Job import gc import types GarbageCycleCountAnnounceEvent = 'announceGarbageCycleDesc2num' class FakeObject: pass class FakeDelObject: def __del__(self): pass def _createGarbage(num=1): for i in xrange(num): a = FakeObject() b = FakeObject() a.other = b b.other = a a = FakeDelObject() b = FakeDelObject() a.other = b b.other = a class GarbageReport(Job): """Detects leaked Python objects (via gc.collect()) and reports on garbage items, garbage-to-garbage references, and garbage cycles. If you just want to dump the report to the log, use GarbageLogger.""" notify = directNotify.newCategory("GarbageReport") def __init__(self, name, log=True, verbose=False, fullReport=False, findCycles=True, threaded=False, doneCallback=None, autoDestroy=False, priority=None, safeMode=False, delOnly=False, collect=True): # if autoDestroy is True, GarbageReport will self-destroy after logging # if false, caller is responsible for calling destroy() # if threaded is True, processing will be performed over multiple frames # if collect is False, we assume that the caller just did a collect and the results # are still in gc.garbage Job.__init__(self, name) # stick the arguments onto a ScratchPad so we can delete them all at once self._args = ScratchPad(name=name, log=log, verbose=verbose, fullReport=fullReport, findCycles=findCycles, doneCallback=doneCallback, autoDestroy=autoDestroy, safeMode=safeMode, delOnly=delOnly, collect=collect) if priority is not None: self.setPriority(priority) jobMgr.add(self) if not threaded: jobMgr.finish(self) def run(self): # do the garbage collection oldFlags = gc.get_debug() if self._args.delOnly: # do a collect without SAVEALL, to identify the instances that are involved in # cycles with instances that define __del__ # cycles that do not involve any instances that define __del__ are cleaned up # automatically by Python, but they also appear in gc.garbage when SAVEALL is set gc.set_debug(0) if self._args.collect: gc.collect() garbageInstances = gc.garbage[:] del gc.garbage[:] # only yield if there's more time-consuming work to do, # if there's no garbage, give instant feedback if len(garbageInstances) > 0: yield None # don't repr the garbage list if we don't have to if self.notify.getDebug(): self.notify.debug('garbageInstances == %s' % fastRepr(garbageInstances)) self.numGarbageInstances = len(garbageInstances) # grab the ids of the garbage instances (objects with __del__) self.garbageInstanceIds = set() for i in xrange(len(garbageInstances)): self.garbageInstanceIds.add(id(garbageInstances[i])) if not (i % 20): yield None # then release the list of instances so that it doesn't interfere with the gc.collect() below del garbageInstances else: self.garbageInstanceIds = set() # do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles # without SAVEALL, gc.garbage only contains objects with __del__ methods gc.set_debug(gc.DEBUG_SAVEALL) if self._args.collect: gc.collect() self.garbage = gc.garbage[:] del gc.garbage[:] # only yield if there's more time-consuming work to do, # if there's no garbage, give instant feedback if len(self.garbage) > 0: yield None # don't repr the garbage list if we don't have to if self.notify.getDebug(): self.notify.debug('self.garbage == %s' % fastRepr(self.garbage)) gc.set_debug(oldFlags) self.numGarbage = len(self.garbage) # only yield if there's more time-consuming work to do, # if there's no garbage, give instant feedback if self.numGarbage > 0: yield None if self._args.verbose: self.notify.info('found %s garbage items' % self.numGarbage) """ spammy # print the types of the garbage first, in case the repr of an object # causes a crash if self.numGarbage > 0: self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):') for result in printNumberedTypesGen(self.garbage): yield None """ # Py obj id -> garbage list index self._id2index = {} self.referrersByReference = {} self.referrersByNumber = {} self.referentsByReference = {} self.referentsByNumber = {} self._id2garbageInfo = {} self.cycles = [] self.cyclesBySyntax = [] self.uniqueCycleSets = set() self.cycleIds = set() # make the id->index table to speed up the next steps for i in xrange(self.numGarbage): self._id2index[id(self.garbage[i])] = i if not (i % 20): yield None # grab the referrers (pointing to garbage) if self._args.fullReport and (self.numGarbage != 0): if self._args.verbose: self.notify.info('getting referrers...') for i in xrange(self.numGarbage): yield None for result in self._getReferrers(self.garbage[i]): yield None byNum, byRef = result self.referrersByNumber[i] = byNum self.referrersByReference[i] = byRef # grab the referents (pointed to by garbage) if self.numGarbage > 0: if self._args.verbose: self.notify.info('getting referents...') for i in xrange(self.numGarbage): yield None for result in self._getReferents(self.garbage[i]): yield None byNum, byRef = result self.referentsByNumber[i] = byNum self.referentsByReference[i] = byRef for i in xrange(self.numGarbage): if hasattr(self.garbage[i], '_garbageInfo') and callable(self.garbage[i]._garbageInfo): try: info = self.garbage[i]._garbageInfo() except Exception, e: info = str(e) self._id2garbageInfo[id(self.garbage[i])] = info yield None else: if not (i % 20): yield None # find the cycles if self._args.findCycles and self.numGarbage > 0: if self._args.verbose: self.notify.info('calculating cycles...') for i in xrange(self.numGarbage): yield None for newCycles in self._getCycles(i, self.uniqueCycleSets): yield None self.cycles.extend(newCycles) # create a representation of the cycle in human-readable form newCyclesBySyntax = [] for cycle in newCycles: cycleBySyntax = '' objs = [] # leave off the last index, it's a repeat of the first index for index in cycle[:-1]: objs.append(self.garbage[index]) yield None # make the list repeat so we can safely iterate off the end numObjs = len(objs) - 1 objs.extend(objs) # state variables for our loop below numToSkip = 0 objAlreadyRepresented = False # if cycle starts off with an instance dict, start with the instance instead startIndex = 0 # + 1 to include a reference back to the first object endIndex = numObjs + 1 if type(objs[-1]) is types.InstanceType and type(objs[0]) is types.DictType: startIndex -= 1 endIndex -= 1 for index in xrange(startIndex, endIndex): if numToSkip: numToSkip -= 1 continue obj = objs[index] if type(obj) is types.InstanceType: if not objAlreadyRepresented: cycleBySyntax += '%s' % obj.__class__.__name__ cycleBySyntax += '.' # skip past the instance dict and get the member obj numToSkip += 1 member = objs[index+2] for key, value in obj.__dict__.iteritems(): if value is member: break yield None else: key = '' cycleBySyntax += '%s' % key objAlreadyRepresented = True elif type(obj) is types.DictType: cycleBySyntax += '{' # get object referred to by dict val = objs[index+1] for key, value in obj.iteritems(): if value is val: break yield None else: key = '' cycleBySyntax += '%s}' % fastRepr(key) objAlreadyRepresented = True elif type(obj) in (types.TupleType, types.ListType): brackets = { types.TupleType: '()', types.ListType: '[]', }[type(obj)] # get object being referenced by container nextObj = objs[index+1] cycleBySyntax += brackets[0] for index in xrange(len(obj)): if obj[index] is nextObj: index = str(index) break yield None else: index = '' cycleBySyntax += '%s%s' % (index, brackets[1]) objAlreadyRepresented = True else: cycleBySyntax += '%s --> ' % itype(obj) objAlreadyRepresented = False newCyclesBySyntax.append(cycleBySyntax) yield None self.cyclesBySyntax.extend(newCyclesBySyntax) # if we're not doing a full report, add this cycle's IDs to the master set if not self._args.fullReport: for cycle in newCycles: yield None self.cycleIds.update(set(cycle)) self.numCycles = len(self.cycles) if self._args.findCycles: s = ['===== GarbageReport: \'%s\' (%s %s) =====' % ( self._args.name, self.numCycles, choice(self.numCycles == 1, 'cycle', 'cycles'))] else: s = ['===== GarbageReport: \'%s\' =====' % ( self._args.name)] if self.numGarbage > 0: # make a list of the ids we will actually be printing if self._args.fullReport: garbageIndices = range(self.numGarbage) else: garbageIndices = list(self.cycleIds) garbageIndices.sort() numGarbage = len(garbageIndices) # log each individual item with a number in front of it if not self._args.fullReport: abbrev = '(abbreviated) ' else: abbrev = '' s.append('===== Garbage Items %s=====' % abbrev) digits = 0 n = numGarbage while n > 0: yield None digits += 1 n /= 10 digits = digits format = '%0' + '%s' % digits + 'i:%s \t%s' for i in xrange(numGarbage): yield None idx = garbageIndices[i] if self._args.safeMode: # in safe mode, don't try to repr any of the objects objStr = repr(itype(self.garbage[idx])) else: objStr = fastRepr(self.garbage[idx]) maxLen = 5000 if len(objStr) > maxLen: snip = '' objStr = '%s%s' % (objStr[:(maxLen-len(snip))], snip) s.append(format % (idx, itype(self.garbage[idx]), objStr)) # also log the types of the objects s.append('===== Garbage Item Types %s=====' % abbrev) for i in xrange(numGarbage): yield None idx = garbageIndices[i] objStr = str(deeptype(self.garbage[idx])) maxLen = 5000 if len(objStr) > maxLen: snip = '' objStr = '%s%s' % (objStr[:(maxLen-len(snip))], snip) s.append(format % (idx, itype(self.garbage[idx]), objStr)) if self._args.findCycles: s.append('===== Garbage Cycles (Garbage Item Numbers) =====') ac = AlphabetCounter() for i in xrange(self.numCycles): yield None s.append('%s:%s' % (ac.next(), self.cycles[i])) if self._args.findCycles: s.append('===== Garbage Cycles (Python Syntax) =====') ac = AlphabetCounter() for i in xrange(len(self.cyclesBySyntax)): yield None s.append('%s:%s' % (ac.next(), self.cyclesBySyntax[i])) if len(self._id2garbageInfo): s.append('===== Garbage Custom Info =====') ac = AlphabetCounter() for i in xrange(len(self.cyclesBySyntax)): yield None counter = ac.next() _id = id(self.garbage[i]) if _id in self._id2garbageInfo: s.append('%s:%s' % (counter, self._id2garbageInfo[_id])) if self._args.fullReport: format = '%0' + '%s' % digits + 'i:%s' s.append('===== Referrers By Number (what is referring to garbage item?) =====') for i in xrange(numGarbage): yield None s.append(format % (i, self.referrersByNumber[i])) s.append('===== Referents By Number (what is garbage item referring to?) =====') for i in xrange(numGarbage): yield None s.append(format % (i, self.referentsByNumber[i])) s.append('===== Referrers (what is referring to garbage item?) =====') for i in xrange(numGarbage): yield None s.append(format % (i, self.referrersByReference[i])) s.append('===== Referents (what is garbage item referring to?) =====') for i in xrange(numGarbage): yield None s.append(format % (i, self.referentsByReference[i])) self._report = s if self._args.log: self.printingBegin() for i in xrange(len(self._report)): if self.numGarbage > 0: yield None self.notify.info(self._report[i]) self.notify.info('===== Garbage Report Done =====') self.printingEnd() yield Job.Done def finished(self): if self._args.doneCallback: self._args.doneCallback(self) if self._args.autoDestroy: self.destroy() def destroy(self): #print 'GarbageReport.destroy' del self._args del self.garbage # don't get rid of these, we might need them #del self.numGarbage #del self.numCycles del self.referrersByReference del self.referrersByNumber del self.referentsByReference del self.referentsByNumber if hasattr(self, 'cycles'): del self.cycles del self._report if hasattr(self, '_reportStr'): del self._reportStr Job.destroy(self) def getNumCycles(self): # if the job hasn't run yet, we don't have a numCycles yet return self.numCycles def getDesc2numDict(self): # dict of python-syntax leak -> number of that type of leak desc2num = {} for cycleBySyntax in self.cyclesBySyntax: desc2num.setdefault(cycleBySyntax, 0) desc2num[cycleBySyntax] += 1 return desc2num def getGarbage(self): return self.garbage def getReport(self): if not hasattr(self, '_reportStr'): self._reportStr = '' for str in self._report: self._reportStr += '\n' + str return self._reportStr def _getReferrers(self, obj): # referrers (pointing to garbage) # returns two lists, first by index into gc.garbage, second by # direct reference yield None byRef = gc.get_referrers(obj) yield None # look to see if each referrer is another garbage item byNum = [] for i in xrange(len(byRef)): if not (i % 20): yield None referrer = byRef[i] num = self._id2index.get(id(referrer), None) byNum.append(num) yield byNum, byRef def _getReferents(self, obj): # referents (pointed to by garbage) # returns two lists, first by index into gc.garbage, second by # direct reference yield None byRef = gc.get_referents(obj) yield None # look to see if each referent is another garbage item byNum = [] for i in xrange(len(byRef)): if not (i % 20): yield None referent = byRef[i] num = self._id2index.get(id(referent), None) byNum.append(num) yield byNum, byRef def _getNormalizedCycle(self, cycle): # returns a representation of a cycle (list of indices) that will be # reliably derived from a unique cycle regardless of ordering # this lets us detect duplicate cycles that appear different because of # which element appears first if len(cycle) == 0: return cycle min = 1<<30 minIndex = None for i in xrange(len(cycle)): elem = cycle[i] if elem < min: min = elem minIndex = i return cycle[minIndex:] + cycle[:minIndex] def _getCycles(self, index, uniqueCycleSets=None): # detect garbage cycles for a particular item of garbage assert self.notify.debugCall() # returns list of lists, sublists are garbage reference cycles cycles = [] # this lets us eliminate duplicate cycles if uniqueCycleSets is None: uniqueCycleSets = set() stateStack = Stack() rootId = index # check if the root object is one of the garbage instances (has __del__) objId = id(self.garbage[rootId]) numDelInstances = choice(objId in self.garbageInstanceIds, 1, 0) stateStack.push(([rootId], rootId, numDelInstances, 0)) while True: yield None if len(stateStack) == 0: break candidateCycle, curId, numDelInstances, resumeIndex = stateStack.pop() if self.notify.getDebug(): if self._args.delOnly: print 'restart: %s root=%s cur=%s numDelInstances=%s resume=%s' % ( candidateCycle, rootId, curId, numDelInstances, resumeIndex) else: print 'restart: %s root=%s cur=%s resume=%s' % ( candidateCycle, rootId, curId, resumeIndex) for index in xrange(resumeIndex, len(self.referentsByNumber[curId])): yield None refId = self.referentsByNumber[curId][index] if self.notify.getDebug(): print ' : %s -> %s' % (curId, refId) if refId == rootId: # we found a cycle! mark it down and move on to the next refId normCandidateCycle = self._getNormalizedCycle(candidateCycle) normCandidateCycleTuple = tuple(normCandidateCycle) if not normCandidateCycleTuple in uniqueCycleSets: # cycles with no instances that define __del__ will be # cleaned up by Python if (not self._args.delOnly) or numDelInstances >= 1: if self.notify.getDebug(): print ' FOUND: ', normCandidateCycle + [normCandidateCycle[0],] cycles.append(normCandidateCycle + [normCandidateCycle[0],]) uniqueCycleSets.add(normCandidateCycleTuple) elif refId in candidateCycle: pass elif refId is not None: # check if this object is one of the garbage instances (has __del__) objId = id(self.garbage[refId]) numDelInstances += choice(objId in self.garbageInstanceIds, 1, 0) # this refId does not complete a cycle. Mark down # where we are in this list of referents, then # start looking through the referents of the new refId stateStack.push((list(candidateCycle), curId, numDelInstances, index+1)) stateStack.push((list(candidateCycle) + [refId], refId, numDelInstances, 0)) break yield cycles class GarbageLogger(GarbageReport): """If you just want to log the current garbage to the log file, make one of these. It automatically destroys itself after logging""" def __init__(self, name, *args, **kArgs): kArgs['log'] = True kArgs['autoDestroy'] = True GarbageReport.__init__(self, name, *args, **kArgs) class _CFGLGlobals: # for checkForGarbageLeaks LastNumGarbage = 0 LastNumCycles = 0 def checkForGarbageLeaks(): gc.collect() numGarbage = len(gc.garbage) if (numGarbage > 0 and config.GetBool('auto-garbage-logging', 0)): if (numGarbage != _CFGLGlobals.LastNumGarbage): print gr = GarbageReport('found garbage', threaded=False, collect=False) print _CFGLGlobals.LastNumGarbage = numGarbage _CFGLGlobals.LastNumCycles = gr.getNumCycles() messenger.send(GarbageCycleCountAnnounceEvent, [gr.getDesc2numDict()]) gr.destroy() notify = directNotify.newCategory("GarbageDetect") if config.GetBool('allow-garbage-cycles', 1): func = notify.warning else: func = notify.error func('%s garbage cycles found, see info above' % _CFGLGlobals.LastNumCycles) return numGarbage def b_checkForGarbageLeaks(wantReply=False): if not __dev__: return 0 # does a garbage collect on the client and the AI # returns number of client garbage leaks # logs leak info and terminates (if configured to do so) try: # if this is the client, tell the AI to check for leaks too base.cr.timeManager except: pass else: if base.cr.timeManager: base.cr.timeManager.d_checkForGarbageLeaks(wantReply=wantReply) return checkForGarbageLeaks()