from direct.showbase.DirectObject import DirectObject from direct.stdpy.threading import Lock, RLock from direct.showbase.MessengerGlobal import messenger from direct.task.TaskManagerGlobal import taskMgr from direct.p3d.PackageInfo import PackageInfo from pandac.PandaModules import TPLow, PStatCollector from direct.directnotify.DirectNotifyGlobal import directNotify class PackageInstaller(DirectObject): """ This class is used in a p3d runtime environment to manage the asynchronous download and installation of packages. If you just want to install a package synchronously, see appRunner.installPackage() for a simpler interface. To use this class, you should subclass from it and override any of the six callback methods: downloadStarted(), packageStarted(), packageProgress(), downloadProgress(), packageFinished(), downloadFinished(). Also see DWBPackageInstaller, which does exactly this, to add a DirectWaitBar GUI. """ notify = directNotify.newCategory("PackageInstaller") globalLock = Lock() nextUniqueId = 1 # This is a chain of state values progressing forward in time. S_initial = 0 # addPackage() calls are being made S_ready = 1 # donePackages() has been called S_started = 2 # download has started S_done = 3 # download is over class PendingPackage: """ This class describes a package added to the installer for download. """ notify = directNotify.newCategory("PendingPackage") def __init__(self, packageName, version, host): self.packageName = packageName self.version = version self.host = host # This will be filled in properly by checkDescFile() or # getDescFile(); in the meantime, set a placeholder. self.package = PackageInfo(host, packageName, version) # Set true when the package has finished downloading, # either successfully or unsuccessfully. self.done = False # Set true or false when self.done has been set. self.success = False # Set true when the packageFinished() callback has been # delivered. self.notified = False # These are used to ensure the callbacks only get # delivered once for a particular package. self.calledPackageStarted = False self.calledPackageFinished = False # This is the amount of stuff we have to process to # install this package, and the amount of stuff we have # processed so far. "Stuff" includes bytes downloaded, # bytes uncompressed, and bytes extracted; and each of # which is weighted differently into one grand total. So, # the total doesn't really represent bytes; it's a # unitless number, which means something only as a ratio # to other packages. Filled in by checkDescFile() or # getDescFile(). self.downloadEffort = 0 # Similar, but this is the theoretical effort if the # package were already downloaded. self.prevDownloadedEffort = 0 def __cmp__(self, pp): """ Python comparision function. This makes all PendingPackages withe same (packageName, version, host) combination be deemed equivalent. """ return cmp((self.packageName, self.version, self.host), (pp.packageName, pp.version, pp.host)) def getProgress(self): """ Returns the download progress of this package in the range 0..1. """ return self.package.downloadProgress def checkDescFile(self): """ Returns true if the desc file is already downloaded and good, or false if it needs to be downloaded. """ if not self.host.hasCurrentContentsFile(): # If the contents file isn't ready yet, we can't check # the desc file yet. return False # All right, get the package info now. package = self.host.getPackage(self.packageName, self.version) if not package: self.notify.warning("Package %s %s not known on %s" % ( self.packageName, self.version, self.host.hostUrl)) return False self.package = package self.package.checkStatus() if not self.package.hasDescFile: return False self.downloadEffort = self.package.getDownloadEffort() self.prevDownloadEffort = 0 if self.downloadEffort == 0: self.prevDownloadedEffort = self.package.getPrevDownloadedEffort() return True def getDescFile(self, http): """ Synchronously downloads the desc files required for the package. """ if not self.host.downloadContentsFile(http): return False # All right, get the package info now. package = self.host.getPackage(self.packageName, self.version) if not package: self.notify.warning("Package %s %s not known on %s" % ( self.packageName, self.version, self.host.hostUrl)) return False self.package = package if not self.package.downloadDescFile(http): return False self.package.checkStatus() self.downloadEffort = self.package.getDownloadEffort() self.prevDownloadEffort = 0 if self.downloadEffort == 0: self.prevDownloadedEffort = self.package.getPrevDownloadedEffort() return True def __init__(self, appRunner, taskChain = 'default'): self.globalLock.acquire() try: self.uniqueId = PackageInstaller.nextUniqueId PackageInstaller.nextUniqueId += 1 finally: self.globalLock.release() self.appRunner = appRunner self.taskChain = taskChain # If we're to be running on an asynchronous task chain, and # the task chain hasn't yet been set up already, create the # default parameters now. if taskChain != 'default' and not taskMgr.hasTaskChain(self.taskChain): taskMgr.setupTaskChain(self.taskChain, numThreads = 1, threadPriority = TPLow) self.callbackLock = Lock() self.calledDownloadStarted = False self.calledDownloadFinished = False # A list of all packages that have been added to the # installer. self.packageLock = RLock() self.packages = [] self.state = self.S_initial # A list of packages that are waiting for their desc files. self.needsDescFile = [] self.descFileTask = None # A list of packages that are waiting to be downloaded and # installed. self.needsDownload = [] self.downloadTask = None # A list of packages that were already done at the time they # were passed to addPackage(). self.earlyDone = [] # A list of packages that have been successfully installed, or # packages that have failed. self.done = [] self.failed = [] # This task is spawned on the default task chain, to update # the status during the download. self.progressTask = None self.accept('PackageInstaller-%s-allHaveDesc' % self.uniqueId, self.__allHaveDesc) self.accept('PackageInstaller-%s-packageStarted' % self.uniqueId, self.__packageStarted) self.accept('PackageInstaller-%s-packageDone' % self.uniqueId, self.__packageDone) def destroy(self): """ Interrupts all pending downloads. No further callbacks will be made. """ self.cleanup() def cleanup(self): """ Interrupts all pending downloads. No further callbacks will be made. """ self.packageLock.acquire() try: if self.descFileTask: taskMgr.remove(self.descFileTask) self.descFileTask = None if self.downloadTask: taskMgr.remove(self.downloadTask) self.downloadTask = None finally: self.packageLock.release() if self.progressTask: taskMgr.remove(self.progressTask) self.progressTask = None self.ignoreAll() def addPackage(self, packageName, version = None, hostUrl = None): """ Adds the named package to the list of packages to be downloaded. Call donePackages() to finish the list. """ if self.state != self.S_initial: raise ValueError, 'addPackage called after donePackages' host = self.appRunner.getHostWithAlt(hostUrl) pp = self.PendingPackage(packageName, version, host) self.packageLock.acquire() try: self.__internalAddPackage(pp) finally: self.packageLock.release() def __internalAddPackage(self, pp): """ Adds the indicated "pending package" to the appropriate list(s) for downloading and installing. Assumes packageLock is already held.""" if pp in self.packages: # Already added. return self.packages.append(pp) # We always add the package to needsDescFile, even if we # already have its desc file; this guarantees that packages # are downloaded in the order they are added. self.needsDescFile.append(pp) if not self.descFileTask: self.descFileTask = taskMgr.add( self.__getDescFileTask, 'getDescFile', taskChain = self.taskChain) def donePackages(self): """ After calling addPackage() for each package to be installed, call donePackages() to mark the end of the list. This is necessary to determine what the complete set of packages is (and therefore how large the total download size is). None of the low-level callbacks will be made before this call. """ if self.state != self.S_initial: # We've already been here. return # Throw the messages for packages that were already done # before we started. for pp in self.earlyDone: self.__donePackage(pp, True) self.earlyDone = [] self.packageLock.acquire() try: if self.state != self.S_initial: return self.state = self.S_ready if not self.needsDescFile: # All package desc files are already available; so begin. self.__prepareToStart() finally: self.packageLock.release() if not self.packages: # Trivial no-op. self.__callDownloadFinished(True) def downloadStarted(self): """ This callback is made at some point after donePackages() is called; at the time of this callback, the total download size is known, and we can sensibly report progress through the whole. """ self.notify.info("downloadStarted") def packageStarted(self, package): """ This callback is made for each package between downloadStarted() and downloadFinished() to indicate the start of a new package. """ self.notify.debug("packageStarted: %s" % (package.packageName)) def packageProgress(self, package, progress): """ This callback is made repeatedly between packageStarted() and packageFinished() to update the current progress on the indicated package only. The progress value ranges from 0 (beginning) to 1 (complete). """ self.notify.debug("packageProgress: %s %s" % (package.packageName, progress)) def downloadProgress(self, overallProgress): """ This callback is made repeatedly between downloadStarted() and downloadFinished() to update the current progress through all packages. The progress value ranges from 0 (beginning) to 1 (complete). """ self.notify.debug("downloadProgress: %s" % (overallProgress)) def packageFinished(self, package, success): """ This callback is made for each package between downloadStarted() and downloadFinished() to indicate that a package has finished downloading. If success is true, there were no problems and the package is now installed. If this package did not require downloading (because it was already downloaded), this callback will be made immediately, *without* a corresponding call to packageStarted(), and may even be made before downloadStarted(). """ self.notify.info("packageFinished: %s %s" % (package.packageName, success)) def downloadFinished(self, success): """ This callback is made when all of the packages have been downloaded and installed (or there has been some failure). If all packages where successfully installed, success is True. If there were no packages that required downloading, this callback will be made immediately, *without* a corresponding call to downloadStarted(). """ self.notify.info("downloadFinished: %s" % (success)) def __prepareToStart(self): """ This is called internally when transitioning from S_ready to S_started. It sets up whatever initial values are needed. Assumes self.packageLock is held. Returns False if there were no packages to download, and the state was therefore transitioned immediately to S_done. """ if not self.needsDownload: self.state = self.S_done return False self.state = self.S_started assert not self.downloadTask self.downloadTask = taskMgr.add( self.__downloadPackageTask, 'downloadPackage', taskChain = self.taskChain) assert not self.progressTask self.progressTask = taskMgr.add( self.__progressTask, 'packageProgress') return True def __allHaveDesc(self): """ This method is called internally when all of the pending packages have their desc info. """ working = True self.packageLock.acquire() try: if self.state == self.S_ready: # We've already called donePackages(), so move on now. working = self.__prepareToStart() finally: self.packageLock.release() if not working: self.__callDownloadFinished(True) def __packageStarted(self, pp): """ This method is called when a single package is beginning to download. """ self.__callDownloadStarted() self.__callPackageStarted(pp) def __packageDone(self, pp): """ This method is called when a single package has been downloaded and installed, or has failed. """ self.__callPackageFinished(pp, pp.success) pp.notified = True # See if there are more packages to go. success = True allDone = True self.packageLock.acquire() try: for pp in self.packages: if pp.notified: success = success and pp.success else: allDone = False finally: self.packageLock.release() if allDone: self.__callDownloadFinished(success) def __callPackageStarted(self, pp): """ Calls the packageStarted() callback for a particular package if it has not already been called, being careful to avoid race conditions. """ self.callbackLock.acquire() try: if not pp.calledPackageStarted: self.packageStarted(pp.package) self.packageProgress(pp.package, 0) pp.calledPackageStarted = True finally: self.callbackLock.release() def __callPackageFinished(self, pp, success): """ Calls the packageFinished() callback for a paricular package if it has not already been called, being careful to avoid race conditions. """ self.callbackLock.acquire() try: if not pp.calledPackageFinished: if success: self.packageProgress(pp.package, 1) self.packageFinished(pp.package, success) pp.calledPackageFinished = True finally: self.callbackLock.release() def __callDownloadStarted(self): """ Calls the downloadStarted() callback if it has not already been called, being careful to avoid race conditions. """ self.callbackLock.acquire() try: if not self.calledDownloadStarted: self.downloadStarted() self.downloadProgress(0) self.calledDownloadStarted = True finally: self.callbackLock.release() def __callDownloadFinished(self, success): """ Calls the downloadFinished() callback if it has not already been called, being careful to avoid race conditions. """ self.callbackLock.acquire() try: if not self.calledDownloadFinished: if success: self.downloadProgress(1) self.downloadFinished(success) self.calledDownloadFinished = True finally: self.callbackLock.release() def __getDescFileTask(self, task): """ This task runs on the aysynchronous task chain; each pass, it extracts one package from self.needsDescFile and downloads its desc file. On success, it adds the package to self.needsDownload. """ self.packageLock.acquire() try: # If we've finished all of the packages that need desc # files, stop the task. if not self.needsDescFile: self.descFileTask = None eventName = 'PackageInstaller-%s-allHaveDesc' % self.uniqueId messenger.send(eventName, taskChain = 'default') return task.done pp = self.needsDescFile[0] del self.needsDescFile[0] finally: self.packageLock.release() # Now serve this one package. if not pp.checkDescFile(): if not pp.getDescFile(self.appRunner.http): self.__donePackage(pp, False) return task.cont # This package is now ready to be downloaded. We always add # it to needsDownload, even if it's already downloaded, to # guarantee ordering of packages. self.packageLock.acquire() try: # Also add any packages required by this one. for packageName, version, host in pp.package.requires: pp2 = self.PendingPackage(packageName, version, host) self.__internalAddPackage(pp2) self.needsDownload.append(pp) finally: self.packageLock.release() return task.cont def __downloadPackageTask(self, task): """ This task runs on the aysynchronous task chain; each pass, it extracts one package from self.needsDownload and downloads it. """ while True: self.packageLock.acquire() try: # If we're done downloading, stop the task. if self.state == self.S_done or not self.needsDownload: self.downloadTask = None self.packageLock.release() yield task.done; return assert self.state == self.S_started pp = self.needsDownload[0] del self.needsDownload[0] except: self.packageLock.release() raise self.packageLock.release() # Now serve this one package. eventName = 'PackageInstaller-%s-packageStarted' % self.uniqueId messenger.send(eventName, [pp], taskChain = 'default') if not pp.package.hasPackage: for token in pp.package.downloadPackageGenerator(self.appRunner.http): if token == pp.package.stepContinue: yield task.cont else: break if token != pp.package.stepComplete: pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName)) pc.start() self.__donePackage(pp, False) pc.stop() yield task.cont continue # Successfully downloaded and installed. pc = PStatCollector(':App:PackageInstaller:donePackage:%s' % (pp.package.packageName)) pc.start() self.__donePackage(pp, True) pc.stop() # Continue the loop without yielding, so we pick up the # next package within this same frame. def __donePackage(self, pp, success): """ Marks the indicated package as done, either successfully or otherwise. """ assert not pp.done if success: pc = PStatCollector(':App:PackageInstaller:install:%s' % (pp.package.packageName)) pc.start() pp.package.installPackage(self.appRunner) pc.stop() self.packageLock.acquire() try: pp.done = True pp.success = success if success: self.done.append(pp) else: self.failed.append(pp) finally: self.packageLock.release() eventName = 'PackageInstaller-%s-packageDone' % self.uniqueId messenger.send(eventName, [pp], taskChain = 'default') def __progressTask(self, task): self.callbackLock.acquire() try: if not self.calledDownloadStarted: # We haven't yet officially started the download. return task.cont if self.calledDownloadFinished: # We've officially ended the download. self.progressTask = None return task.done downloadEffort = 0 currentDownloadSize = 0 for pp in self.packages: downloadEffort += pp.downloadEffort + pp.prevDownloadedEffort packageProgress = pp.getProgress() currentDownloadSize += pp.downloadEffort * packageProgress + pp.prevDownloadedEffort if pp.calledPackageStarted and not pp.calledPackageFinished: self.packageProgress(pp.package, packageProgress) if not downloadEffort: progress = 1 else: progress = float(currentDownloadSize) / float(downloadEffort) self.downloadProgress(progress) finally: self.callbackLock.release() return task.cont