""" This module is used to build a "Package", a collection of files within a Panda3D Multifile, which can be easily be downloaded and/or patched onto a client machine, for the purpose of running a large application. """ # Important to import panda3d first, to avoid naming conflicts with # Python's "string" and "Loader" names that are imported later. from panda3d.core import * import sys import os import glob import marshal import string import types import getpass import platform import struct from direct.p3d.FileSpec import FileSpec from direct.p3d.SeqValue import SeqValue from direct.showbase import Loader from direct.showbase import AppRunnerGlobal from direct.showutil import FreezeTool from direct.directnotify.DirectNotifyGlobal import * vfs = VirtualFileSystem.getGlobalPtr() class PackagerError(StandardError): pass class OutsideOfPackageError(PackagerError): pass class ArgumentError(PackagerError): pass class Packager: notify = directNotify.newCategory("Packager") class PackFile: def __init__(self, package, filename, newName = None, deleteTemp = False, explicit = False, compress = None, extract = None, text = None, unprocessed = None, executable = None, dependencyDir = None, platformSpecific = None, required = False): assert isinstance(filename, Filename) self.filename = Filename(filename) self.newName = newName self.deleteTemp = deleteTemp self.explicit = explicit self.compress = compress self.extract = extract self.text = text self.unprocessed = unprocessed self.executable = executable self.dependencyDir = dependencyDir self.platformSpecific = platformSpecific self.required = required if not self.newName: self.newName = self.filename.cStr() ext = Filename(self.newName).getExtension() if ext == 'pz': # Strip off a .pz extension; we can compress files # within the Multifile without it. filename = Filename(self.newName) filename.setExtension('') self.newName = filename.cStr() ext = Filename(self.newName).getExtension() if self.compress is None: self.compress = True packager = package.packager if self.compress is None: self.compress = (ext not in packager.uncompressibleExtensions and ext not in packager.imageExtensions) if self.executable is None: self.executable = (ext in packager.executableExtensions) if self.executable and self.dependencyDir is None: # By default, install executable dependencies in the # same directory with the executable itself. self.dependencyDir = Filename(self.newName).getDirname() if self.extract is None: self.extract = self.executable or (ext in packager.extractExtensions) if self.platformSpecific is None: self.platformSpecific = self.executable or (ext in packager.platformSpecificExtensions) if self.unprocessed is None: self.unprocessed = self.executable or (ext in packager.unprocessedExtensions) if self.executable: # Look up the filename along the system PATH, if necessary. if not self.filename.resolveFilename(packager.executablePath): # If it wasn't found, try looking it up under its # basename only. Sometimes a Mac user will copy # the library file out of a framework and put that # along the PATH, instead of the framework itself. basename = Filename(self.filename.getBasename()) if basename.resolveFilename(packager.executablePath): self.filename = basename if ext in packager.textExtensions and not self.executable: self.filename.setText() else: self.filename.setBinary() # Convert the filename to an unambiguous filename for # searching. self.filename.makeTrueCase() if self.filename.exists() or not self.filename.isLocal(): self.filename.makeCanonical() def isExcluded(self, package): """ Returns true if this file should be excluded or skipped, false otherwise. """ if self.newName.lower() in package.skipFilenames: return True if not self.explicit: # Make sure it's not one of our auto-excluded system # files. (But only make this check if this file was # not explicitly added.) basename = Filename(self.newName).getBasename() if not package.packager.caseSensitive: basename = basename.lower() if basename in package.packager.excludeSystemFiles: return True for exclude in package.packager.excludeSystemGlobs: if exclude.matches(basename): return True # Also check if it was explicitly excluded. As above, # omit this check for an explicitly-added file: if you # both include and exclude a file, the file is # included. for exclude in package.excludedFilenames: if exclude.matches(self.filename): return True # A platform-specific file is implicitly excluded from # not-platform-specific packages. if self.platformSpecific and package.platformSpecificConfig is False: return True return False class ExcludeFilename: def __init__(self, packager, filename, caseSensitive): self.packager = packager self.localOnly = (not filename.getDirname()) if not self.localOnly: filename = Filename(filename) filename.makeCanonical() self.glob = GlobPattern(filename.cStr()) if self.packager.platform.startswith('win'): self.glob.setCaseSensitive(False) elif self.packager.platform.startswith('osx'): self.glob.setCaseSensitive(False) def matches(self, filename): if self.localOnly: return self.glob.matches(filename.getBasename()) else: return self.glob.matches(filename.cStr()) class PackageEntry: """ This corresponds to a entry in the contents.xml file. """ def __init__(self): # The "seq" value increments automatically with each publish. self.packageSeq = SeqValue() # The "set_ver" value is optionally specified in the pdef # file and does not change unless the user says it does. self.packageSetVer = SeqValue() def getKey(self): """ Returns a tuple used for sorting the PackageEntry objects uniquely per package. """ return (self.packageName, self.platform, self.version) def fromFile(self, packageName, platform, version, solo, perPlatform, installDir, descFilename, importDescFilename): self.packageName = packageName self.platform = platform self.version = version self.solo = solo self.perPlatform = perPlatform self.descFile = FileSpec() self.descFile.fromFile(installDir, descFilename) self.importDescFile = None if importDescFilename: self.importDescFile = FileSpec() self.importDescFile.fromFile(installDir, importDescFilename) def loadXml(self, xpackage): self.packageName = xpackage.Attribute('name') self.platform = xpackage.Attribute('platform') self.version = xpackage.Attribute('version') solo = xpackage.Attribute('solo') self.solo = int(solo or '0') perPlatform = xpackage.Attribute('per_platform') self.perPlatform = int(perPlatform or '0') self.packageSeq = SeqValue() self.packageSeq.loadXml(xpackage, 'seq') self.packageSetVer = SeqValue() self.packageSetVer.loadXml(xpackage, 'set_ver') self.descFile = FileSpec() self.descFile.loadXml(xpackage) self.importDescFile = None ximport = xpackage.FirstChildElement('import') if ximport: self.importDescFile = FileSpec() self.importDescFile.loadXml(ximport) def makeXml(self): """ Returns a new TiXmlElement. """ xpackage = TiXmlElement('package') xpackage.SetAttribute('name', self.packageName) if self.platform: xpackage.SetAttribute('platform', self.platform) if self.version: xpackage.SetAttribute('version', self.version) if self.solo: xpackage.SetAttribute('solo', '1') if self.perPlatform: xpackage.SetAttribute('per_platform', '1') self.packageSeq.storeXml(xpackage, 'seq') self.packageSetVer.storeXml(xpackage, 'set_ver') self.descFile.storeXml(xpackage) if self.importDescFile: ximport = TiXmlElement('import') self.importDescFile.storeXml(ximport) xpackage.InsertEndChild(ximport) return xpackage class HostEntry: def __init__(self, url = None, downloadUrl = None, descriptiveName = None, hostDir = None, mirrors = None): self.url = url self.downloadUrl = downloadUrl self.descriptiveName = descriptiveName self.hostDir = hostDir self.mirrors = mirrors or [] self.altHosts = {} def loadXml(self, xhost, packager): self.url = xhost.Attribute('url') self.downloadUrl = xhost.Attribute('download_url') self.descriptiveName = xhost.Attribute('descriptive_name') self.hostDir = xhost.Attribute('host_dir') self.mirrors = [] xmirror = xhost.FirstChildElement('mirror') while xmirror: url = xmirror.Attribute('url') self.mirrors.append(url) xmirror = xmirror.NextSiblingElement('mirror') xalthost = xhost.FirstChildElement('alt_host') while xalthost: url = xalthost.Attribute('url') he = packager.addHost(url) he.loadXml(xalthost, packager) xalthost = xalthost.NextSiblingElement('alt_host') def makeXml(self, packager = None): """ Returns a new TiXmlElement. """ xhost = TiXmlElement('host') xhost.SetAttribute('url', self.url) if self.downloadUrl and self.downloadUrl != self.url: xhost.SetAttribute('download_url', self.downloadUrl) if self.descriptiveName: xhost.SetAttribute('descriptive_name', self.descriptiveName) if self.hostDir: xhost.SetAttribute('host_dir', self.hostDir) for mirror in self.mirrors: xmirror = TiXmlElement('mirror') xmirror.SetAttribute('url', mirror) xhost.InsertEndChild(xmirror) if packager: altHosts = self.altHosts.items() altHosts.sort() for keyword, alt in altHosts: he = packager.hosts.get(alt, None) if he: xalthost = he.makeXml() xalthost.SetValue('alt_host') xalthost.SetAttribute('keyword', keyword) xhost.InsertEndChild(xalthost) return xhost class Package: """ This is the full information on a particular package we are constructing. Don't confuse it with PackageEntry, above, which contains only the information found in the toplevel contents.xml file.""" def __init__(self, packageName, packager): self.packageName = packageName self.packager = packager self.notify = packager.notify # The platform is initially None until we know the file is # platform-specific. self.platform = None # This is always true on modern packages. self.perPlatform = True # The arch string, though, is pre-loaded from the system # arch string, so we can sensibly call otool. self.arch = self.packager.arch self.version = None self.host = None self.p3dApplication = False self.solo = False self.compressionLevel = 0 self.importedMapsDir = 'imported_maps' self.mainModule = None self.signParams = [] self.requires = [] # This may be set explicitly in the pdef file to a # particular sequence value. self.packageSetVer = SeqValue() # This is the set of config variables assigned to the # package. self.configs = {} # This is the set of files and modules, already included # by required packages, that we can skip. self.skipFilenames = {} self.skipModules = {} # This is a list of ExcludeFilename objects, representing # the files that have been explicitly excluded. self.excludedFilenames = [] # This is the list of files we will be adding, and a pair # of cross references. self.files = [] self.sourceFilenames = {} self.targetFilenames = {} # This is the set of files and modules that are # required and may not be excluded from the package. self.requiredFilenames = [] self.requiredModules = [] # This records the current list of modules we have added so # far. self.freezer = FreezeTool.Freezer(platform = self.packager.platform) def close(self): """ Writes out the contents of the current package. Returns True if the package was constructed successfully, False if one or more required files or modules are missing. """ if not self.p3dApplication and not self.packager.allowPackages: message = 'Cannot generate packages without an installDir; use -i' raise PackagerError, message if not self.host: self.host = self.packager.host # Check the version config variable. version = self.configs.get('version', None) if version is not None: self.version = version del self.configs['version'] # Check the platform_specific config variable. This has # only three settings: None (unset), True, or False. self.platformSpecificConfig = self.configs.get('platform_specific', None) if self.platformSpecificConfig is not None: # First, convert it to an int, in case it's "0" or "1". try: self.platformSpecificConfig = int(self.platformSpecificConfig) except ValueError: pass # Then, make it a bool. self.platformSpecificConfig = bool(self.platformSpecificConfig) del self.configs['platform_specific'] # A special case when building the "panda3d" package. We # enforce that the version number matches what we've been # compiled with. if self.packageName == 'panda3d': if self.version is None: self.version = PandaSystem.getPackageVersionString() if self.version != PandaSystem.getPackageVersionString(): message = 'mismatched Panda3D version: requested %s, but Panda3D is built as %s' % (self.version, PandaSystem.getPackageVersionString()) raise PackagerError, message if self.host != PandaSystem.getPackageHostUrl(): message = 'mismatched Panda3D host: requested %s, but Panda3D is built as %s' % (self.host, PandaSystem.getPackageHostUrl()) raise PackagerError, message if self.p3dApplication: # Default compression level for an app. self.compressionLevel = 6 # Every p3dapp requires panda3d. if 'panda3d' not in [p.packageName for p in self.requires]: assert not self.packager.currentPackage self.packager.currentPackage = self self.packager.do_require('panda3d') self.packager.currentPackage = None # If this flag is set, enable allow_python_dev. if self.packager.allowPythonDev: self.configs['allow_python_dev'] = True if not self.p3dApplication and not self.version: # If we don't have an implicit version, inherit the # version from the 'panda3d' package on our require # list. for p2 in self.requires: if p2.packageName == 'panda3d' and p2.version: self.version = p2.version break if self.solo: result = self.installSolo() else: result = self.installMultifile() if self.p3dApplication: allowPythonDev = self.configs.get('allow_python_dev', 0) if int(allowPythonDev): print "\n*** Generating %s.p3d with allow_python_dev enabled ***\n" % (self.packageName) return result def considerPlatform(self): # Check to see if any of the files are platform-specific, # making the overall package platform-specific. platformSpecific = self.platformSpecificConfig for file in self.files: if file.isExcluded(self): # Skip this file. continue if file.platformSpecific: platformSpecific = True if platformSpecific and self.platformSpecificConfig is not False: if not self.platform: self.platform = self.packager.platform if self.platform and self.platform.startswith('osx_'): # Get the OSX "arch" specification. self.arch = self.platform[4:] def installMultifile(self): """ Installs the package, either as a p3d application, or as a true package. Either is implemented with a Multifile. """ self.multifile = Multifile() # Write the multifile to a temporary filename until we # know enough to determine the output filename. multifileFilename = Filename.temporary('', self.packageName + '.', '.mf') self.multifile.openReadWrite(multifileFilename) if self.p3dApplication: # p3d files should be tagged to make them executable. self.multifile.setHeaderPrefix('#! /usr/bin/env panda3d\n') else: # Package multifiles might be patched, and therefore # don't want to record an internal timestamp, which # would make patching less efficient. self.multifile.setRecordTimestamp(False) # Make sure that all required files are present. missing = [] for file in self.requiredFilenames: if file not in self.files or file.isExcluded(self): missing.append(file.filename.getBasename()) if len(missing) > 0: self.notify.warning("Cannot build package %s, missing required files: %r" % (self.packageName, missing)) self.cleanup() return False self.extracts = [] self.components = [] # Add the explicit py files that were requested by the # pdef file. These get turned into Python modules. for file in self.files: if file.isExcluded(self): # Skip this file. continue if file.unprocessed: # Unprocessed files get dealt with below. continue ext = Filename(file.newName).getExtension() if ext == 'dc': # Add the modules named implicitly in the dc file. self.addDcImports(file) elif ext == 'py': self.addPyFile(file) # Add the main module, if any. if not self.mainModule and self.p3dApplication: message = 'No main_module specified for application %s' % (self.packageName) raise PackagerError, message if self.mainModule: moduleName, newName = self.mainModule if newName not in self.freezer.modules: self.freezer.addModule(moduleName, newName = newName) # Now all module files have been added. Exclude modules # already imported in a required package, and not # explicitly included by this package. for moduleName, mdef in self.skipModules.items(): if moduleName not in self.freezer.modules: self.freezer.excludeModule( moduleName, allowChildren = mdef.allowChildren, forbid = mdef.forbid, fromSource = 'skip') # Pick up any unfrozen Python files. self.freezer.done() # But first, make sure that all required modules are present. missing = [] moduleDict = dict(self.freezer.getModuleDefs()).keys() for module in self.requiredModules: if module not in moduleDict: missing.append(module) if len(missing) > 0: self.notify.warning("Cannot build package %s, missing required modules: %r" % (self.packageName, missing)) self.cleanup() return False # OK, we can add it. self.freezer.addToMultifile(self.multifile, self.compressionLevel) self.addExtensionModules() # Add known module names. self.moduleNames = {} modules = self.freezer.modules.items() modules.sort() for newName, mdef in modules: if mdef.guess: # Not really a module. continue if mdef.fromSource == 'skip': # This record already appeared in a required # module; don't repeat it now. continue if mdef.exclude and mdef.implicit: # Don't bother mentioning implicitly-excluded # (i.e. missing) modules. continue if newName == '__main__': # Ignore this special case. continue self.moduleNames[newName] = mdef xmodule = TiXmlElement('module') xmodule.SetAttribute('name', newName) if mdef.exclude: xmodule.SetAttribute('exclude', '1') if mdef.forbid: xmodule.SetAttribute('forbid', '1') if mdef.exclude and mdef.allowChildren: xmodule.SetAttribute('allowChildren', '1') self.components.append(('m', newName.lower(), xmodule)) # Now look for implicit shared-library dependencies. if self.packager.platform.startswith('win'): self.__addImplicitDependenciesWindows() elif self.packager.platform.startswith('osx'): self.__addImplicitDependenciesOSX() else: self.__addImplicitDependenciesPosix() # Now add all the real, non-Python files (except model # files). This will include the extension modules we just # discovered above. for file in self.files: if file.isExcluded(self): # Skip this file. continue ext = Filename(file.newName).getExtension() if file.unprocessed: # Add an unprocessed file verbatim. self.addComponent(file) elif ext == 'py': # Already handled, above. pass elif file.isExcluded(self): # Skip this file. pass elif ext == 'egg' or ext == 'bam': # Skip model files this pass. pass elif ext == 'dc': # dc files get a special treatment. self.addDcFile(file) elif ext == 'prc': # So do prc files. self.addPrcFile(file) else: # Any other file. self.addComponent(file) # Now add the model files. It's important to add these # after we have added all of the texture files, so we can # determine which textures need to be implicitly pulled # in. # We walk through a copy of the files list, since we might # be adding more files (textures) to this list as we # discover them in model files referenced in this list. for file in self.files[:]: if file.isExcluded(self): # Skip this file. continue ext = Filename(file.newName).getExtension() if file.unprocessed: # Already handled, above. pass elif ext == 'py': # Already handled, above. pass elif file.isExcluded(self): # Skip this file. pass elif ext == 'egg': self.addEggFile(file) elif ext == 'bam': self.addBamFile(file) else: # Handled above. pass # Check to see if we should be platform-specific. self.considerPlatform() # Now that we've processed all of the component files, # (and set our platform if necessary), we can generate the # output filename and write the output files. self.packageBasename = self.packageName packageDir = self.packageName if self.version: self.packageBasename += '.' + self.version packageDir += '/' + self.version if self.platform: self.packageBasename += '.' + self.platform packageDir += '/' + self.platform self.packageDesc = self.packageBasename + '.xml' self.packageImportDesc = self.packageBasename + '.import.xml' if self.p3dApplication: self.packageBasename += self.packager.p3dSuffix self.packageBasename += '.p3d' packageDir = '' else: self.packageBasename += '.mf' packageDir += '/' self.packageDir = packageDir self.packageFilename = packageDir + self.packageBasename self.packageDesc = packageDir + self.packageDesc self.packageImportDesc = packageDir + self.packageImportDesc print "Generating %s" % (self.packageFilename) if self.p3dApplication: self.packageFullpath = Filename(self.packager.p3dInstallDir, self.packageFilename) self.packageFullpath.makeDir() self.makeP3dInfo() else: self.packageFullpath = Filename(self.packager.installDir, self.packageFilename) self.packageFullpath.makeDir() self.multifile.repack() # Also sign the multifile before we close it. for certificate, chain, pkey, password in self.signParams: self.multifile.addSignature(certificate, chain or '', pkey or '', password or '') self.multifile.close() if not multifileFilename.renameTo(self.packageFullpath): self.notify.error("Cannot move %s to %s" % (multifileFilename, self.packageFullpath)) if self.p3dApplication: # No patches for an application; just move it into place. # Make the application file executable. os.chmod(self.packageFullpath.toOsSpecific(), 0o755) else: self.readDescFile() self.packageSeq += 1 self.perPlatform = True # always true on modern packages. self.compressMultifile() self.writeDescFile() self.writeImportDescFile() # Now that we've written out the desc file, we don't # need to keep around the uncompressed archive # anymore. self.packageFullpath.unlink() # Replace or add the entry in the contents. pe = Packager.PackageEntry() pe.fromFile(self.packageName, self.platform, self.version, False, self.perPlatform, self.packager.installDir, self.packageDesc, self.packageImportDesc) pe.packageSeq = self.packageSeq pe.packageSetVer = self.packageSetVer self.packager.contents[pe.getKey()] = pe self.packager.contentsChanged = True self.cleanup() return True def installSolo(self): """ Installs the package as a "solo", which means we simply copy the one file into the install directory. This is primarily intended for the "coreapi" plugin, which is just a single dll and a jpg file; but it can support other kinds of similar "solo" packages as well. """ self.considerPlatform() self.perPlatform = False # Not true on "solo" packages. packageDir = self.packageName if self.platform: packageDir += '/' + self.platform if self.version: packageDir += '/' + self.version if not self.packager.allowPackages: message = 'Cannot generate packages without an installDir; use -i' raise PackagerError, message installPath = Filename(self.packager.installDir, packageDir) # Remove any files already in the installPath. origFiles = vfs.scanDirectory(installPath) if origFiles: for origFile in origFiles: origFile.getFilename().unlink() files = [] for file in self.files: if file.isExcluded(self): # Skip this file. continue files.append(file) if not files: # No files, never mind. return if len(files) != 1: raise PackagerError, 'Multiple files in "solo" package %s' % (self.packageName) Filename(installPath, '').makeDir() file = files[0] targetPath = Filename(installPath, file.newName) targetPath.setBinary() file.filename.setBinary() if not file.filename.copyTo(targetPath): self.notify.warning("Could not copy %s to %s" % ( file.filename, targetPath)) # Replace or add the entry in the contents. pe = Packager.PackageEntry() pe.fromFile(self.packageName, self.platform, self.version, True, self.perPlatform, self.packager.installDir, Filename(packageDir, file.newName), None) peOrig = self.packager.contents.get(pe.getKey(), None) if peOrig: pe.packageSeq = peOrig.packageSeq + 1 pe.packageSetVer = peOrig.packageSetVer if self.packageSetVer: pe.packageSetVer = self.packageSetVer self.packager.contents[pe.getKey()] = pe self.packager.contentsChanged = True self.cleanup() return True def cleanup(self): # Now that all the files have been packed, we can delete # the temporary files. for file in self.files: if file.deleteTemp: file.filename.unlink() def addFile(self, *args, **kw): """ Adds the named file to the package. Returns the file object, or None if it was not added by this call. """ file = Packager.PackFile(self, *args, **kw) if file.filename in self.sourceFilenames: # Don't bother, it's already here. return None lowerName = file.newName.lower() if lowerName in self.targetFilenames: # Another file is already in the same place. file2 = self.targetFilenames[lowerName] self.packager.notify.warning( "%s is shadowing %s" % (file2.filename, file.filename)) return None self.sourceFilenames[file.filename] = file if file.required: self.requiredFilenames.append(file) if file.text is None and not file.filename.exists(): if not file.isExcluded(self): self.packager.notify.warning("No such file: %s" % (file.filename)) return None self.files.append(file) self.targetFilenames[lowerName] = file return file def excludeFile(self, filename): """ Excludes the named file (or glob pattern) from the package. """ xfile = Packager.ExcludeFilename(self.packager, filename, self.packager.caseSensitive) self.excludedFilenames.append(xfile) def __addImplicitDependenciesWindows(self): """ Walks through the list of files, looking for dll's and exe's that might include implicit dependencies on other dll's and assembly manifests. Tries to determine those dependencies, and adds them back into the filelist. """ # We walk through the list as we modify it. That's OK, # because we want to follow the transitive closure of # dependencies anyway. for file in self.files: if not file.executable: continue if file.isExcluded(self): # Skip this file. continue if file.filename.getExtension().lower() == "manifest": filenames = self.__parseManifest(file.filename) if filenames is None: self.notify.warning("Unable to determine dependent assemblies from %s" % (file.filename)) continue else: tempFile = Filename.temporary('', 'p3d_', '.txt') command = 'dumpbin /dependents "%s" >"%s"' % ( file.filename.toOsSpecific(), tempFile.toOsSpecific()) try: os.system(command) except: pass filenames = None if tempFile.exists(): filenames = self.__parseDependenciesWindows(tempFile) tempFile.unlink() if filenames is None: self.notify.warning("Unable to determine dependencies from %s" % (file.filename)) filenames = [] # Extract the manifest file so we can figure out # the dependent assemblies. tempFile = Filename.temporary('', 'p3d_', '.manifest') resindex = 2 if file.filename.getExtension().lower() == "exe": resindex = 1 command = 'mt -inputresource:"%s";#%d -out:"%s" > nul' % ( file.filename.toOsSpecific(), resindex, tempFile.toOsSpecific()) try: out = os.system(command) except: pass afilenames = None if tempFile.exists(): afilenames = self.__parseManifest(tempFile) tempFile.unlink() # Also check for an explicit private-assembly # manifest file on disk. mfile = file.filename + '.manifest' if mfile.exists(): if afilenames is None: afilenames = [] afilenames += self.__parseManifest(mfile) # Since it's an explicit manifest file, it # means we should include the manifest # file itself in the package. newName = Filename(file.dependencyDir, mfile.getBasename()) self.addFile(mfile, newName = newName.cStr(), explicit = False, executable = True) if afilenames is None and out != 31: self.notify.warning("Unable to determine dependent assemblies from %s" % (file.filename)) if afilenames is not None: filenames += afilenames # Attempt to resolve the dependent filename relative # to the original filename, before we resolve it along # the PATH. path = DSearchPath(Filename(file.filename.getDirname())) for filename in filenames: filename = Filename.fromOsSpecific(filename) filename.resolveFilename(path) filename.makeTrueCase() newName = Filename(file.dependencyDir, filename.getBasename()) self.addFile(filename, newName = newName.cStr(), explicit = False, executable = True) def __parseDependenciesWindows(self, tempFile): """ Reads the indicated temporary file, the output from dumpbin /dependents, to determine the list of dll's this executable file depends on. """ lines = open(tempFile.toOsSpecific(), 'rU').readlines() li = 0 while li < len(lines): line = lines[li] li += 1 if line.find(' has the following dependencies') != -1: break if li < len(lines): line = lines[li] if line.strip() == '': # Skip a blank line. li += 1 # Now we're finding filenames, until the next blank line. filenames = [] while li < len(lines): line = lines[li] li += 1 line = line.strip() if line == '': # We're done. return filenames filenames.append(line) # Hmm, we ran out of data. Oh well. if not filenames: # Some parse error. return None # At least we got some data. return filenames def __parseManifest(self, tempFile): """ Reads the indicated application manifest file, to determine the list of dependent assemblies this executable file depends on. """ doc = TiXmlDocument(tempFile.toOsSpecific()) if not doc.LoadFile(): return None assembly = doc.FirstChildElement("assembly") if not assembly: return None # Pick up assemblies that it depends on filenames = [] dependency = assembly.FirstChildElement("dependency") while dependency: depassembly = dependency.FirstChildElement("dependentAssembly") if depassembly: ident = depassembly.FirstChildElement("assemblyIdentity") if ident: name = ident.Attribute("name") if name: filenames.append(name + ".manifest") dependency = dependency.NextSiblingElement("dependency") # Pick up direct dll dependencies that it lists dfile = assembly.FirstChildElement("file") while dfile: name = dfile.Attribute("name") if name: filenames.append(name) dfile = dfile.NextSiblingElement("file") return filenames def __locateFrameworkLibrary(self, library): """ Locates the given library inside its framework on the default framework paths, and returns its location as Filename. """ # If it's already a full existing path, we # don't search for it anymore, of course. if Filename.fromOsSpecific(library).exists(): return Filename.fromOsSpecific(library) # DSearchPath appears not to work for directories. fpath = [] fpath.append(Filename("/Library/Frameworks")) fpath.append(Filename("/System/Library/Frameworks")) fpath.append(Filename("/Developer/Library/Frameworks")) fpath.append(Filename("/Users/%s" % getpass.getuser(), "Library/Frameworks")) if "HOME" in os.environ: fpath.append(Filename(os.environ["HOME"], "Library/Frameworks")) ffilename = Filename(library.split('.framework/', 1)[0].split('/')[-1] + '.framework') ffilename = Filename(ffilename, library.split('.framework/', 1)[-1]) # Look under the system root first, if supplied. if self.packager.systemRoot: for i in fpath: fw = Filename(self.packager.systemRoot, i) if Filename(fw, ffilename).exists(): return Filename(fw, ffilename) for i in fpath: if Filename(i, ffilename).exists(): return Filename(i, ffilename) # Not found? Well, let's just return the framework + file # path, the user will be presented with a warning later. return ffilename def __alterFrameworkDependencies(self, file, framework_deps): """ Copies the given library file to a temporary directory, and alters the dependencies so that it doesn't contain absolute framework dependencies. """ if not file.deleteTemp: # Copy the file to a temporary location because we # don't want to modify the original (there's a big # chance that we break it). # Copy it every time, because the source file might # have changed since last time we ran. assert file.filename.exists(), "File doesn't exist: %s" % file.filename tmpfile = Filename.temporary('', "p3d_" + file.filename.getBasename()) tmpfile.setBinary() file.filename.copyTo(tmpfile) file.filename = tmpfile file.deleteTemp = True # Alter the dependencies to have a relative path rather than absolute for filename in framework_deps: loc = self.__locateFrameworkLibrary(filename) if loc == file.filename: os.system('install_name_tool -id "%s" "%s"' % (os.path.basename(filename), file.filename.toOsSpecific())) elif "/System/" in loc.toOsSpecific(): # Let's keep references to system frameworks absolute os.system('install_name_tool -change "%s" "%s" "%s"' % (filename, loc.toOsSpecific(), file.filename.toOsSpecific())) else: os.system('install_name_tool -change "%s" "%s" "%s"' % (filename, os.path.basename(filename), file.filename.toOsSpecific())) def __addImplicitDependenciesOSX(self): """ Walks through the list of files, looking for dylib's and executables that might include implicit dependencies on other dylib's. Tries to determine those dependencies, and adds them back into the filelist. """ # We walk through the list as we modify it. That's OK, # because we want to follow the transitive closure of # dependencies anyway. for file in self.files: if not file.executable: continue if file.isExcluded(self): # Skip this file. continue tempFile = Filename.temporary('', 'p3d_', '.txt') command = '/usr/bin/otool -arch all -L "%s" >"%s"' % ( file.filename.toOsSpecific(), tempFile.toOsSpecific()) if self.arch: command = '/usr/bin/otool -arch %s -L "%s" >"%s"' % ( self.arch, file.filename.toOsSpecific(), tempFile.toOsSpecific()) exitStatus = os.system(command) if exitStatus != 0: self.notify.warning('Command failed: %s' % (command)) filenames = None if tempFile.exists(): filenames = self.__parseDependenciesOSX(tempFile) tempFile.unlink() if filenames is None: self.notify.warning("Unable to determine dependencies from %s" % (file.filename)) continue # Attempt to resolve the dependent filename relative # to the original filename, before we resolve it along # the PATH. path = DSearchPath(Filename(file.filename.getDirname())) # Find the dependencies that are referencing a framework framework_deps = [] for filename in filenames: if '.framework/' in filename: framework_deps.append(filename) if len(framework_deps) > 0: # Fixes dependencies like @executable_path/../Library/Frameworks/Cg.framework/Cg self.__alterFrameworkDependencies(file, framework_deps) for filename in filenames: if '.framework/' in filename: # It references a framework, and besides the fact # that those often contain absolute paths, they # aren't commonly on the library path either. filename = self.__locateFrameworkLibrary(filename) filename.setBinary() else: # It's just a normal library - find it on the path. filename = Filename.fromOsSpecific(filename) filename.setBinary() if filename.isLocal(): filename.resolveFilename(path) else: # It's a fully-specified filename; look # for it under the system root first. if self.packager.systemRoot: f2 = Filename(self.packager.systemRoot + filename.cStr()) if f2.exists(): filename = f2 # Skip libraries and frameworks in system directory if "/System/" in filename.toOsSpecific(): continue newName = Filename(file.dependencyDir, filename.getBasename()) self.addFile(filename, newName = newName.cStr(), explicit = False, executable = True) def __parseDependenciesOSX(self, tempFile): """ Reads the indicated temporary file, the output from otool -L, to determine the list of dylibs this executable file depends on. """ lines = open(tempFile.toOsSpecific(), 'rU').readlines() filenames = [] for line in lines: if line[0] not in string.whitespace: continue line = line.strip() s = line.find(' (compatibility') if s != -1: line = line[:s] else: s = line.find('.dylib') if s != -1: line = line[:s + 6] else: continue filenames.append(line) return filenames def __readAndStripELF(self, file): """ Reads the indicated ELF binary, and returns a list with dependencies. If it contains data that should be stripped, it writes the stripped library to a temporary file. Returns None if the file failed to read (e.g. not an ELF file). """ # Read the first 16 bytes, which identify the ELF file. elf = open(file.filename.toOsSpecific(), 'rb') try: ident = elf.read(16) except IOError: elf.close() return None if not ident.startswith("\177ELF"): # No elf magic! Beware of orcs. return None # Make sure we read in the correct endianness and integer size byteOrder = "<>"[ord(ident[5]) - 1] elfClass = ord(ident[4]) - 1 # 0 = 32-bits, 1 = 64-bits headerStruct = byteOrder + ("HHIIIIIHHHHHH", "HHIQQQIHHHHHH")[elfClass] sectionStruct = byteOrder + ("4xI8xIII8xI", "4xI16xQQI12xQ")[elfClass] dynamicStruct = byteOrder + ("iI", "qQ")[elfClass] type, machine, version, entry, phoff, shoff, flags, ehsize, phentsize, phnum, shentsize, shnum, shstrndx \ = struct.unpack(headerStruct, elf.read(struct.calcsize(headerStruct))) dynamicSections = [] stringTables = {} # Seek to the section header table and find the .dynamic section. elf.seek(shoff) for i in range(shnum): type, offset, size, link, entsize = struct.unpack_from(sectionStruct, elf.read(shentsize)) if type == 6 and link != 0: # DYNAMIC type, links to string table dynamicSections.append((offset, size, link, entsize)) stringTables[link] = None # Read the relevant string tables. for idx in stringTables.keys(): elf.seek(shoff + idx * shentsize) type, offset, size, link, entsize = struct.unpack_from(sectionStruct, elf.read(shentsize)) if type != 3: continue elf.seek(offset) stringTables[idx] = elf.read(size) # Loop through the dynamic sections and rewrite it if it has an rpath/runpath. rewriteSections = [] filenames = [] rpath = [] for offset, size, link, entsize in dynamicSections: elf.seek(offset) data = elf.read(entsize) tag, val = struct.unpack_from(dynamicStruct, data) newSectionData = "" startReplace = None pad = 0 # Read tags until we find a NULL tag. while tag != 0: if tag == 1: # A NEEDED entry. Read it from the string table. filenames.append(stringTables[link][val : stringTables[link].find('\0', val)]) elif tag == 15 or tag == 29: rpath += stringTables[link][val : stringTables[link].find('\0', val)].split(':') # An RPATH or RUNPATH entry. if not startReplace: startReplace = elf.tell() - entsize if startReplace: pad += entsize elif startReplace is not None: newSectionData += data data = elf.read(entsize) tag, val = struct.unpack_from(dynamicStruct, data) if startReplace is not None: newSectionData += data + ("\0" * pad) rewriteSections.append((startReplace, newSectionData)) elf.close() # No rpaths/runpaths found, so nothing to do any more. if len(rewriteSections) == 0: return filenames # Attempt to resolve any of the directly # dependent filenames along the RPATH. for f in range(len(filenames)): filename = filenames[f] for rdir in rpath: if os.path.isfile(os.path.join(rdir, filename)): filenames[f] = os.path.join(rdir, filename) break if not file.deleteTemp: # Copy the file to a temporary location because we # don't want to modify the original (there's a big # chance that we break it). tmpfile = Filename.temporary('', "p3d_" + file.filename.getBasename()) tmpfile.setBinary() file.filename.copyTo(tmpfile) file.filename = tmpfile file.deleteTemp = True # Open the temporary file and rewrite the dynamic sections. elf = open(file.filename.toOsSpecific(), 'r+b') for offset, data in rewriteSections: elf.seek(offset) elf.write(data) elf.write("\0" * pad) elf.close() return filenames def __addImplicitDependenciesPosix(self): """ Walks through the list of files, looking for so's and executables that might include implicit dependencies on other so's. Tries to determine those dependencies, and adds them back into the filelist. """ # We walk through the list as we modify it. That's OK, # because we want to follow the transitive closure of # dependencies anyway. for file in self.files: if not file.executable: continue if file.isExcluded(self): # Skip this file. continue # Check if this is an ELF binary. filenames = self.__readAndStripELF(file) # If that failed, perhaps ldd will help us. if filenames is None: tempFile = Filename.temporary('', 'p3d_', '.txt') command = 'ldd "%s" >"%s"' % ( file.filename.toOsSpecific(), tempFile.toOsSpecific()) try: os.system(command) except: pass if tempFile.exists(): filenames = self.__parseDependenciesPosix(tempFile) tempFile.unlink() if filenames is None: self.notify.warning("Unable to determine dependencies from %s" % (file.filename)) continue # Attempt to resolve the dependent filename relative # to the original filename, before we resolve it along # the PATH. path = DSearchPath(Filename(file.filename.getDirname())) for filename in filenames: # These vDSO's provided by Linux aren't # supposed to be anywhere on the system. if filename in ["linux-gate.so.1", "linux-vdso.so.1"]: continue filename = Filename.fromOsSpecific(filename) filename.resolveFilename(path) filename.setBinary() newName = Filename(file.dependencyDir, filename.getBasename()) self.addFile(filename, newName = newName.cStr(), explicit = False, executable = True) def __parseDependenciesPosix(self, tempFile): """ Reads the indicated temporary file, the output from ldd, to determine the list of so's this executable file depends on. """ lines = open(tempFile.toOsSpecific(), 'rU').readlines() filenames = [] for line in lines: line = line.strip() s = line.find(' => ') if s == -1: continue line = line[:s].strip() filenames.append(line) return filenames def addExtensionModules(self): """ Adds the extension modules detected by the freezer to the current list of files. """ freezer = self.freezer for moduleName, filename in freezer.extras: filename = Filename.fromOsSpecific(filename) newName = filename.getBasename() if '.' in moduleName: newName = '/'.join(moduleName.split('.')[:-1]) newName += '/' + filename.getBasename() # Sometimes the PYTHONPATH has the wrong case in it. filename.makeTrueCase() self.addFile(filename, newName = newName, explicit = False, extract = True, executable = True, platformSpecific = True) freezer.extras = [] def makeP3dInfo(self): """ Makes the p3d_info.xml file that defines the application startup parameters and such. """ doc = TiXmlDocument() decl = TiXmlDeclaration("1.0", "utf-8", "") doc.InsertEndChild(decl) xpackage = TiXmlElement('package') xpackage.SetAttribute('name', self.packageName) if self.platform: xpackage.SetAttribute('platform', self.platform) if self.version: xpackage.SetAttribute('version', self.version) xpackage.SetAttribute('main_module', self.mainModule[1]) self.__addConfigs(xpackage) requireHosts = {} for package in self.requires: xrequires = TiXmlElement('requires') xrequires.SetAttribute('name', package.packageName) if package.version: xrequires.SetAttribute('version', package.version) xrequires.SetAttribute('host', package.host) package.packageSeq.storeXml(xrequires, 'seq') package.packageSetVer.storeXml(xrequires, 'set_ver') requireHosts[package.host] = True xpackage.InsertEndChild(xrequires) for host in requireHosts.keys(): he = self.packager.hosts.get(host, None) if he: xhost = he.makeXml(packager = self.packager) xpackage.InsertEndChild(xhost) doc.InsertEndChild(xpackage) # Write the xml file to a temporary file on disk, so we # can add it to the multifile. filename = Filename.temporary('', 'p3d_', '.xml') # This should really be setText() for an xml file, but it # doesn't really matter that much since tinyxml can read # it either way; and if we use setBinary() it will remain # compatible with older versions of the core API that # didn't understand the SF_text flag. filename.setBinary() doc.SaveFile(filename.toOsSpecific()) # It's important not to compress this file: the core API # runtime can't decode compressed subfiles. self.multifile.addSubfile('p3d_info.xml', filename, 0) self.multifile.flush() filename.unlink() def compressMultifile(self): """ Compresses the .mf file into an .mf.pz file. """ if self.oldCompressedBasename: # Remove the previous compressed file first. compressedPath = Filename(self.packager.installDir, Filename(self.packageDir, self.oldCompressedBasename)) compressedPath.unlink() newCompressedFilename = '%s.pz' % (self.packageFilename) # Now build the new version. compressedPath = Filename(self.packager.installDir, newCompressedFilename) if not compressFile(self.packageFullpath, compressedPath, 6): message = 'Unable to write %s' % (compressedPath) raise PackagerError, message def readDescFile(self): """ Reads the existing package.xml file before rewriting it. We need this to preserve the list of patches, and similar historic data, between sessions. """ self.packageSeq = SeqValue() self.packageSetVer = SeqValue() self.patchVersion = None self.patches = [] self.oldCompressedBasename = None packageDescFullpath = Filename(self.packager.installDir, self.packageDesc) doc = TiXmlDocument(packageDescFullpath.toOsSpecific()) if not doc.LoadFile(): return xpackage = doc.FirstChildElement('package') if not xpackage: return perPlatform = xpackage.Attribute('per_platform') self.perPlatform = int(perPlatform or '0') self.packageSeq.loadXml(xpackage, 'seq') self.packageSetVer.loadXml(xpackage, 'set_ver') xcompressed = xpackage.FirstChildElement('compressed_archive') if xcompressed: compressedFilename = xcompressed.Attribute('filename') if compressedFilename: self.oldCompressedBasename = compressedFilename patchVersion = xpackage.Attribute('patch_version') if not patchVersion: patchVersion = xpackage.Attribute('last_patch_version') if patchVersion: self.patchVersion = patchVersion # Extract the base_version, top_version, and patch # entries, if any, and preserve these entries verbatim for # the next version. xbase = xpackage.FirstChildElement('base_version') if xbase: self.patches.append(xbase.Clone()) xtop = xpackage.FirstChildElement('top_version') if xtop: self.patches.append(xtop.Clone()) xpatch = xpackage.FirstChildElement('patch') while xpatch: self.patches.append(xpatch.Clone()) xpatch = xpatch.NextSiblingElement('patch') def writeDescFile(self): """ Makes the package.xml file that describes the package and its contents, for download. """ packageDescFullpath = Filename(self.packager.installDir, self.packageDesc) doc = TiXmlDocument(packageDescFullpath.toOsSpecific()) decl = TiXmlDeclaration("1.0", "utf-8", "") doc.InsertEndChild(decl) xpackage = TiXmlElement('package') xpackage.SetAttribute('name', self.packageName) if self.platform: xpackage.SetAttribute('platform', self.platform) if self.version: xpackage.SetAttribute('version', self.version) if self.perPlatform: xpackage.SetAttribute('per_platform', '1') if self.patchVersion: xpackage.SetAttribute('last_patch_version', self.patchVersion) self.packageSeq.storeXml(xpackage, 'seq') self.packageSetVer.storeXml(xpackage, 'set_ver') self.__addConfigs(xpackage) for package in self.requires: xrequires = TiXmlElement('requires') xrequires.SetAttribute('name', package.packageName) if self.platform and package.platform: xrequires.SetAttribute('platform', package.platform) if package.version: xrequires.SetAttribute('version', package.version) package.packageSeq.storeXml(xrequires, 'seq') package.packageSetVer.storeXml(xrequires, 'set_ver') xrequires.SetAttribute('host', package.host) xpackage.InsertEndChild(xrequires) xuncompressedArchive = self.getFileSpec( 'uncompressed_archive', self.packageFullpath, self.packageBasename) xpackage.InsertEndChild(xuncompressedArchive) xcompressedArchive = self.getFileSpec( 'compressed_archive', self.packageFullpath + '.pz', self.packageBasename + '.pz') xpackage.InsertEndChild(xcompressedArchive) # Copy in the patch entries read from the previous version # of the desc file. for xpatch in self.patches: xpackage.InsertEndChild(xpatch) self.extracts.sort() for name, xextract in self.extracts: xpackage.InsertEndChild(xextract) doc.InsertEndChild(xpackage) doc.SaveFile() def __addConfigs(self, xpackage): """ Adds the XML config values defined in self.configs to the indicated XML element. """ if self.configs: xconfig = TiXmlElement('config') for variable, value in self.configs.items(): if isinstance(value, types.UnicodeType): xconfig.SetAttribute(variable, value.encode('utf-8')) elif isinstance(value, types.BooleanType): # True or False must be encoded as 1 or 0. xconfig.SetAttribute(variable, str(int(value))) else: xconfig.SetAttribute(variable, str(value)) xpackage.InsertEndChild(xconfig) def writeImportDescFile(self): """ Makes the package.import.xml file that describes the package and its contents, for other packages and applications that may wish to "require" this one. """ packageImportDescFullpath = Filename(self.packager.installDir, self.packageImportDesc) doc = TiXmlDocument(packageImportDescFullpath.toOsSpecific()) decl = TiXmlDeclaration("1.0", "utf-8", "") doc.InsertEndChild(decl) xpackage = TiXmlElement('package') xpackage.SetAttribute('name', self.packageName) if self.platform: xpackage.SetAttribute('platform', self.platform) if self.version: xpackage.SetAttribute('version', self.version) xpackage.SetAttribute('host', self.host) self.packageSeq.storeXml(xpackage, 'seq') self.packageSetVer.storeXml(xpackage, 'set_ver') requireHosts = {} requireHosts[self.host] = True for package in self.requires: xrequires = TiXmlElement('requires') xrequires.SetAttribute('name', package.packageName) if self.platform and package.platform: xrequires.SetAttribute('platform', package.platform) if package.version: xrequires.SetAttribute('version', package.version) xrequires.SetAttribute('host', package.host) package.packageSeq.storeXml(xrequires, 'seq') package.packageSetVer.storeXml(xrequires, 'set_ver') requireHosts[package.host] = True xpackage.InsertEndChild(xrequires) # Make sure we also write the full host descriptions for # any hosts we reference, so we can find these guys later. for host in requireHosts.keys(): he = self.packager.hosts.get(host, None) if he: xhost = he.makeXml(packager = self.packager) xpackage.InsertEndChild(xhost) self.components.sort() for type, name, xcomponent in self.components: xpackage.InsertEndChild(xcomponent) doc.InsertEndChild(xpackage) doc.SaveFile() def readImportDescFile(self, filename): """ Reads the import desc file. Returns True on success, False on failure. """ self.packageSeq = SeqValue() self.packageSetVer = SeqValue() doc = TiXmlDocument(filename.toOsSpecific()) if not doc.LoadFile(): return False xpackage = doc.FirstChildElement('package') if not xpackage: return False self.packageName = xpackage.Attribute('name') self.platform = xpackage.Attribute('platform') self.version = xpackage.Attribute('version') self.host = xpackage.Attribute('host') # Get any new host descriptors. xhost = xpackage.FirstChildElement('host') while xhost: he = self.packager.HostEntry() he.loadXml(xhost, self) if he.url not in self.packager.hosts: self.packager.hosts[he.url] = he xhost = xhost.NextSiblingElement('host') self.packageSeq.loadXml(xpackage, 'seq') self.packageSetVer.loadXml(xpackage, 'set_ver') self.requires = [] xrequires = xpackage.FirstChildElement('requires') while xrequires: packageName = xrequires.Attribute('name') platform = xrequires.Attribute('platform') version = xrequires.Attribute('version') host = xrequires.Attribute('host') if packageName: package = self.packager.findPackage( packageName, platform = platform, version = version, host = host, requires = self.requires) if package: self.requires.append(package) xrequires = xrequires.NextSiblingElement('requires') self.targetFilenames = {} xcomponent = xpackage.FirstChildElement('component') while xcomponent: name = xcomponent.Attribute('filename') if name: self.targetFilenames[name.lower()] = True xcomponent = xcomponent.NextSiblingElement('component') self.moduleNames = {} xmodule = xpackage.FirstChildElement('module') while xmodule: moduleName = xmodule.Attribute('name') exclude = int(xmodule.Attribute('exclude') or 0) forbid = int(xmodule.Attribute('forbid') or 0) allowChildren = int(xmodule.Attribute('allowChildren') or 0) if moduleName: mdef = FreezeTool.Freezer.ModuleDef( moduleName, exclude = exclude, forbid = forbid, allowChildren = allowChildren) self.moduleNames[moduleName] = mdef xmodule = xmodule.NextSiblingElement('module') return True def getFileSpec(self, element, pathname, newName): """ Returns an xcomponent or similar element with the file information for the indicated file. """ xspec = TiXmlElement(element) size = pathname.getFileSize() timestamp = pathname.getTimestamp() hv = HashVal() hv.hashFile(pathname) hash = hv.asHex() xspec.SetAttribute('filename', newName) xspec.SetAttribute('size', str(size)) xspec.SetAttribute('timestamp', str(timestamp)) xspec.SetAttribute('hash', hash) return xspec def addPyFile(self, file): """ Adds the indicated python file, identified by filename instead of by module name, to the package. """ # Convert the raw filename back to a module name, so we # can see if we've already loaded this file. We assume # that all Python files within the package will be rooted # at the top of the package. filename = file.newName.rsplit('.', 1)[0] moduleName = filename.replace("/", ".") if moduleName.endswith('.__init__'): moduleName = moduleName.rsplit('.', 1)[0] if moduleName in self.freezer.modules: # This Python file is already known. We don't have to # deal with it again. return self.freezer.addModule(moduleName, filename = file.filename) def addEggFile(self, file): # Precompile egg files to bam's. np = self.packager.loader.loadModel(file.filename) if not np: raise StandardError, 'Could not read egg file %s' % (file.filename) bamName = Filename(file.newName) bamName.setExtension('bam') self.addNode(np.node(), file.filename, bamName.cStr()) def addBamFile(self, file): # Load the bam file so we can massage its textures. bamFile = BamFile() if not bamFile.openRead(file.filename): raise StandardError, 'Could not read bam file %s' % (file.filename) if not bamFile.resolve(): raise StandardError, 'Could not resolve bam file %s' % (file.filename) node = bamFile.readNode() if not node: raise StandardError, 'Not a model file: %s' % (file.filename) self.addNode(node, file.filename, file.newName) def addNode(self, node, filename, newName): """ Converts the indicated node to a bam stream, and adds the bam file to the multifile under the indicated newName. """ # If the Multifile already has a file by this name, don't # bother adding it again. if self.multifile.findSubfile(newName) >= 0: return # Be sure to import all of the referenced textures, and tell # them their new location within the multifile. for tex in NodePath(node).findAllTextures(): if not tex.hasFullpath() and tex.hasRamImage(): # We need to store this texture as a raw-data image. # Clear the newName so this will happen # automatically. tex.clearFilename() tex.clearAlphaFilename() else: # We can store this texture as a file reference to its # image. Copy the file into our multifile, and rename # its reference in the texture. if tex.hasFilename(): tex.setFilename(self.addFoundTexture(tex.getFullpath())) if tex.hasAlphaFilename(): tex.setAlphaFilename(self.addFoundTexture(tex.getAlphaFullpath())) # Now generate an in-memory bam file. Tell the bam writer to # keep the textures referenced by their in-multifile path. bamFile = BamFile() stream = StringStream() bamFile.openWrite(stream) bamFile.getWriter().setFileTextureMode(bamFile.BTMUnchanged) bamFile.writeObject(node) bamFile.close() # Clean the node out of memory. node.removeAllChildren() # Now we have an in-memory bam file. stream.seekg(0) self.multifile.addSubfile(newName, stream, self.compressionLevel) # Flush it so the data gets written to disk immediately, so we # don't have to keep it around in ram. self.multifile.flush() xcomponent = TiXmlElement('component') xcomponent.SetAttribute('filename', newName) self.components.append(('c', newName.lower(), xcomponent)) def addFoundTexture(self, filename): """ Adds the newly-discovered texture to the output, if it has not already been included. Returns the new name within the package tree. """ filename = Filename(filename) filename.makeCanonical() file = self.sourceFilenames.get(filename, None) if file: # Never mind, it's already on the list. return file.newName # We have to copy the image into the plugin tree somewhere. newName = self.importedMapsDir + '/' + filename.getBasename() uniqueId = 0 while newName.lower() in self.targetFilenames: uniqueId += 1 newName = '%s/%s_%s.%s' % ( self.importedMapsDir, filename.getBasenameWoExtension(), uniqueId, filename.getExtension()) file = self.addFile( filename, newName = newName, explicit = False, compress = False) if file: # If we added the file in this pass, then also # immediately add it to the multifile (because we # won't be visiting the files list again). self.addComponent(file) return newName def addDcFile(self, file): """ Adds a dc file to the archive. A dc file gets its internal comments and parameter names stripped out of the final result automatically. This is as close as we can come to "compiling" a dc file, since all of the remaining symbols are meaningful at runtime. """ # First, read in the dc file from panda3d.direct import DCFile dcFile = DCFile() if not dcFile.read(file.filename): self.notify.error("Unable to parse %s." % (file.filename)) # And then write it out without the comments and such. stream = StringStream() if not dcFile.write(stream, True): self.notify.error("Unable to write %s." % (file.filename)) file.text = stream.getData() self.addComponent(file) def addDcImports(self, file): """ Adds the Python modules named by the indicated dc file. """ from panda3d.direct import DCFile dcFile = DCFile() if not dcFile.read(file.filename): self.notify.error("Unable to parse %s." % (file.filename)) for n in range(dcFile.getNumImportModules()): moduleName = dcFile.getImportModule(n) moduleSuffixes = [] if '/' in moduleName: moduleName, suffixes = moduleName.split('/', 1) moduleSuffixes = suffixes.split('/') self.freezer.addModule(moduleName) for suffix in self.packager.dcClientSuffixes: if suffix in moduleSuffixes: self.freezer.addModule(moduleName + suffix) for i in range(dcFile.getNumImportSymbols(n)): symbolName = dcFile.getImportSymbol(n, i) symbolSuffixes = [] if '/' in symbolName: symbolName, suffixes = symbolName.split('/', 1) symbolSuffixes = suffixes.split('/') # "from moduleName import symbolName". # Maybe this symbol is itself a module; if that's # the case, we need to add it to the list also. self.freezer.addModule('%s.%s' % (moduleName, symbolName), implicit = True) for suffix in self.packager.dcClientSuffixes: if suffix in symbolSuffixes: self.freezer.addModule('%s.%s%s' % (moduleName, symbolName, suffix), implicit = True) def addPrcFile(self, file): """ Adds a prc file to the archive. Like the dc file, this strips comments and such before adding. It's also possible to set prcEncryptionKey and/or prcSignCommand to further manipulate prc files during processing. """ # First, read it in. if file.text: textLines = file.text.split('\n') else: textLines = open(file.filename.toOsSpecific(), 'rU').readlines() # Then write it out again, without the comments. tempFilename = Filename.temporary('', 'p3d_', '.prc') tempFilename.setBinary() # Binary is more reliable for signing. temp = open(tempFilename.toOsSpecific(), 'w') for line in textLines: line = line.strip() if line and line[0] != '#': # Write the line out only if it's not a comment. temp.write(line + '\n') temp.close() if self.packager.prcSignCommand: # Now sign the file. command = '%s -n "%s"' % ( self.packager.prcSignCommand, tempFilename.toOsSpecific()) self.notify.info(command) exitStatus = os.system(command) if exitStatus != 0: self.notify.error('Command failed: %s' % (command)) if self.packager.prcEncryptionKey: # And now encrypt it. if file.newName.endswith('.prc'): # Change .prc -> .pre file.newName = file.newName[:-1] + 'e' preFilename = Filename.temporary('', 'p3d_', '.pre') preFilename.setBinary() tempFilename.setText() encryptFile(tempFilename, preFilename, self.packager.prcEncryptionKey) tempFilename.unlink() tempFilename = preFilename if file.deleteTemp: file.filename.unlink() file.filename = tempFilename file.text = None file.deleteTemp = True self.addComponent(file) def addComponent(self, file): compressionLevel = 0 if file.compress: compressionLevel = self.compressionLevel if file.text: stream = StringStream(file.text) self.multifile.addSubfile(file.newName, stream, compressionLevel) self.multifile.flush() elif file.executable and self.arch: if not self.__addOsxExecutable(file): return else: # Copy an ordinary file into the multifile. self.multifile.addSubfile(file.newName, file.filename, compressionLevel) if file.extract: if file.text: # Better write it to a temporary file, so we can # get its hash. tfile = Filename.temporary('', 'p3d_') open(tfile.toOsSpecific(), 'wb').write(file.text) xextract = self.getFileSpec('extract', tfile, file.newName) tfile.unlink() else: # The file data exists on disk already. xextract = self.getFileSpec('extract', file.filename, file.newName) self.extracts.append((file.newName.lower(), xextract)) xcomponent = TiXmlElement('component') xcomponent.SetAttribute('filename', file.newName) self.components.append(('c', file.newName.lower(), xcomponent)) def __addOsxExecutable(self, file): """ Adds an executable or shared library to the multifile, with respect to OSX's fat-binary features. Returns true on success, false on failure. """ compressionLevel = 0 if file.compress: compressionLevel = self.compressionLevel # If we're on OSX and adding only files for a # particular architecture, use lipo to strip out the # part of the file for that architecture. # First, we need to verify that it is in fact a # universal binary. tfile = Filename.temporary('', 'p3d_') tfile.setBinary() command = '/usr/bin/lipo -info "%s" >"%s"' % ( file.filename.toOsSpecific(), tfile.toOsSpecific()) exitStatus = os.system(command) if exitStatus != 0: self.notify.warning("Not an executable file: %s" % (file.filename)) # Just add it anyway. file.filename.setBinary() self.multifile.addSubfile(file.newName, file.filename, compressionLevel) return True # The lipo command succeeded, so it really is an # executable file. Parse the lipo output to figure out # which architectures the file supports. arches = [] lipoData = open(tfile.toOsSpecific(), 'r').read() tfile.unlink() if ':' in lipoData: arches = lipoData.rsplit(':', 1)[1] arches = arches.split() if arches == [self.arch]: # The file only contains the one architecture that # we want anyway. file.filename.setBinary() self.multifile.addSubfile(file.newName, file.filename, compressionLevel) return True if self.arch not in arches: # The file doesn't support the architecture that we # want at all. Omit the file. self.notify.warning("%s doesn't support architecture %s" % ( file.filename, self.arch)) return False # The file contains multiple architectures. Get # out just the one we want. command = '/usr/bin/lipo -thin %s -output "%s" "%s"' % ( self.arch, tfile.toOsSpecific(), file.filename.toOsSpecific()) exitStatus = os.system(command) if exitStatus != 0: self.notify.error('Command failed: %s' % (command)) self.multifile.addSubfile(file.newName, tfile, compressionLevel) if file.deleteTemp: file.filename.unlink() file.filename = tfile file.deleteTemp = True return True def requirePackage(self, package): """ Indicates a dependency on the given package. This also implicitly requires all of the package's requirements as well (though this transitive requirement happens at runtime, not here at build time). """ if package not in self.requires: self.requires.append(package) for lowerName in package.targetFilenames.keys(): ext = Filename(lowerName).getExtension() if ext not in self.packager.nonuniqueExtensions: self.skipFilenames[lowerName] = True for moduleName, mdef in package.moduleNames.items(): self.skipModules[moduleName] = mdef # Packager constructor def __init__(self, platform = None): # The following are config settings that the caller may adjust # before calling any of the command methods. # The platform string. self.setPlatform(platform) # This should be set to a Filename. self.installDir = None # If specified, this is a directory to search first for any # library references, before searching the system. # Particularly useful on OSX to reference the universal SDK. self.systemRoot = None # Set this true to treat setHost() the same as addHost(), thus # ignoring any request to specify a particular download host, # e.g. for testing and development. self.ignoreSetHost = False # This will be appended to the basename of any .p3d package, # before the .p3d extension. self.p3dSuffix = '' # The download URL at which these packages will eventually be # hosted. self.hosts = {} self.host = PandaSystem.getPackageHostUrl() self.addHost(self.host) # The maximum amount of time a client should cache the # contents.xml before re-querying the server, in seconds. self.maxAge = 0 # The contents seq: a tuple of integers, representing the # current seq value. The contents seq generally increments # with each modification to the contents.xml file. There is # also a package seq for each package, which generally # increments with each modification to the package. # The contents seq and package seq are used primarily for # documentation purposes, to note when a new version is # released. The package seq value can also be used to verify # that the contents.xml, desc.xml, and desc.import.xml files # were all built at the same time. # Although the package seqs are used at runtime to verify that # the latest contents.xml file has been downloaded, they are # not otherwise used at runtime, and they are not binding on # the download version. The md5 hash, not the package seq, is # actually used to differentiate different download versions. self.contentsSeq = SeqValue() # A search list for previously-built local packages. # We use a bit of caution to read the Filenames out of the # config variable. Since cvar.getDirectories() returns a list # of references to Filename objects stored within the config # variable itself, we have to make a copy of each Filename # returned, so they will persist beyond the lifespan of the # config variable. cvar = ConfigVariableSearchPath('pdef-path') self.installSearch = list(map(Filename, cvar.getDirectories())) # The system PATH, for searching dll's and exe's. self.executablePath = DSearchPath() # By convention, we include sys.path at the front of # self.executablePath, mainly to aid makepanda when building # an rtdist build. for dirname in sys.path: self.executablePath.appendDirectory(Filename.fromOsSpecific(dirname)) # Now add the actual system search path. if self.platform.startswith('win'): self.addWindowsSearchPath(self.executablePath, "PATH") elif self.platform.startswith('osx'): self.addPosixSearchPath(self.executablePath, "DYLD_LIBRARY_PATH") self.addPosixSearchPath(self.executablePath, "LD_LIBRARY_PATH") self.addPosixSearchPath(self.executablePath, "PATH") self.executablePath.appendDirectory('/lib') self.executablePath.appendDirectory('/usr/lib') self.executablePath.appendDirectory('/usr/local/lib') else: self.addPosixSearchPath(self.executablePath, "LD_LIBRARY_PATH") self.addPosixSearchPath(self.executablePath, "PATH") self.executablePath.appendDirectory('/lib') self.executablePath.appendDirectory('/usr/lib') self.executablePath.appendDirectory('/usr/local/lib') import platform if platform.uname()[1]=="pcbsd": self.executablePath.appendDirectory('/usr/PCBSD/local/lib') # Set this flag true to automatically add allow_python_dev to # any applications. self.allowPythonDev = False # Fill this with a list of (certificate, chain, pkey, # password) tuples to automatically sign each p3d file # generated. self.signParams = [] # Optional signing and encrypting features. self.encryptionKey = None self.prcEncryptionKey = None self.prcSignCommand = None # This is a list of filename extensions and/or basenames that # indicate files that should be encrypted within the # multifile. This provides obfuscation only, not real # security, since the decryption key must be part of the # client and is therefore readily available to any hacker. # Not only is this feature useless, but using it also # increases the size of your patchfiles, since encrypted files # can't really be patched. But it's here if you really want # it. ** Note: Actually, this isn't implemented yet. #self.encryptExtensions = [] #self.encryptFiles = [] # This is the list of DC import suffixes that should be # available to the client. Other suffixes, like AI and UD, # are server-side only and should be ignored by the Scrubber. self.dcClientSuffixes = ['OV'] # Is this file system case-sensitive? self.caseSensitive = True if self.platform.startswith('win'): self.caseSensitive = False elif self.platform.startswith('osx'): self.caseSensitive = False # Get the list of filename extensions that are recognized as # image files. self.imageExtensions = [] for type in PNMFileTypeRegistry.getGlobalPtr().getTypes(): self.imageExtensions += type.getExtensions() # Other useful extensions. The .pz extension is implicitly # stripped. # Model files. self.modelExtensions = [ 'egg', 'bam' ] # Text files that are copied (and compressed) to the package # with end-of-line conversion. self.textExtensions = [ 'prc', 'ptf', 'txt', 'cg', 'sha', 'dc', 'xml' ] # Binary files that are copied (and compressed) without # processing. self.binaryExtensions = [ 'ttf', 'TTF', 'mid', 'ico' ] # Files that can have an existence in multiple different # packages simultaneously without conflict. self.nonuniqueExtensions = [ 'prc' ] # Files that represent an executable or shared library. if self.platform.startswith('win'): self.executableExtensions = [ 'dll', 'pyd', 'exe' ] elif self.platform.startswith('osx'): self.executableExtensions = [ 'so', 'dylib' ] else: self.executableExtensions = [ 'so' ] # Files that represent a Windows "manifest" file. These files # must be explicitly extracted to disk so the OS can find # them. if self.platform.startswith('win'): self.manifestExtensions = [ 'manifest' ] else: self.manifestExtensions = [ ] # Extensions that are automatically remapped by convention. self.remapExtensions = {} if self.platform.startswith('win'): pass elif self.platform.startswith('osx'): self.remapExtensions = { 'dll' : 'dylib', 'pyd' : 'so', 'exe' : '' } else: self.remapExtensions = { 'dll' : 'so', 'pyd' : 'so', 'exe' : '' } # Files that should be extracted to disk. self.extractExtensions = self.executableExtensions[:] + self.manifestExtensions[:] + [ 'ico' ] # Files that indicate a platform dependency. self.platformSpecificExtensions = self.executableExtensions[:] # Binary files that are considered uncompressible, and are # copied without compression. self.uncompressibleExtensions = [ 'mp3', 'ogg', 'wav', 'rml', 'rcss', 'otf' ] # wav files are compressible, but p3openal_audio won't load # them compressed. # rml, rcss and otf files must be added here because # libRocket wants to be able to seek in these files. # Files which are not to be processed further, but which # should be added exactly byte-for-byte as they are. self.unprocessedExtensions = [] # System files that should never be packaged. For # case-insensitive filesystems (like Windows and OSX), put the # lowercase filename here. Case-sensitive filesystems should # use the correct case. self.excludeSystemFiles = [ 'kernel32.dll', 'user32.dll', 'wsock32.dll', 'ws2_32.dll', 'advapi32.dll', 'opengl32.dll', 'glu32.dll', 'gdi32.dll', 'shell32.dll', 'ntdll.dll', 'ws2help.dll', 'rpcrt4.dll', 'imm32.dll', 'ddraw.dll', 'shlwapi.dll', 'secur32.dll', 'dciman32.dll', 'comdlg32.dll', 'comctl32.dll', 'ole32.dll', 'oleaut32.dll', 'gdiplus.dll', 'winmm.dll', 'libsystem.b.dylib', 'libmathcommon.a.dylib', 'libmx.a.dylib', 'libstdc++.6.dylib', 'libobjc.a.dylib', 'libauto.dylib', ] # As above, but with filename globbing to catch a range of # filenames. self.excludeSystemGlobs = [ GlobPattern('d3dx9_*.dll'), GlobPattern('libGL.so*'), GlobPattern('libGLU.so*'), GlobPattern('libGLcore.so*'), GlobPattern('libGLES*.so*'), GlobPattern('libEGL.so*'), GlobPattern('libX11.so*'), GlobPattern('libXau.so*'), GlobPattern('libXdmcp.so*'), GlobPattern('libxcb*.so*'), GlobPattern('libc.so*'), GlobPattern('libgcc_s.so*'), GlobPattern('libdl.so*'), GlobPattern('libm.so*'), GlobPattern('libnvidia*.so*'), GlobPattern('libpthread.so*'), GlobPattern('libthr.so*'), GlobPattern('ld-linux.so*'), ] # A Loader for loading models. self.loader = Loader.Loader(self) self.sfxManagerList = None self.musicManager = None # This is filled in during readPackageDef(). self.packageList = [] # A table of all known packages by name. self.packages = {} # A list of PackageEntry objects read from the contents.xml # file. self.contents = {} def setPlatform(self, platform = None): """ Sets the platform that this Packager will compute for. On OSX, this can be used to specify the particular architecture we are building; on other platforms, it is probably a mistake to set this. You should call this before doing anything else with the Packager. It's even better to pass the platform string to the constructor. """ self.platform = platform or PandaSystem.getPlatform() # OSX uses this "arch" string for the otool and lipo commands. self.arch = None if self.platform.startswith('osx_'): self.arch = self.platform[4:] def setHost(self, host, downloadUrl = None, descriptiveName = None, hostDir = None, mirrors = None): """ Specifies the URL that will ultimately host these contents. """ if not self.ignoreSetHost: self.host = host self.addHost(host, downloadUrl = downloadUrl, descriptiveName = descriptiveName, hostDir = hostDir, mirrors = mirrors) def addHost(self, host, downloadUrl = None, descriptiveName = None, hostDir = None, mirrors = None): """ Adds a host to the list of known download hosts. This information will be written into any p3d files that reference this host; this can be used to pre-define the possible mirrors for a given host, for instance. Returns the newly-created HostEntry object.""" scheme = URLSpec(host).getScheme() if scheme == 'https' and downloadUrl is None: # If we specified an SSL-protected host URL, but no # explicit download URL, then assume the download URL is # the same, over cleartext. url = URLSpec(host) url.setScheme('http') downloadUrl = url.getUrl() he = self.hosts.get(host, None) if he is None: # Define a new host entry he = self.HostEntry(host, downloadUrl = downloadUrl, descriptiveName = descriptiveName, hostDir = hostDir, mirrors = mirrors) self.hosts[host] = he else: # Update an existing host entry if downloadUrl is not None: he.downloadUrl = downloadUrl if descriptiveName is not None: he.descriptiveName = descriptiveName if hostDir is not None: he.hostDir = hostDir if mirrors is not None: he.mirrors = mirrors return he def addAltHost(self, keyword, altHost, origHost = None, downloadUrl = None, descriptiveName = None, hostDir = None, mirrors = None): """ Adds an alternate host to any already-known host. This defines an alternate server that may be contacted, if specified on the HTML page, which hosts a different version of the server's contents. (This is different from a mirror, which hosts an identical version of the server's contents.) """ if not origHost: origHost = self.host self.addHost(altHost, downloadUrl = downloadUrl, descriptiveName = descriptiveName, hostDir = hostDir, mirrors = mirrors) he = self.addHost(origHost) he.altHosts[keyword] = altHost def addWindowsSearchPath(self, searchPath, varname): """ Expands $varname, interpreting as a Windows-style search path, and adds its contents to the indicated DSearchPath. """ path = ExecutionEnvironment.getEnvironmentVariable(varname) if len(path) == 0: if varname not in os.environ: return path = os.environ[varname] for dirname in path.split(';'): dirname = Filename.fromOsSpecific(dirname) if dirname.makeTrueCase(): searchPath.appendDirectory(dirname) def addPosixSearchPath(self, searchPath, varname): """ Expands $varname, interpreting as a Posix-style search path, and adds its contents to the indicated DSearchPath. """ path = ExecutionEnvironment.getEnvironmentVariable(varname) if len(path) == 0: if varname not in os.environ: return path = os.environ[varname] for dirname in path.split(':'): dirname = Filename.fromOsSpecific(dirname) if dirname.makeTrueCase(): searchPath.appendDirectory(dirname) def setup(self): """ Call this method to initialize the class after filling in some of the values in the constructor. """ self.knownExtensions = self.imageExtensions + self.modelExtensions + self.textExtensions + self.binaryExtensions + self.uncompressibleExtensions + self.unprocessedExtensions self.currentPackage = None if self.installDir: # If we were given an install directory, we can build # packages as well as plain p3d files, and it all goes # into the specified directory. self.p3dInstallDir = self.installDir self.allowPackages = True else: # If we don't have an actual install directory, we can # only build p3d files, and we drop them into the current # directory. self.p3dInstallDir = '.' self.allowPackages = False if not PandaSystem.getPackageVersionString() or not PandaSystem.getPackageHostUrl(): raise PackagerError, 'This script must be run using a version of Panda3D that has been built\nfor distribution. Try using ppackage.p3d or packp3d.p3d instead.\nIf you are running this script for development purposes, you may also\nset the Config variable panda-package-host-url to the URL you expect\nto download these contents from (for instance, a file:// URL).' self.readContentsFile() def close(self): """ Called after reading all of the package def files, this performs any final cleanup appropriate. """ self.writeContentsFile() def buildPatches(self, packages): """ Call this after calling close(), to build patches for the indicated packages. """ # We quietly ignore any p3d applications or solo packages # passed in the packages list; we only build patches for # actual Multifile-based packages. packageNames = [] for package in packages: if not package.p3dApplication and not package.solo: packageNames.append(package.packageName) if packageNames: from PatchMaker import PatchMaker pm = PatchMaker(self.installDir) pm.buildPatches(packageNames = packageNames) def readPackageDef(self, packageDef, packageNames = None): """ Reads the named .pdef file and constructs the named packages, or all packages if packageNames is None. Raises an exception if the pdef file is invalid. Returns the list of packages constructed. """ self.notify.info('Reading %s' % (packageDef)) # We use exec to "read" the .pdef file. This has the nice # side-effect that the user can put arbitrary Python code in # there to control conditional execution, and such. # Set up the namespace dictionary for exec. globals = {} globals['__name__'] = packageDef.getBasenameWoExtension() globals['__dir__'] = Filename(packageDef.getDirname()).toOsSpecific() globals['__file__'] = packageDef.toOsSpecific() globals['packageDef'] = packageDef globals['platform'] = self.platform globals['packager'] = self # We'll stuff all of the predefined functions, and the # predefined classes, in the global dictionary, so the pdef # file can reference them. # By convention, the existence of a method of this class named # do_foo(self) is sufficient to define a pdef method call # foo(). for methodName in self.__class__.__dict__.keys(): if methodName.startswith('do_'): name = methodName[3:] c = func_closure(name) globals[name] = c.generic_func globals['p3d'] = class_p3d globals['package'] = class_package globals['solo'] = class_solo # Now exec the pdef file. Assuming there are no syntax # errors, and that the pdef file doesn't contain any really # crazy Python code, all this will do is fill in the # '__statements' list in the module scope. # It appears that having a separate globals and locals # dictionary causes problems with resolving symbols within a # class scope. So, we just use one dictionary, the globals. execfile(packageDef.toOsSpecific(), globals) packages = [] # Now iterate through the statements and operate on them. statements = globals.get('__statements', []) if not statements: self.notify.info("No packages defined.") try: for (lineno, stype, name, args, kw) in statements: if stype == 'class': if packageNames is None or name in packageNames: classDef = globals[name] p3dApplication = (class_p3d in classDef.__bases__) solo = (class_solo in classDef.__bases__) self.beginPackage(name, p3dApplication = p3dApplication, solo = solo) statements = classDef.__dict__.get('__statements', []) if not statements: self.notify.info("No files added to %s" % (name)) for (lineno, stype, sname, args, kw) in statements: if stype == 'class': raise PackagerError, 'Nested classes not allowed' self.__evalFunc(sname, args, kw) package = self.endPackage() if package is not None: packages.append(package) elif packageNames is not None: # If the name is explicitly specified, this means # we should abort if the package faild to construct. raise PackagerError, 'Failed to construct %s' % name else: self.__evalFunc(name, args, kw) except PackagerError: # Append the line number and file name to the exception # error message. inst = sys.exc_info()[1] if not inst.args: inst.args = ('Error',) inst.args = (inst.args[0] + ' on line %s of %s' % (lineno, packageDef),) raise return packages def __evalFunc(self, name, args, kw): """ This is called from readPackageDef(), above, to call the function do_name(*args, **kw), as extracted from the pdef file. """ funcname = 'do_%s' % (name) func = getattr(self, funcname) try: func(*args, **kw) except OutsideOfPackageError: message = '%s encountered outside of package definition' % (name) raise OutsideOfPackageError, message def __expandTabs(self, line, tabWidth = 8): """ Expands tab characters in the line to 8 spaces. """ p = 0 while p < len(line): if line[p] == '\t': # Expand a tab. nextStop = ((p + tabWidth) / tabWidth) * tabWidth numSpaces = nextStop - p line = line[:p] + ' ' * numSpaces + line[p + 1:] p = nextStop else: p += 1 return line def __countLeadingWhitespace(self, line): """ Returns the number of leading whitespace characters in the line. """ line = self.__expandTabs(line) return len(line) - len(line.lstrip()) def __stripLeadingWhitespace(self, line, whitespaceCount): """ Removes the indicated number of whitespace characters, but no more. """ line = self.__expandTabs(line) line = line[:whitespaceCount].lstrip() + line[whitespaceCount:] return line def __parseArgs(self, words, argList): args = {} while len(words) > 1: arg = words[-1] if '=' not in arg: return args parameter, value = arg.split('=', 1) parameter = parameter.strip() value = value.strip() if parameter not in argList: message = 'Unknown parameter %s' % (parameter) raise PackagerError, message if parameter in args: message = 'Duplicate parameter %s' % (parameter) raise PackagerError, message args[parameter] = value del words[-1] def beginPackage(self, packageName, p3dApplication = False, solo = False): """ Begins a new package specification. packageName is the basename of the package. Follow this with a number of calls to file() etc., and close the package with endPackage(). """ if self.currentPackage: raise PackagerError, 'unclosed endPackage %s' % (self.currentPackage.packageName) package = self.Package(packageName, self) self.currentPackage = package package.p3dApplication = p3dApplication package.solo = solo if not package.p3dApplication and not self.allowPackages: message = 'Cannot generate packages without an installDir; use -i' raise PackagerError, message def endPackage(self): """ Closes the current package specification. This actually generates the package file. Returns the finished package, or None if the package failed to close (e.g. missing files). """ if not self.currentPackage: raise PackagerError, 'unmatched endPackage' package = self.currentPackage package.signParams += self.signParams[:] self.currentPackage = None if not package.close(): return None self.packageList.append(package) self.packages[(package.packageName, package.platform, package.version)] = package self.currentPackage = None return package def findPackage(self, packageName, platform = None, version = None, host = None, requires = None): """ Searches for the named package from a previous publish operation along the install search path. If requires is not None, it is a list of Package objects that are already required. The new Package object must be compatible with the existing Packages, or an error is returned. This is also useful for determining the appropriate package version to choose when a version is not specified. Returns the Package object, or None if the package cannot be located. """ # Is it a package we already have resident? package = self.packages.get((packageName, platform or self.platform, version, host), None) if package: return package # Look on the searchlist. for dirname in self.installSearch: package = self.__scanPackageDir(dirname, packageName, platform or self.platform, version, host, requires = requires) if not package: package = self.__scanPackageDir(dirname, packageName, platform, version, host, requires = requires) if package and host and package.host != host: # Wrong host. package = None if package: break if not package: # Query the indicated host. package = self.__findPackageOnHost(packageName, platform or self.platform, version or None, host, requires = requires) if not package: package = self.__findPackageOnHost(packageName, platform, version, host, requires = requires) if package: package = self.packages.setdefault((package.packageName, package.platform, package.version, package.host), package) self.packages[(packageName, platform or self.platform, version, host)] = package return package return None def __scanPackageDir(self, rootDir, packageName, platform, version, host, requires = None): """ Scans a directory on disk, looking for *.import.xml files that match the indicated packageName and optional version. If a suitable xml file is found, reads it and returns the assocated Package definition. If a version is not specified, and multiple versions are available, the highest-numbered version that matches will be selected. """ packages = [] if version: # A specific version package. versionList = [version] else: # An unversioned package, or any old version. versionList = [None, '*'] for version in versionList: packageDir = Filename(rootDir, packageName) basename = packageName if version: # A specific or nonspecific version package. packageDir = Filename(packageDir, version) basename += '.%s' % (version) if platform: packageDir = Filename(packageDir, platform) basename += '.%s' % (platform) # Actually, the host means little for this search, since we're # only looking in a local directory at this point. basename += '.import.xml' filename = Filename(packageDir, basename) filelist = glob.glob(filename.toOsSpecific()) if not filelist: # It doesn't exist in the nested directory; try the root # directory. filename = Filename(rootDir, basename) filelist = glob.glob(filename.toOsSpecific()) for file in filelist: package = self.__readPackageImportDescFile(Filename.fromOsSpecific(file)) packages.append(package) self.__sortImportPackages(packages) for package in packages: if package and self.__packageIsValid(package, requires, platform): return package return None def __findPackageOnHost(self, packageName, platform, version, hostUrl, requires = None): appRunner = AppRunnerGlobal.appRunner if not appRunner: # We don't download import files from a host unless we're # running in a packaged environment ourselves. It would # be possible to do this, but a fair bit of work for not # much gain--this is meant to be run in a packaged # environment. return None host = appRunner.getHost(hostUrl) if not host.readContentsFile(): if not host.downloadContentsFile(appRunner.http): return None packageInfos = [] packageInfo = host.getPackage(packageName, version, platform = platform) if not packageInfo and not version: # No explicit version is specified, first fallback: look # for the compiled-in version. packageInfo = host.getPackage(packageName, PandaSystem.getPackageVersionString(), platform = platform) if not packageInfo and not version: # No explicit version is specified, second fallback: get # the highest-numbered version available. packageInfos = host.getPackages(packageName, platform = platform) self.__sortPackageInfos(packageInfos) if packageInfo and not packageInfos: packageInfos = [packageInfo] for packageInfo in packageInfos: if not packageInfo or not packageInfo.importDescFile: continue # Now we've retrieved a PackageInfo. Get the import desc file # from it. filename = Filename(host.hostDir, 'imports/' + packageInfo.importDescFile.basename) if not appRunner.freshenFile(host, packageInfo.importDescFile, filename): self.notify.error("Couldn't download import file.") continue # Now that we have the import desc file, use it to load one of # our Package objects. package = self.Package('', self) if not package.readImportDescFile(filename): continue if self.__packageIsValid(package, requires, platform): return package # Couldn't find a suitable package. return None def __sortImportPackages(self, packages): """ Given a list of Packages read from *.import.xml filenames, sorts them in reverse order by version, so that the highest-numbered versions appear first in the list. """ tuples = [] for package in packages: version = self.__makeVersionTuple(package.version) tuples.append((version, file)) tuples.sort(reverse = True) return [t[1] for t in tuples] def __sortPackageInfos(self, packages): """ Given a list of PackageInfos retrieved from a Host, sorts them in reverse order by version, so that the highest-numbered versions appear first in the list. """ tuples = [] for package in packages: version = self.__makeVersionTuple(package.packageVersion) tuples.append((version, file)) tuples.sort(reverse = True) return [t[1] for t in tuples] def __makeVersionTuple(self, version): """ Converts a version string into a tuple for sorting, by separating out numbers into separate numeric fields, so that version numbers sort numerically where appropriate. """ if not version: return ('',) words = [] p = 0 while p < len(version): # Scan to the first digit. w = '' while p < len(version) and version[p] not in string.digits: w += version[p] p += 1 words.append(w) # Scan to the end of the string of digits. w = '' while p < len(version) and version[p] in string.digits: w += version[p] p += 1 if w: words.append(int(w)) return tuple(words) def __packageIsValid(self, package, requires, platform): """ Returns true if the package is valid, meaning it can be imported without conflicts with existing packages already required (such as different versions of panda3d). """ if package.platform and package.platform != platform: # Incorrect platform. return False if not requires: # No other restrictions. return True # Really, we only check the panda3d package. The other # packages will list this as a dependency, and this is all # that matters. panda1 = self.__findPackageInRequires('panda3d', [package] + package.requires) panda2 = self.__findPackageInRequires('panda3d', requires) if not panda1 or not panda2: return True if panda1.version == panda2.version: return True print 'Rejecting package %s, version "%s": depends on %s, version "%s" instead of version "%s"' % ( package.packageName, package.version, panda1.packageName, panda1.version, panda2.version) return False def __findPackageInRequires(self, packageName, list): """ Returns the first package with the indicated name in the list of packages, or in the list of packages required by the packages in the list. """ for package in list: if package.packageName == packageName: return package p2 = self.__findPackageInRequires(packageName, package.requires) if p2: return p2 return None def __readPackageImportDescFile(self, filename): """ Reads the named xml file as a Package, and returns it if valid, or None otherwise. """ package = self.Package('', self) if package.readImportDescFile(filename): return package return None def do_setVer(self, value): """ Sets an explicit set_ver number for the package, as a tuple of integers, or as a string of dot-separated integers. """ self.currentPackage.packageSetVer = SeqValue(value) def do_config(self, **kw): """ Called with any number of keyword parameters. For each keyword parameter, sets the corresponding p3d config variable to the given value. This will be written into the p3d_info.xml file at the top of the application, or to the package desc file for a package file. """ if not self.currentPackage: raise OutsideOfPackageError for keyword, value in kw.items(): self.currentPackage.configs[keyword] = value def do_require(self, *args, **kw): """ Indicates a dependency on the named package(s), supplied as a name. Attempts to install this package will implicitly install the named package also. Files already included in the named package will be omitted from this one when building it. """ self.requirePackagesNamed(args, **kw) def requirePackagesNamed(self, names, version = None, host = None): """ Indicates a dependency on the named package(s), supplied as a name. Attempts to install this package will implicitly install the named package also. Files already included in the named package will be omitted from this one when building it. """ if not self.currentPackage: raise OutsideOfPackageError for packageName in names: # A special case when requiring the "panda3d" package. We # supply the version number which we've been compiled with # as a default. pversion = version phost = host if packageName == 'panda3d': if not pversion: pversion = PandaSystem.getPackageVersionString() if not phost: phost = PandaSystem.getPackageHostUrl() package = self.findPackage(packageName, version = pversion, host = phost, requires = self.currentPackage.requires) if not package: message = 'Unknown package %s, version "%s"' % (packageName, version) raise PackagerError, message self.requirePackage(package) def requirePackage(self, package): """ Indicates a dependency on the indicated package, supplied as a Package object. Attempts to install this package will implicitly install the named package also. Files already included in the named package will be omitted from this one. """ if not self.currentPackage: raise OutsideOfPackageError # A special case when requiring the "panda3d" package. We # complain if the version number doesn't match what we've been # compiled with. if package.packageName == 'panda3d': if package.version != PandaSystem.getPackageVersionString(): self.notify.warning("Requiring panda3d version %s, which does not match the current build of Panda, which is version %s." % (package.version, PandaSystem.getPackageVersionString())) elif package.host != PandaSystem.getPackageHostUrl(): self.notify.warning("Requiring panda3d host %s, which does not match the current build of Panda, which is host %s." % (package.host, PandaSystem.getPackageHostUrl())) self.currentPackage.requirePackage(package) def do_module(self, *args, **kw): """ Adds the indicated Python module(s) to the current package. """ self.addModule(args, **kw) def addModule(self, moduleNames, newName = None, filename = None, required = False): if not self.currentPackage: raise OutsideOfPackageError if (newName or filename) and len(moduleNames) != 1: raise PackagerError, 'Cannot specify newName with multiple modules' if required: self.currentPackage.requiredModules += moduleNames for moduleName in moduleNames: self.currentPackage.freezer.addModule(moduleName, newName = newName, filename = filename) def do_excludeModule(self, *args): """ Marks the indicated Python module as not to be included. """ if not self.currentPackage: raise OutsideOfPackageError for moduleName in args: self.currentPackage.freezer.excludeModule(moduleName) def do_mainModule(self, moduleName, newName = None, filename = None): """ Names the indicated module as the "main" module of the application or exe. """ if not self.currentPackage: raise OutsideOfPackageError if self.currentPackage.mainModule and self.currentPackage.mainModule[0] != moduleName: self.notify.warning("Replacing mainModule %s with %s" % ( self.currentPackage.mainModule[0], moduleName)) if not newName: newName = moduleName if filename: filename = Filename(filename) newFilename = Filename('/'.join(moduleName.split('.'))) newFilename.setExtension(filename.getExtension()) self.currentPackage.addFile( filename, newName = newFilename.cStr(), explicit = True, extract = True, required = True) self.currentPackage.mainModule = (moduleName, newName) def do_sign(self, certificate, chain = None, pkey = None, password = None): """ Signs the resulting p3d file (or package multifile) with the indicated certificate. If needed, the chain file should contain the list of additional certificate authorities needed to validate the signing certificate. The pkey file should contain the private key. It is also legal for the certificate file to contain the chain and private key embedded within it. If the private key is encrypted, the password should be supplied. """ self.currentPackage.signParams.append((certificate, chain, pkey, password)) def do_setupPanda3D(self, p3dpythonName=None, p3dpythonwName=None): """ A special convenience command that adds the minimum startup modules for a panda3d package, intended for developers producing their own custom panda3d for download. Should be called before any other Python modules are named. """ # First, freeze just VFSImporter.py into its own # _vfsimporter.pyd file. This one is a special case, because # we need this code in order to load python files from the # Multifile, so this file can't itself be in the Multifile. # This requires a bit of care, because we only want to freeze # VFSImporter.py, and not any other part of direct. self.do_excludeModule('direct') # Import the actual VFSImporter module to get its filename on # disk. from direct.showbase import VFSImporter filename = Filename.fromOsSpecific(VFSImporter.__file__) self.do_module('VFSImporter', filename = filename) self.do_freeze('_vfsimporter', compileToExe = False) self.do_file('libpandaexpress.dll'); # Now that we're done freezing, explicitly add 'direct' to # counteract the previous explicit excludeModule(). self.do_module('direct') # This is the key Python module that is imported at runtime to # start an application running. self.do_module('direct.p3d.AppRunner') # This is the main program that drives the runtime Python. It # is responsible for loading _vfsimporter.pyd, and then # importing direct.p3d.AppRunner, to start an application # running. The program comes in two parts: an executable, and # an associated dynamic library. Note that the .exe and .dll # extensions are automatically replaced with the appropriate # platform-specific extensions. if self.platform.startswith('osx'): # On Mac, we package up a P3DPython.app bundle. This # includes specifications in the plist file to avoid # creating a dock icon and stuff. resources = [] # Find p3dpython.plist in the direct source tree. import direct plist = Filename(direct.__path__[0], 'plugin/p3dpython.plist') ## # Find panda3d.icns in the models tree. ## filename = Filename('plugin_images/panda3d.icns') ## found = filename.resolveFilename(getModelPath().getValue()) ## if not found: ## found = filename.resolveFilename("models") ## if found: ## resources.append(filename) self.do_makeBundle('P3DPython.app', plist, executable = 'p3dpython', resources = resources, dependencyDir = '') else: # Anywhere else, we just ship the executable file p3dpython.exe. if p3dpythonName is None: p3dpythonName = 'p3dpython' else: self.do_config(p3dpython_name=p3dpythonName) if self.platform.startswith('win'): self.do_file('p3dpython.exe', newName=p3dpythonName+'.exe') else: self.do_file('p3dpython.exe', newName=p3dpythonName) # The "Windows" executable appends a 'w' to whatever name is used # above, unless an override name is explicitly specified. if self.platform.startswith('win'): if p3dpythonwName is None: p3dpythonwName = p3dpythonName+'w' else: self.do_config(p3dpythonw_name=p3dpythonwName) if self.platform.startswith('win'): self.do_file('p3dpythonw.exe', newName=p3dpythonwName+'.exe') else: self.do_file('p3dpythonw.exe', newName=p3dpythonwName) self.do_file('libp3dpython.dll') def do_freeze(self, filename, compileToExe = False): """ Freezes all of the current Python code into either an executable (if compileToExe is true) or a dynamic library (if it is false). The resulting compiled binary is added to the current package under the indicated filename. The filename should not include an extension; that will be added. """ if not self.currentPackage: raise OutsideOfPackageError package = self.currentPackage freezer = package.freezer if package.mainModule and not compileToExe: self.notify.warning("Ignoring main_module for dll %s" % (filename)) package.mainModule = None if not package.mainModule and compileToExe: message = "No main_module specified for exe %s" % (filename) raise PackagerError, message if package.mainModule: moduleName, newName = package.mainModule if compileToExe: # If we're producing an exe, the main module must # be called "__main__". newName = '__main__' package.mainModule = (moduleName, newName) if newName not in freezer.modules: freezer.addModule(moduleName, newName = newName) else: freezer.modules[newName] = freezer.modules[moduleName] freezer.done(compileToExe = compileToExe) dirname = '' basename = filename if '/' in basename: dirname, basename = filename.rsplit('/', 1) dirname += '/' basename = freezer.generateCode(basename, compileToExe = compileToExe) package.addFile(Filename(basename), newName = dirname + basename, deleteTemp = True, explicit = True, extract = True) package.addExtensionModules() if not package.platform: package.platform = self.platform # Reset the freezer for more Python files. freezer.reset() package.mainModule = None def do_makeBundle(self, bundleName, plist, executable = None, resources = None, dependencyDir = None): """ Constructs a minimal OSX "bundle" consisting of an executable and a plist file, with optional resource files (such as icons), and adds it to the package under the given name. """ contents = bundleName + '/Contents' self.addFiles([plist], newName = contents + '/Info.plist', extract = True) if executable: basename = Filename(executable).getBasename() self.addFiles([executable], newName = contents + '/MacOS/' + basename, extract = True, executable = True, dependencyDir = dependencyDir) if resources: self.addFiles(resources, newDir = contents + '/Resources', extract = True, dependencyDir = dependencyDir) def do_file(self, *args, **kw): """ Adds the indicated file or files to the current package. See addFiles(). """ self.addFiles(args, **kw) def addFiles(self, filenames, text = None, newName = None, newDir = None, extract = None, executable = None, deleteTemp = False, literal = False, dependencyDir = None, required = False): """ Adds the indicated arbitrary files to the current package. filenames is a list of Filename or string objects, and each may include shell globbing characters. Each file is placed in the named directory, or the toplevel directory if no directory is specified. Certain special behavior is invoked based on the filename extension. For instance, .py files may be automatically compiled and stored as Python modules. If newDir is not None, it specifies the directory in which the file should be placed. In this case, all files matched by the filename expression are placed in the named directory. If newName is not None, it specifies a new filename. In this case, newDir is ignored, and the filename expression must match only one file. If newName and newDir are both None, the file is placed in the toplevel directory, regardless of its source directory. If text is nonempty, it contains the text of the file. In this case, the filename is not read, but the supplied text is used instead. If extract is true, the file is explicitly extracted at runtime. If executable is true, the file is marked as an executable filename, for special treatment. If deleteTemp is true, the file is a temporary file and will be deleted after its contents are copied to the package. If literal is true, then the file extension will be respected exactly as it appears, and glob characters will not be expanded. If this is false, then .dll or .exe files will be renamed to .dylib and no extension on OSX (or .so on Linux); and glob characters will be expanded. If required is true, then the file is marked a vital part of the package. The package will not be built if this file somehow cannot be added to the package. """ if not self.currentPackage: raise OutsideOfPackageError files = [] explicit = True for filename in filenames: filename = Filename(filename) if literal: thisFiles = [filename.toOsSpecific()] else: ext = filename.getExtension() # A special case, since OSX and Linux don't have a # standard extension for program files. if executable is None and ext == 'exe': executable = True newExt = self.remapExtensions.get(ext, None) if newExt is not None: filename.setExtension(newExt) thisFiles = glob.glob(filename.toOsSpecific()) if not thisFiles: thisFiles = [filename.toOsSpecific()] if newExt == 'dll' or (ext == 'dll' and newExt is None): # Go through the dsoFilename interface on Windows, # to insert a _d if we are running on a debug # build. dllFilename = Filename(filename) dllFilename.setExtension('so') dllFilename = Filename.dsoFilename(dllFilename.cStr()) if dllFilename != filename: thisFiles = glob.glob(filename.toOsSpecific()) if not thisFiles: # We have to resolve this filename to # determine if it's a _d or not. if dllFilename.resolveFilename(self.executablePath): thisFiles = [dllFilename.toOsSpecific()] else: thisFiles = [filename.toOsSpecific()] if len(thisFiles) > 1: explicit = False files += thisFiles prefix = '' if newDir is not None: prefix = Filename(newDir).cStr() if prefix and prefix[-1] != '/': prefix += '/' if newName: if len(files) != 1: message = 'Cannot install multiple files on target filename %s' % (newName) raise PackagerError, message if text: if len(files) != 1: message = 'Cannot install text to multiple files' raise PackagerError, message if not newName: newName = str(filenames[0]) for filename in files: filename = Filename.fromOsSpecific(filename) basename = filename.getBasename() name = newName if not name: name = prefix + basename self.currentPackage.addFile( filename, newName = name, extract = extract, explicit = explicit, executable = executable, text = text, deleteTemp = deleteTemp, dependencyDir = dependencyDir, required = required) def do_exclude(self, filename): """ Marks the indicated filename as not to be included. The filename may include shell globbing characters, and may or may not include a dirname. (If it does not include a dirname, it refers to any file with the given basename from any directory.)""" if not self.currentPackage: raise OutsideOfPackageError filename = Filename(filename) self.currentPackage.excludeFile(filename) def do_dir(self, dirname, newDir = None, unprocessed = None): """ Adds the indicated directory hierarchy to the current package. The directory hierarchy is walked recursively, and all files that match a known extension are added to the package. newDir specifies the directory name within the package which the contents of the named directory should be installed to. If it is omitted, the contents of the named directory are installed to the root of the package. If unprocessed is false (the default), bam files are loaded and scanned for textures, and these texture paths within the bam files are manipulated to point to the new paths within the package. If unprocessed is true, this operation is bypassed, and bam files are packed exactly as they are. """ if not self.currentPackage: raise OutsideOfPackageError dirname = Filename(dirname) if not newDir: newDir = '' # Adding the directory to sys.path is a cheesy way to help the # modulefinder find it. sys.path.append(dirname.toOsSpecific()) self.__recurseDir(dirname, newDir, unprocessed = unprocessed) def __recurseDir(self, filename, newName, unprocessed = None): dirList = vfs.scanDirectory(filename) if dirList: # It's a directory name. Recurse. prefix = newName if prefix and prefix[-1] != '/': prefix += '/' for subfile in dirList: filename = subfile.getFilename() self.__recurseDir(filename, prefix + filename.getBasename(), unprocessed = unprocessed) return # It's a file name. Add it. ext = filename.getExtension() if ext == 'py': self.currentPackage.addFile(filename, newName = newName, explicit = False, unprocessed = unprocessed) else: if ext == 'pz': # Strip off an implicit .pz extension. newFilename = Filename(filename) newFilename.setExtension('') newFilename = Filename(newFilename.cStr()) ext = newFilename.getExtension() if ext in self.knownExtensions: if ext in self.textExtensions: filename.setText() else: filename.setBinary() self.currentPackage.addFile(filename, newName = newName, explicit = False, unprocessed = unprocessed) def readContentsFile(self): """ Reads the contents.xml file at the beginning of processing. """ self.hosts = {} # Since we've blown away the self.hosts map, we have to make # sure that our own host at least is added to the map. self.addHost(self.host) self.maxAge = 0 self.contentsSeq = SeqValue() self.contents = {} self.contentsChanged = False if not self.allowPackages: # Don't bother. return contentsFilename = Filename(self.installDir, 'contents.xml') doc = TiXmlDocument(contentsFilename.toOsSpecific()) if not doc.LoadFile(): # Couldn't read file. return xcontents = doc.FirstChildElement('contents') if xcontents: maxAge = xcontents.Attribute('max_age') if maxAge: self.maxAge = int(maxAge) self.contentsSeq.loadXml(xcontents) xhost = xcontents.FirstChildElement('host') if xhost: he = self.HostEntry() he.loadXml(xhost, self) self.hosts[he.url] = he self.host = he.url xpackage = xcontents.FirstChildElement('package') while xpackage: pe = self.PackageEntry() pe.loadXml(xpackage) self.contents[pe.getKey()] = pe xpackage = xpackage.NextSiblingElement('package') def writeContentsFile(self): """ Rewrites the contents.xml file at the end of processing. """ if not self.contentsChanged: # No need to rewrite. return contentsFilename = Filename(self.installDir, 'contents.xml') doc = TiXmlDocument(contentsFilename.toOsSpecific()) decl = TiXmlDeclaration("1.0", "utf-8", "") doc.InsertEndChild(decl) xcontents = TiXmlElement('contents') if self.maxAge: xcontents.SetAttribute('max_age', str(self.maxAge)) self.contentsSeq += 1 self.contentsSeq.storeXml(xcontents) if self.host: he = self.hosts.get(self.host, None) if he: xhost = he.makeXml(packager = self) xcontents.InsertEndChild(xhost) contents = self.contents.items() contents.sort() for key, pe in contents: xpackage = pe.makeXml() xcontents.InsertEndChild(xpackage) doc.InsertEndChild(xcontents) doc.SaveFile() # The following class and function definitions represent a few sneaky # Python tricks to allow the pdef syntax to contain the pseudo-Python # code they do. These tricks bind the function and class definitions # into a bit table as they are parsed from the pdef file, so we can # walk through that table later and perform the operations requested # in order. class metaclass_def(type): """ A metaclass is invoked by Python when the class definition is read, for instance to define a child class. By defining a metaclass for class_p3d and class_package, we can get a callback when we encounter "class foo(p3d)" in the pdef file. The callback actually happens after all of the code within the class scope has been parsed first. """ def __new__(self, name, bases, dict): # At the point of the callback, now, "name" is the name of the # class we are instantiating, "bases" is the list of parent # classes, and "dict" is the class dictionary we have just # parsed. # If "dict" contains __metaclass__, then we must be parsing # class_p3d or class_ppackage, below--skip it. But if it # doesn't contain __metaclass__, then we must be parsing # "class foo(p3d)" (or whatever) from the pdef file. if '__metaclass__' not in dict: # Get the context in which this class was created # (presumably, the module scope) out of the stack frame. frame = sys._getframe(1) mdict = frame.f_locals lineno = frame.f_lineno # Store the class name on a statements list in that # context, so we can later resolve the class names in # the order they appeared in the file. mdict.setdefault('__statements', []).append((lineno, 'class', name, None, None)) return type.__new__(self, name, bases, dict) class class_p3d: __metaclass__ = metaclass_def pass class class_package: __metaclass__ = metaclass_def pass class class_solo: __metaclass__ = metaclass_def pass class func_closure: """ This class is used to create a closure on the function name, and also allows the *args, **kw syntax. In Python, the lambda syntax, used with default parameters, is used more often to create a closure (that is, a binding of one or more variables into a callable object), but that syntax doesn't work with **kw. Fortunately, a class method is also a form of a closure, because it binds self; and this doesn't have any syntax problems with **kw. """ def __init__(self, name): self.name = name def generic_func(self, *args, **kw): """ This method is bound to all the functions that might be called from the pdef file. It's a special function; when it is called, it does nothing but store its name and arguments in the caller's local scope, where they can be pulled out later. """ # Get the context in which this function was called (presumably, # the class dictionary) out of the stack frame. frame = sys._getframe(1) cldict = frame.f_locals lineno = frame.f_lineno # Store the function on a statements list in that context, so we # can later walk through the function calls for each class. cldict.setdefault('__statements', []).append((lineno, 'func', self.name, args, kw))