diff --git a/samples/dbg/PySideKick/Call.py b/samples/dbg/PySideKick/Call.py new file mode 100644 index 0000000..224d652 --- /dev/null +++ b/samples/dbg/PySideKick/Call.py @@ -0,0 +1,242 @@ +# Copyright (c) 2009-2010, Cloud Matrix Pty. Ltd. +# All rights reserved; available under the terms of the BSD License. +""" + +PySideKick.Call: helpers for managing function calls +===================================================== + + +This module defines a collection of helpers for managing function calls in +cooperation with the Qt event loop. We have: + + * qCallAfter: call function after current event has been processed + * qCallLater: call function after sleeping for some interval + * qCallInMainThread: (blockingly) call function in the main GUI thread + * qCallInWorkerThread: (nonblockingly) call function in worker thread + + +There is also a decorator to apply these helpers to all calls to a function: + + * qCallUsing(helper): route all calls to a function through the helper + +""" + +import sys +import thread +import threading +from functools import wraps +import Queue + +import PySideKick +from PySideKick import QtCore, qIsMainThread + + +class qCallAfter(QtCore.QObject): + """Call the given function on a subsequent iteration of the event loop. + + This helper arranges for the given function to be called on a subsequent + iteration of the main event loop. It's most useful inside event handlers, + where you may want to defer some work until after the event has finished + being processed. + + The implementation is as a singleton QObject subclass. It maintains a + queue of functions to the called, and posts an event to itself whenever + a new function is queued. + """ + + def __init__(self): + super(qCallAfter,self).__init__(None) + self.app = None + self.event_id = QtCore.QEvent.registerEventType() + self.event_type = QtCore.QEvent.Type(self.event_id) + self.pending_func_queue = Queue.Queue() + self.func_queue = Queue.Queue() + + def customEvent(self,event): + if event.type() == self.event_type: + self._popCall() + + def __call__(self,func,*args,**kwds): + global qCallAfter + # If the app is running, dispatch the event directly. + if self.app is not None: + self.func_queue.put((func,args,kwds)) + self._postEvent() + return + # Otherwise, we have some bootstrapping to do! + # Before dispatching, there must be a running app and we must + # be on the same thread as it. + app = QtCore.QCoreApplication.instance() + if app is None or not qIsMainThread(): + self.pending_func_queue.put((func,args,kwds)) + return + # This is the first call with a running app and from the + # main thread. If it turns out we're not on the main thread, + # replace ourselves with a fresh instance. + if hasattr(self,"thread"): + if self.thread() is not QtCore.QThread.currentThread(): + qCallAfter = self.__class__() + else: + self.app = app + else: + self.app = app + # OK, we now have the official qCallAfter instance. + # Flush all pending events. + try: + while True: + (pfunc,pargs,pkwds) = self.pending_func_queue.get(False) + qCallAfter(pfunc,*pargs,**pkwds) + except Queue.Empty: + pass + qCallAfter(func,*args,**kwds) + + def _popCall(self): + (func,args,kwds) = self.func_queue.get(False) + func(*args,**kwds) + + def _postEvent(self): + event = QtCore.QEvent(self.event_type) + try: + self.app.postEvent(self,event) + except RuntimeError: + # This can happen if the app has been destroyed. + # Immediately empty the queue. + try: + while True: + self._popCall() + except Queue.Empty: + pass + +# Optimistically create the singleton instance of qCallAfter. +# If this module is imported from a non-gui thread then this instance will +# eventually be replaced, but usually it'll be the correct one. +qCallAfter = qCallAfter() + + + +class Future(object): + """Primative "future" class for executing functions in another thread. + + Instances of this class represent a compuation sent to another thread. + Call the "get_result" method to wait until completion and get the result + (or raise the exception). + + Existing Future objects are recycled to avoid the overhead of allocating + a new lock primitive for each async call. + """ + + _READY_INSTANCES = [] + + def __init__(self): + self.ready = threading.Event() + self.result = None + self.exception = None + + @classmethod + def get_or_create(cls): + try: + return cls._READY_INSTANCES.pop(0) + except IndexError: + return cls() + + def recycle(self): + self.result = None + self.exception = None + self.ready.clear() + self._READY_INSTANCES.append(self) + + def call_function(self,func,*args,**kwds): + try: + self.result = func(*args,**kwds) + except Exception: + self.exception = sys.exc_info() + finally: + self.ready.set() + + def get_result(self): + self.ready.wait() + try: + if self.exception is None: + return self.result + else: + raise self.exception[0], \ + self.exception[1], \ + self.exception[2] + finally: + self.recycle() + + def is_ready(self): + return self.read.isSet() + + +def qCallInMainThread(func,*args,**kwds): + """Synchronously call the given function in the main thread. + + This helper arranges for the given function to be called in the main + event loop, then blocks and waits for the result. It's a simple way to + call functions that manipulate the GUI from a non-GUI thread. + """ + if qIsMainThread(): + func(*args,**kwds) + else: + future = Future.get_or_create() + qCallAfter(future.call_function,func,*args,**kwds) + return future.get_result() + + +def qCallInWorkerThread(func,*args,**kwds): + """Asynchronously call the given function in a background worker thread. + + This helper arranges for the given function to be executed by a background + worker thread. Eventually it'll get a thread pool; for now each task + spawns a new background thread. + + If you need to know the result of the function call, this helper returns + a Future object; use f.is_ready() to test whether it's ready and call + f.get_result() to get the return value or raise the exception. + """ + future = Future.get_or_create() + def runit(): + future.call_function(func,*args,**kwds) + threading.Thread(target=runit).start() + return future + + +def qCallLater(interval,func,*args,**kwds): + """Asynchronously call the given function after a timeout. + + This helper is similar to qCallAfter, but it waits at least 'interval' + seconds before executing the function. To cancel the call before the + sleep interval has expired, call 'cancel' on the returned object. + + Currently this is a thin wrapper around threading.Timer; eventually it + will be integrated with Qt's own timer mechanisms. + """ + def runit(): + qCallAfter(func,*args,**kwds) + t = threading.Timer(interval,runit) + t.start() + return t + + +def qCallUsing(helper): + """Function/method decorator to always apply a function call helper. + + This decorator can be used to ensure that a function is always called + using one of the qCall helpers. For example, the following function can + be safely called from any thread, as it will transparently apply the + qCallInMainThread helper whenever it is called: + + @qCallUsing(qCallInMainThread) + def prompt_for_input(msg): + # ... pop up a dialog, return input + + """ + def decorator(func): + @wraps(func) + def wrapper(*args,**kwds): + return helper(func,*args,**kwds) + return wrapper + return decorator + + diff --git a/samples/dbg/PySideKick/Console.py b/samples/dbg/PySideKick/Console.py new file mode 100644 index 0000000..cf98b6c --- /dev/null +++ b/samples/dbg/PySideKick/Console.py @@ -0,0 +1,143 @@ +# Copyright (c) 2009-2010, Cloud Matrix Pty. Ltd. +# All rights reserved; available under the terms of the BSD License. +""" +PySideKick.Console: a simple embeddable python shell +===================================================== + + +This module provides the call QPythonConsole, a python shell that can be +embedded in your GUI. + +""" + +import sys +from code import InteractiveConsole as _InteractiveConsole + +from PySideKick import QtCore, QtGui + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + + +class _QPythonConsoleInterpreter(_InteractiveConsole): + """InteractiveConsole subclass that sends all output to the GUI.""" + + def __init__(self,ui,locals=None): + _InteractiveConsole.__init__(self,locals) + self.ui = ui + + def write(self,data): + if data: + if data[-1] == "\n": + data = data[:-1] + self.ui.output.appendPlainText(data) + + def runsource(self,source,filename="<input>",symbol="single"): + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = sys.stderr = collector = StringIO() + try: + more = _InteractiveConsole.runsource(self,source,filename,symbol) + finally: + if sys.stdout is collector: + sys.stdout = old_stdout + if sys.stderr is collector: + sys.stderr = old_stderr + self.write(collector.getvalue()) + return more + + +class _QPythonConsoleUI(object): + """UI layout container for QPythonConsole.""" + def __init__(self,parent): + if parent.layout() is None: + parent.setLayout(QtGui.QHBoxLayout()) + layout = QtGui.QVBoxLayout() + layout.setSpacing(0) + # Output console: a fixed-pitch-font text area. + self.output = QtGui.QPlainTextEdit(parent) + self.output.setReadOnly(True) + self.output.setUndoRedoEnabled(False) + self.output.setMaximumBlockCount(5000) + fmt = QtGui.QTextCharFormat() + fmt.setFontFixedPitch(True) + self.output.setCurrentCharFormat(fmt) + layout.addWidget(self.output) + parent.layout().addLayout(layout) + # Input console, a prompt displated next to a lineedit + layout2 = QtGui.QHBoxLayout() + self.prompt = QtGui.QLabel(parent) + self.prompt.setText(">>> ") + layout2.addWidget(self.prompt) + self.input = QtGui.QLineEdit(parent) + layout2.addWidget(self.input) + layout.addLayout(layout2) + + +class QPythonConsole(QtGui.QWidget): + """A simple python console to embed in your GUI. + + This widget provides a simple interactive python console that you can + embed in your GUI (e.g. for debugging purposes). Use it like so: + + self.debug_window.layout().addWidget(QPythonConsole()) + + You can customize the variables that are available in the shell by + passing a dict as the "locals" argument. + """ + + def __init__(self,parent=None,locals=None): + super(QPythonConsole,self).__init__(parent) + self.ui = _QPythonConsoleUI(self) + self.interpreter = _QPythonConsoleInterpreter(self.ui,locals) + self.ui.input.returnPressed.connect(self._on_enter_line) + self.ui.input.installEventFilter(self) + self.history = [] + self.history_pos = 0 + + def _on_enter_line(self): + line = self.ui.input.text() + self.ui.input.setText("") + self.interpreter.write(self.ui.prompt.text() + line) + more = self.interpreter.push(line) + if line: + self.history.append(line) + self.history_pos = len(self.history) + while len(self.history) > 100: + self.history = self.history[1:] + self.history_pos -= 1 + if more: + self.ui.prompt.setText("... ") + else: + self.ui.prompt.setText(">>> ") + + def eventFilter(self,obj,event): + if event.type() == QtCore.QEvent.KeyPress: + if event.key() == QtCore.Qt.Key_Up: + self.go_history(-1) + elif event.key() == QtCore.Qt.Key_Down: + self.go_history(1) + return False + + def go_history(self,offset): + if offset < 0: + self.history_pos = max(0,self.history_pos + offset) + elif offset > 0: + self.history_pos = min(len(self.history),self.history_pos + offset) + try: + line = self.history[self.history_pos] + except IndexError: + line = "" + self.ui.input.setText(line) + + +if __name__ == "__main__": + import sys, os + app = QtGui.QApplication(sys.argv) + win = QtGui.QMainWindow() + win.setCentralWidget(QPythonConsole()) + win.show() + app.exec_() + diff --git a/samples/dbg/PySideKick/Hatchet.py b/samples/dbg/PySideKick/Hatchet.py new file mode 100644 index 0000000..f969ffa --- /dev/null +++ b/samples/dbg/PySideKick/Hatchet.py @@ -0,0 +1,1467 @@ +""" +PySideKick.Hatchet: hack frozen PySide apps down to size +========================================================= + +Hatchet is a tool for reducing the size of frozen PySide applications, by +re-building the PySide binaries to include only those classes and functions +that are actually used by the application. + +In its simplest use, you give Hatchet the path to a frozen python application +and let it work its magic: + + python -m PySideKick.Hatchet /path/to/frozen/app + +You might want to go for a coffee while it runs, or maybe even a pizza -- it +will take a while. Here are the things Hatchet will do to your frozen app: + + * extract all the identifiers used throughout the application. + * from this, calculate the set of all PySide classes and methods that the + application might refer to. + * download and unpack the latest PySide sources. + * hack the PySide sources to build only those classes and methods used + by the application. + * configure the PySide sources with some additional tricks to reduce + the final size of the binaries + * build the new PySide binaries and insert them into the application. + +The result can be a substantial reduction in the frozen application size. +I've seen the size drop by more than half compared to a naively-compiled +PySide binary. + +For finer control over the details of the binary-hacking process, you can +use and customize the "Hatchet" class. See its docstring for more details. + +In order to successfully rebuild the PySide binary, you *must* have the +necessary build environment up and running. This includes the Qt libraries +and development files, cmake, and the shiboken bindings generator. If +these are installed in a non-standard location, you can set the environment +variable CMAKE_INSTALL_PREFIX to let Hatchet find them. + +See the following pages for how to build PySide from source: + + http://developer.qt.nokia.com/wiki/Building_PySide_on_Linux + http://developer.qt.nokia.com/wiki/Building_PySide_on_Windows + http://developer.qt.nokia.com/wiki/Building_PySide_on_Mac_OS_X + +If you need to customize the build process, make a subclass of Hatchet and +override the "build_pyside_source" method. + +""" + +import sys +import os +import imp +import re +import zipfile +import tarfile +import tempfile +import tokenize +import shutil +import modulefinder +import urlparse +import urllib +import urllib2 +import hashlib +import subprocess +import logging +import inspect +from xml.dom import minidom +from collections import deque +from distutils import sysconfig +from textwrap import dedent + +import PySideKick + +# Download details for the latest PySide release. +PYSIDE_SOURCE_MD5 = "5589a883cebcb799a48b184a46db6386" +PYSIDE_SOURCE_URL = "http://www.pyside.org/files/pyside-qt4.7+1.0.0.tar.bz2" + + +# Name of file used to mark cached build directories +BUILD_OK_MARKER = "PySideKick.Hatchet.Built.txt" + + +# Classes that must not be hacked out of the PySide binary. +# These are used for various things internally. +KEEP_CLASSES = set(( + "QApplication", + "QWidget", + "QFlag", + "QFlags", + "QBuffer", + "QVariant", + "QByteArray", + "QLayout", # used by glue code for QWidget. Can we remove it somehow? + "QDeclarativeItem", # used internally by QtDeclarative +)) + + +# Methods that must not be hacked off of various objects. +# Mostly this is voodoo to stop things from segfaulting. +KEEP_METHODS = { + "*": set(("metaObject", # much breakage ensues if this is missing! + "devType", # rejecting this segfaults on by linux box + "metric", # without this fonts don't display correctly + )), + "QBitArray": set(("setBit",)), + "QByteArray": set(("insert",)), + "QFileDialog": set(("*",)), +} + + +# These are protected methods, and win32 can't use the "protected hack" +# to access them so we are forced to generate the bindings. +if sys.platform == "win32": + KEEP_METHODS["QObject"] = set(( + "connectNotify", + "disconnectNotify", + )) + + +# Simple regular expression for matching valid python identifiers. +_identifier_re = re.compile("^"+tokenize.Name+"$") +is_identifier = _identifier_re.match + + +class Hatchet(object): + """Class for hacking unused code out of the PySide binaries. + + A Hatchet object controls what and how to hack things out of the PySide + binaries for a frozen application. It must be given a path to a directory + containing a frozen PySide application. When you call the hack() method + it will: + + * extract all the identifiers used throughout the application. + * from this, calculate the set of all PySide classes and methods + that the application might refer to. + * download and unpack the latest PySide sources. + * hack the PySide sources to build only those classes and methods + used by the application. + * configure the PySide sources with some additional tricks to reduce + the final size of the binaries + * build the new PySide binaries and insert them into the application. + + You can customize the behaviour of the Hatchet by adjusting the following + attributes: + + * mf: a ModuleFinder object used to locate the app's code. + * typedb: a TypeDB instance used to get information about + the classes and methods available in PySide. + * keep_classes: a set of class names that must not be removed. + * keep_methods: a dict mapping class names to methods on those + classes that must not be removed. + + You can adjust the modules searched for Qt identifiers by calling + the following methods: + + * add_file: directly add a .py or .pyc file + * add_directory: add the entire contents of a directory + * add_zipfile: add the entire contents of a zipfile + + If you don't call any of these methods, the contents of the given appdir + will be used. + """ + + SOURCE_URL = PYSIDE_SOURCE_URL + SOURCE_MD5 = PYSIDE_SOURCE_MD5 + + def __init__(self,appdir,mf=None,typedb=None,logger=None): + self.appdir = appdir + if mf is None: + mf = modulefinder.ModuleFinder() + self.mf = mf + if logger is None: + logger = logging.getLogger("PySideKick.Hatchet") + self.logger = logger + if typedb is None: + typedb = TypeDB(logger=self.logger) + self.typedb = typedb + self.keep_classes = set() + self.keep_methods = {} + + def hack(self): + """Hack away at the PySide binary for this frozen application. + + This method is the main entry-point for using the Hatchet class. + It will examine the frozen application to find out what classes and + methods it uses, then replace its PySide modules with new binaries + hacked down to exclude useless code + """ + if not self.mf.modules: + self.add_directory(self.appdir) + self.analyse_code() + # Try to use a cached build if possible. + # We use a hash of the build parameters to identify the correct dir. + fp = self.get_build_fingerprint() + remove_builddir = False + builddir = get_cache_dir("Hatchet","build",fp) + if builddir is None: + remove_builddir = True + builddir = tempfile.mkdtemp() + try: + self.logger.debug("building PySide in %r",builddir) + if os.path.exists(os.path.join(builddir,BUILD_OK_MARKER)): + for nm in os.listdir(builddir): + if nm != BUILD_OK_MARKER: + sourcedir = os.path.join(builddir,nm) + break + else: + msg = "Broken cached builddir: %s" % (builddir,) + raise RuntimeError(msg) + self.logger.debug("using cached builddir: %r",sourcedir) + else: + sourcefile = self.fetch_pyside_source() + sourcedir = self.unpack_tarball(sourcefile,builddir) + self.hack_pyside_source(sourcedir) + self.build_pyside_source(sourcedir) + with open(os.path.join(builddir,BUILD_OK_MARKER),"wt") as f: + f.write(dedent(""" + This PySide directory was built using PySideKick.Hatchet. + Don't use it for a regular install of PySide. + """)) + self.copy_hacked_pyside_modules(sourcedir,self.appdir) + except: + remove_builddir = True + raise + finally: + if remove_builddir: + shutil.rmtree(builddir) + + def add_script(self,pathname,follow_imports=True): + """Add an additional script for the frozen application. + + This method adds the specified script to the internal modulefinder. + It and all of its imports will be examined for pyside-related + identifiers that must not be hacked out of the binary. + + To incude only the given file and not its imports, specify the + "follow_imports" keyword argument as False. + """ + try: + if not follow_imports: + self.mf.scan_code = lambda *a: None + self.mf.run_script(pathname) + finally: + if not follow_imports: + del self.mf.scan_code + + def add_file(self,pathname,pkgname="",follow_imports=True): + """Add an additional python source file for the frozen application. + + This method adds the specified *.py or *.pyc file to the modulefinder. + It and all of its imports will be examined for pyside-related + identifiers that must not be hacked out of the binary. + + To incude only the given file and not its imports, specify the + "follow_imports" keyword argument as False. + """ + if pkgname and not pkgname.endswith("."): + pkgname += "." + nm = os.path.basename(pathname) + base,ext = os.path.splitext(nm) + if ext == ".py": + fp = open(pathname,"rt") + stuff = (ext, "r", imp.PY_SOURCE,) + elif ext == ".pyc": + fp = open(pathname,"rb") + stuff = (ext, "r", imp.PY_COMPILED,) + else: + raise ValueError("unknown file type: %r" % (nm,)) + try: + if not follow_imports: + self.mf.scan_code = lambda *a: None + self.mf.load_module(pkgname + base,fp,pathname,stuff) + finally: + if not follow_imports: + del self.mf.scan_code + fp.close() + + def add_zipfile(self,pathname,follow_imports=True): + """Add an additional python zipfile for the frozen application. + + This method adds the specified zipfile to the internal modulefinder. + All of its contained python modules, along with their imports, will + be examined for pyside-related identifiers that must not be hacked + out of the binary. + + To incude only the contained files and not their imports, specify the + "follow_imports" keyword argument as False. + """ + tdir = tempfile.mkdtemp() + if not tdir.endswith(os.path.sep): + tdir += os.path.sep + try: + zf = zipfile.ZipFile(pathname,"r") + try: + for nm in zf.namelist(): + dstnm = os.path.join(tdir,nm) + if not dstnm.startswith(tdir): + continue + if not os.path.isdir(os.path.dirname(dstnm)): + os.makedirs(os.path.dirname(dstnm)) + with open(dstnm,"wb") as f: + f.write(zf.read(nm)) + finally: + zf.close() + self.add_directory(tdir,follow_imports=follow_imports) + finally: + shutil.rmtree(tdir) + + def add_directory(self,pathname,fqname="",follow_imports=True): + """Add an additional python directory for the frozen application. + + This method adds the specified directory to the internal modulefinder. + All of its contained python files, along with their imports, will be + examined for pyside-related identifiers that must not be hacked out + of the binary. + + To incude only the contained files and not their imports, specify the + "follow_imports" keyword argument as False. + """ + if fqname and not fqname.endswith("."): + fqname += "." + rkwds = dict(follow_imports=follow_imports) + for nm in os.listdir(pathname): + subpath = os.path.join(pathname,nm) + if os.path.isdir(subpath): + for ininm in ("__init__.py","__init__.pyc",): + inipath = os.path.join(subpath,ininm) + if os.path.exists(inipath): + self.mf.load_package(fqname + nm,subpath) + self.add_directory(subpath,fqname+nm+".",**rkwds) + break + else: + self.add_directory(subpath,**rkwds) + else: + if nm.endswith(".py") or nm.endswith(".pyc"): + self.add_file(subpath,fqname,**rkwds) + elif nm.endswith(".zip"): + self.add_zipfile(subpath,**rkwds) + elif nm.endswith(".exe"): + try: + self.add_zipfile(subpath,**rkwds) + except (zipfile.BadZipfile,): + pass + else: + if sys.platform != "win32": + try: + if "executable" in _bt("file",subpath): + self.add_zipfile(subpath,**rkwds) + except (EnvironmentError,zipfile.BadZipfile,): + pass + + def analyse_code(self): + """Analyse the code of the frozen application. + + This is the top-level method to start the code analysis process. + It must be called after adding any extra files or directories, and + before attempting to hack up a new version of PySide. + """ + self.expand_kept_classes() + + def expand_kept_classes(self): + """Find classes and methods that might be used by the application. + + This method examines the code in use by the application, and finds + classes that it might use by matching their names against the + identifiers present in the code. + + Any such classes found are added to the "keep_classes" attribute. + """ + self.logger.debug("expanding kept classes") + # Find all python identifiers used in the application. + # It's a wide net, but it's easier than type inference! ;-) + used_ids = set() + for m in self.mf.modules.itervalues(): + if m.__code__ is not None: + self.logger.debug("examining code: %s",m) + self.find_identifiers_in_code(m.__code__,used_ids) + # Keep all classes used directly in code. + for classnm in self.typedb.iterclasses(): + if classnm in used_ids: + self.logger.debug("keeping class: %s [used]",classnm) + self.keep_classes.add(classnm) + if classnm in KEEP_CLASSES: + self.logger.debug("keeping class: %s [pinned]",classnm) + self.keep_classes.add(classnm) + # Keep all superclasses of all kept classes + for classnm in list(self.keep_classes): + for sclassnm in self.typedb.superclasses(classnm): + if sclassnm not in self.keep_classes: + msg = "keeping class: %s [sup %s]" + self.logger.debug(msg,sclassnm,classnm) + self.keep_classes.add(sclassnm) + # Now iteratively expand the kept classess with possible return types + # of any methods called on the kept classes. + todo_classes = deque(self.keep_classes) + while todo_classes: + classnm = todo_classes.popleft() + num_done = len(self.keep_classes) - len(todo_classes) + num_todo = len(self.keep_classes) + self.logger.debug("expanding methods of %s (class %d of %d)", + classnm,num_done,num_todo) + kept_methods = self.expand_kept_methods(classnm,used_ids) + for methnm in self.typedb.itermethods(classnm): + if methnm not in kept_methods: + continue + self.logger.debug("expanding method %s::%s",classnm,methnm) + for rtype in self.typedb.relatedtypes(classnm,methnm): + for sclassnm in self.typedb.superclasses(rtype): + if sclassnm not in self.keep_classes: + msg = "keeping class: %s [rtyp %s::%s]" + self.logger.debug(msg,sclassnm,classnm,methnm) + self.keep_classes.add(sclassnm) + todo_classes.append(sclassnm) + + def expand_kept_methods(self,classnm,used_ids): + """Find all methods that must be kept for the given class. + + This method uses the given set of of used identifiers to find all + methods on the given class that might be used by the application. + Any such methods found are added to the "keep_methods" attribute, + keyed by classname. + + The set of kept methods is also returned. + """ + kept_methods = self.keep_methods.setdefault(classnm,set()) + for methnm in self.typedb.itermethods(classnm): + if methnm in kept_methods: + continue + msg = "keeping method: %s::%s [%s]" + if methnm in used_ids: + self.logger.debug(msg,classnm,methnm,"used") + kept_methods.add(methnm) + elif methnm + "_" in used_ids: + self.logger.debug(msg,classnm,methnm,"used") + kept_methods.add(methnm) + elif methnm == classnm: + self.logger.debug(msg,classnm,methnm,"constructor") + kept_methods.add(methnm) + elif "*" in kept_methods: + self.logger.debug(msg,classnm,methnm,"star") + kept_methods.add(methnm) + elif methnm in self.keep_methods.get("*",()): + self.logger.debug(msg,classnm,methnm,"star") + kept_methods.add(methnm) + elif methnm in KEEP_METHODS.get(classnm,()): + self.logger.debug(msg,classnm,methnm,"pinned") + kept_methods.add(methnm) + elif "*" in KEEP_METHODS.get(classnm,()): + self.logger.debug(msg,classnm,methnm,"pinned") + kept_methods.add(methnm) + elif methnm in KEEP_METHODS.get("*",()): + self.logger.debug(msg,classnm,methnm,"pinned") + kept_methods.add(methnm) + else: + # TODO: is this just superstition on my part? + # Shiboken doesn't like it when we reject methods + # that have a pure virtual override somewhere in the + # inheritence chain. + for sclassnm in self.typedb.superclasses(classnm): + if self.typedb.ispurevirtual(sclassnm,methnm): + self.logger.debug(msg,classnm,methnm,"virtual") + kept_methods.add(methnm) + break + return kept_methods + + def find_rejections(self): + """Find classes and methods that can be rejected from PySide. + + This method uses the set of kept classes and methods to determine + the classes and methods that can be hacked out of the PySide binary. + + It generates tuples of the form ("ClassName",) for useless classes, + and the form ("ClassName","methodName",) for useless methods on + otherwise useful classes. + """ + for classnm in self.typedb.iterclasses(): + if classnm not in self.keep_classes: + yield (classnm,) + else: + for methnm in self.typedb.itermethods(classnm): + if methnm not in self.keep_methods.get(classnm,()): + yield (classnm,methnm,) + + def find_identifiers_in_code(self,code,ids=None): + """Find any possible identifiers used by the given code. + + This method performs a simplistic search for the identifiers used in + the given code object. It will detect attribute accesses and the use + of getattr with a constant string, but can't do anything fancy about + names created at runtime. It will also find plenty of false positives. + + The set of all identifiers used in the code is returned. If the + argument 'ids' is not None, it is taken to be the set that is being + built (mostly this is for easy recursive walking of code objects). + """ + if ids is None: + ids = set() + for name in code.co_names: + ids.add(name) + for const in code.co_consts: + if isinstance(const,basestring) and is_identifier(const): + ids.add(const) + elif isinstance(const,type(code)): + self.find_identifiers_in_code(const,ids) + return ids + + def fetch_pyside_source(self): + """Fetch the sources for latest pyside version. + + This method fetches the sources for the latest pyside version. + If the environment variable PYSIDEKICK_DOWNLOAD_CACHE is set then + we first look there for a cached version. PIP_DOWNLOAD_CACHE is + used as a fallback location. + """ + cachedir = get_cache_dir("Hatchet","src") + nm = os.path.basename(urlparse.urlparse(self.SOURCE_URL).path) + if cachedir is None: + (fd,cachefile) = tempfile.mkstemp() + os.close(fd) + else: + # Use cached version if it has correct md5. + cachefile = os.path.join(cachedir,nm) + if os.path.exists(cachefile): + if not self._check_pyside_source_md5(cachefile): + os.unlink(cachefile) + # Download if we can't use the cached version + if cachedir is None or not os.path.exists(cachefile): + self.logger.info("downloading %s",self.SOURCE_URL) + fIn = urllib2.urlopen(self.SOURCE_URL) + try: + with open(cachefile,"wb") as fOut: + shutil.copyfileobj(fIn,fOut) + finally: + fIn.close() + if not self._check_pyside_source_md5(cachefile): + msg = "corrupted download: %s" % (PYSIDE_SOURCE_URL,) + raise RuntimeError(msg) + return cachefile + + def _check_pyside_source_md5(self,cachefile): + """Check the MD5 of a downloaded source file.""" + if self.SOURCE_MD5 is None: + return True + md5 = hashlib.md5() + with open(cachefile,"rb") as f: + data = f.read(1024*32) + while data: + md5.update(data) + data = f.read(1024*32) + if md5.hexdigest() != self.SOURCE_MD5: + self.logger.critical("bad MD5 for %r",cachefile) + self.logger.critical(" %s != %s",md5.hexdigest(), + self.SOURCE_MD5) + return False + return True + + def unpack_tarball(self,sourcefile,destdir): + """Unpack the given tarball into the given directory. + + This method unpacks the given tarball file into the given directory. + It returns the path to the "root" directory of the tarball, i.e. the + first directory that contains an actual file. This is usually the + directory you want for e.g. building a source distribution. + """ + self.logger.info("unpacking %r => %r",sourcefile,destdir) + tf = tarfile.open(sourcefile,"r:*") + if not destdir.endswith(os.path.sep): + destdir += os.path.sep + try: + for nm in tf.getnames(): + destpath = os.path.abspath(os.path.join(destdir,nm)) + # Since we've checked the MD5 we should be safe from + # malicious filenames, but you can't be too careful... + if not destpath.startswith(destdir): + raise RuntimeError("tarball contains malicious paths!") + tf.extractall(destdir) + finally: + tf.close() + rootdir = destdir + names = os.listdir(rootdir) + while len(names) == 1: + rootdir = os.path.join(rootdir,names[0]) + names = os.listdir(rootdir) + return rootdir + + def hack_pyside_source(self,sourcedir): + """Hack useless code out of the given PySide source directory. + + This is where the fun happens! We generate a list of classes and + methods to reject from the build, and modify the PySide source dir + to make it happen. This involves two steps: + + * adding <rejection> elements to the typesystem files + * removing <class>_wrapper.cpp entries from the makefiles + + """ + self.logger.info("hacking PySide sources in %r",sourcedir) + logger = self.logger + # Find all rejections and store them for quick reference. + reject_classes = set() + reject_methods = {} + num_rejected_methods = 0 + for rej in self.find_rejections(): + if len(rej) == 1: + logger.debug("reject %s",rej[0]) + reject_classes.add(rej[0]) + else: + logger.debug("reject %s::%s",rej[0],rej[1]) + num_rejected_methods += 1 + reject_methods.setdefault(rej[0],set()).add(rej[1]) + logger.info("keeping %d classes",len(self.keep_classes)) + logger.info("rejecting %d classes, %d methods",len(reject_classes), + num_rejected_methods) + # Find each top-level module directory and patch the contained files. + psdir = os.path.join(sourcedir,"PySide") + moddirs = [] + for modnm in os.listdir(psdir): + if not modnm.startswith("Qt") and not modnm == "phonon": + continue + moddir = os.path.join(psdir,modnm) + if os.path.isdir(moddir): + # Add <rejection> records for each class and method. + # Also strip any modifications to rejected functions. + def adjust_typesystem_file(dom): + tsnode = None + for c in dom.childNodes: + if c.nodeType != c.ELEMENT_NODE: + continue + if c.tagName == "typesystem": + tsnode = c + break + else: + return dom + # Adjust the existings decls to meet our needs + TYPE_TAGS = ("enum-type","value-type","object-type",) + for cn in list(tsnode.childNodes): + if cn.nodeType != c.ELEMENT_NODE: + continue + if cn.tagName in TYPE_TAGS: + # Remove delcaration of any rejected classes. + clsnm = cn.getAttribute("name") + if clsnm in reject_classes: + tsnode.removeChild(cn) + continue + # Remove any modifications for rejected methods + if clsnm not in reject_methods: + continue + FUNC_TAGS = ("modify-function","add-function",) + for mfn in list(cn.childNodes): + if mfn.nodeType != c.ELEMENT_NODE: + continue + if mfn.tagName in FUNC_TAGS: + sig = mfn.getAttribute("signature") + fnm = sig.split("(")[0] + if fnm in reject_methods[clsnm]: + cn.removeChild(mfn) + # Add explicit rejection records. + for cls in reject_classes: + rn = dom.createElement("rejection") + rn.setAttribute("class",cls) + tsnode.appendChild(rn) + nl = dom.createTextNode("\n") + tsnode.appendChild(nl) + for (cls,nms) in reject_methods.iteritems(): + for nm in nms: + rn = dom.createElement("rejection") + rn.setAttribute("class",cls) + rn.setAttribute("function-name",nm) + tsnode.appendChild(rn) + rn = dom.createElement("rejection") + rn.setAttribute("class",cls) + rn.setAttribute("field-name",nm) + tsnode.appendChild(rn) + nl = dom.createTextNode("\n") + tsnode.appendChild(nl) + return dom + for (dirnm,_,filenms) in os.walk(moddir): + for filenm in filenms: + if filenm.startswith("typesystem_") and "xml" in filenm: + tsfile = os.path.join(dirnm,filenm) + self.patch_xml_file(adjust_typesystem_file,tsfile) + # Remove rejected classes from the build deps list + remaining_sources = [] + def dont_build_class(lines): + for ln in lines: + for rejcls in reject_classes: + if rejcls.lower()+"_" in ln: + if "wrapper.cpp" in ln: + if "_module_wrapper.cpp" not in ln: + break + if "_"+rejcls[1:].lower()+"_" in ln: + if "wrapper.cpp" in ln: + if "_module_wrapper.cpp" not in ln: + break + if rejcls in ln and "check_qt_class" in ln: + break + else: + if "wrapper.cpp" in ln: + remaining_sources.append(ln) + yield ln + self.patch_file(dont_build_class,moddir,"CMakeLists.txt") + # If there aren't any sources left to build in that module, + # remove it from the main PySide build file. + if len(remaining_sources) < 2: + def dont_build_module(lines): + for ln in lines: + if modnm not in ln: + yield ln + logger.debug("module empty, not building: %s",modnm) + self.patch_file(dont_build_module,psdir,"CMakeLists.txt") + + def patch_file(self,patchfunc,*paths): + """Patch the given file by applying a line-filtering function. + + This method allows easy patching of a build file by applying a + python function. + + The specified "patchfunc" must be a line filtering function - it takes + as input the sequence of lines from the file, and outputs a modified + sequence of lines. + """ + filepath = os.path.join(*paths) + self.logger.debug("patching file %r",filepath) + mod = os.stat(filepath).st_mode + (fd,tf) = tempfile.mkstemp() + try: + os.close(fd) + with open(tf,"wt") as fOut: + with open(filepath,"rt") as fIn: + for ln in patchfunc(fIn): + fOut.write(ln) + fOut.flush() + os.chmod(tf,mod) + if sys.platform == "win32": + os.unlink(filepath) + except: + os.unlink(tf) + raise + else: + os.rename(tf,filepath) + + def patch_xml_file(self,patchfunc,*paths): + """Patch the given file by applying an xml-filtering function. + + This method allows easy patching of a build file by applying a + python function. + + The specified "patchfunc" must be an xml filtering function - it takes + as input a DOM object and returns a modified DOM. + """ + filepath = os.path.join(*paths) + self.logger.debug("patching file %r",filepath) + mod = os.stat(filepath).st_mode + with open(filepath,"rt") as fIn: + xml = minidom.parse(fIn) + xml = patchfunc(xml) + (fd,tf) = tempfile.mkstemp() + try: + os.close(fd) + xmlstr = xml.toxml().encode("utf8") + with open(tf,"wt") as fOut: + fOut.write(xmlstr) + fOut.flush() + os.chmod(tf,mod) + if sys.platform == "win32": + os.unlink(filepath) + except: + os.unlink(tf) + raise + else: + os.rename(tf,filepath) + + def build_pyside_source(self,sourcedir): + """Build the PySide sources in the given directory. + + This is a simple wrapper around PySide's `cmake; make;` build process. + For it to work, you must have the necessary tools installed on your + system (e.g. cmake, shiboken) + """ + self.logger.info("building PySide in %r",sourcedir) + olddir = os.getcwd() + os.chdir(sourcedir) + try: + # Here we have some more tricks for getting smaller binaries: + # * CMAKE_BUILD_TYPE=MinSizeRel, to enable -Os + # * -fno-exceptions, to skip generation of stack-handling code + # We also try to use compiler options from python so that the + # libs will match as closely as possible. + env = os.environ.copy() + env = self.get_build_env(env) + cmd = ["cmake", + "-DCMAKE_BUILD_TYPE=MinSizeRel", + "-DCMAKE_VERBOSE_MAKEFILE=ON", + "-DBUILD_TESTS=False", + "-DPYTHON_EXECUTABLE="+sys.executable, + "-DPYTHON_INCLUDE_DIR="+sysconfig.get_python_inc() + ] + if "CMAKE_INSTALL_PREFIX" in env: + cmd.append( + "-DCMAKE_INSTALL_PREFIX="+env["CMAKE_INSTALL_PREFIX"] + ) + if "ALTERNATIVE_QT_INCLUDE_DIR" in env: + qt_include_dir = env["ALTERNATIVE_QT_INCLUDE_DIR"] + if qt_include_dir: + cmd.append( + "-DALTERNATIVE_QT_INCLUDE_DIR=" + qt_include_dir + ) + elif sys.platform == "darwin": + if os.path.exists("/Library/Frameworks/QtCore.framework"): + cmd.append( + "-DALTERNATIVE_QT_INCLUDE_DIR=/Library/Frameworks" + ) + subprocess.check_call(cmd,env=env) + # The actual build program is "nmake" on win32 + if sys.platform == "win32": + cmd = ["nmake"] + else: + cmd = ["make"] + subprocess.check_call(cmd,env=env) + finally: + os.chdir(olddir) + + def get_build_env(self,env=None): + """Get environment variables for the build.""" + if env is None: + env = {} + def get_config_var(nm): + val = sysconfig.get_config_var(nm) + if val is None: + val = os.environ.get(nm) + return val + cc = get_config_var("CC") + if cc is not None: + env.setdefault("CC",cc) + cxx = get_config_var("CXX") + if cxx is not None: + env.setdefault("CXX",cxx) + cflags = get_config_var("CFLAGS") + if cflags is not None: + env.setdefault("CFLAGS",cflags) + cxxflags = env.get("CXXFLAGS",os.environ.get("CXXFLAGS","")) + cxxflags += " " + (sysconfig.get_config_var("CFLAGS") or "") + if sys.platform != "win32": + cxxflags += " -fno-exceptions" + if "linux" in sys.platform: + cxxflags += " -Wl,--gc-sections" + env["CXXFLAGS"] = cxxflags + if "linux" in sys.platform: + ldflags = env.get("LDFLAGS",os.environ.get("LDFLAGS","")) + ldflags += " " + sysconfig.get_config_var("LDFLAGS") + env["LDFLAGS"] = ldflags + return env + + def get_build_fingerprint(self): + """Get a unique fingerprint identifying all our build parameters. + + This method produces a unique fingerprint (actually an md5 hash) for + the full set of build parameters used in this Hatchet object. This + includes the PySide and PySideKick version, list of rejections, and + build flags. + """ + fp = hashlib.md5() + # Include info about python, pyside and pysidekick + fp.update(sys.version) + fp.update(sys.platform) + fp.update(sys.executable) + fp.update(self.SOURCE_URL) + fp.update(self.SOURCE_MD5) + fp.update(PySideKick.__version__) + try: + fp.update(inspect.getsource(sys.modules[__name__])) + except (NameError,KeyError,): + pass + # Include info about the build environment + for (k,v) in sorted(self.get_build_env().items()): + fp.update(k) + fp.update(v) + # Include info about the rejections used + # OK, I think that should cover it... + for rej in self.find_rejections(): + fp.update(str(rej)) + return fp.hexdigest() + + def copy_hacked_pyside_modules(self,sourcedir,destdir): + """Copy PySide modules from build dir back into the frozen app.""" + self.logger.debug("copying modules from %r => %r",sourcedir,destdir) + def is_dll(nm): + if nm.endswith(".so"): + return True + if nm.endswith(".so"): + return True + # Find all the build modules we're able to copy over + psdir = os.path.join(sourcedir,"PySide") + modules = [] + for modnm in os.listdir(psdir): + if modnm.startswith("Qt"): + if modnm.endswith(".so") or modnm.endswith(".pyd"): + modules.append(modnm) + # Search for similarly-named files in the destdir and replace them + for (dirnm,_,filenms) in os.walk(destdir): + for filenm in filenms: + filepath = os.path.join(dirnm,filenm) + newfilepath = None + # If it's a PySide module, try to copy new version + if "PySide" in filepath: + for modnm in modules: + if filenm.endswith(modnm): + newfilepath = os.path.join(psdir,modnm) + break + # If it's the pyside support lib, replace that as well + elif filenm.startswith("libpyside"): + newfilepath = os.path.join(sourcedir,"libpyside",filenm) + if not os.path.exists(newfilepath): + newfilepath = None + elif filenm.startswith("pyside") and filenm.endswith(".dll"): + newfilepath = os.path.join(sourcedir,"libpyside",filenm) + if not os.path.exists(newfilepath): + newfilepath = None + # If it's the shiboken lib, try to find that and replace it. + # This is necessary if it's a different version to the + # one bundled with the application. + elif "shiboken." in filenm: + if "CMAKE_INSTALL_PREFIX" in os.environ: + instprf = os.environ["CMAKE_INSTALL_PREFIX"] + for dirnm in ("bin","lib",): + newfilepath = os.path.join(instprf,dirnm,filenm) + if os.path.exists(newfilepath): + break + newfilepath = None + # Copy the new lib into place, and mangle it to look + # like the old one (e.g. linker paths). + if newfilepath is not None: + self.copy_linker_paths(filepath,newfilepath) + self.logger.info("copying %r => %r",newfilepath,filepath) + os.unlink(filepath) + shutil.copy2(newfilepath,filepath) + if "linux" in sys.platform: + try: + _do("strip",filepath) + except subprocess.CalledProcessError: + pass + elif sys.platform == "darwin": + try: + _do("strip","-S","-x",filepath) + except subprocess.CalledProcessError: + pass + + if sys.platform == "darwin": + def copy_linker_paths(self,srcfile,dstfile): + """Copy runtime linker paths from source to destination. + + On MacOSX, this uses install_name_tool to copy intallnames out + of the sourcefile and into the destfile. + """ + srclinks = _bt("otool","-L",srcfile).strip().split("\n") + dstlinks = _bt("otool","-L",dstfile).strip().split("\n") + for dstlink in dstlinks: + if "compatibility version" not in dstlink: + continue + dstlibpath = dstlink.strip().split()[0] + dstlibname = os.path.basename(dstlibpath) + for srclink in srclinks: + if "compatibility version" not in srclink: + continue + srclibpath = srclink.strip().split()[0] + srclibname = os.path.basename(srclibpath) + if srclibname == dstlibname: + _do("install_name_tool","-change", + dstlibpath,srclibpath,dstfile) + break + elif sys.platform == "win32": + def copy_linker_paths(self,srcfile,dstfile): + """Copy runtime linker paths from source to destination. + + On win32, this does nothing. + """ + pass + else: + def copy_linker_paths(self,srcfile,dstfile): + """Copy runtime linker paths from source to destination. + + On Linux-like platforms, this uses readelf and patchelf to copy + the rpath from sourcefile to destfile. + """ + rpath = None + for ln in _bt("readelf","-d",srcfile).split("\n"): + if "RPATH" in ln and "Library rpath:" in ln: + rpath = ln.rsplit("[",1)[1].split("]",1)[0] + break + if rpath is None: + for ln in _bt("readelf","-d",srcfile).split("\n"): + if "RUNPATH" in ln and "Library runpath:" in ln: + rpath = ln.rsplit("[",1)[1].split("]",1)[0] + break + if rpath is not None: + _do("patchelf","--set-rpath",rpath,dstfile) + + +def get_cache_dir(*paths): + """Get the directory in which we can cache downloads etc. + + This function uses the environment variable PYSIDEKICK_DOWNLOAD_CACHE, + or failing that PIP_DOWNLOAD_CACHE, to construct a directory in which + we can cache downloaded files and other expensive-to-generate content. + + If caching is disabled, None is returned. + """ + cachedir = os.environ.get("PYSIDEKICK_DOWNLOAD_CACHE",None) + if cachedir is None: + cachedir = os.environ.get("PIP_DOWNLOAD_CACHE",None) + if cachedir is not None: + cachedir = os.path.join(cachedir,"PySideKick") + if cachedir is not None: + cachedir = os.path.join(cachedir,*paths) + if not os.path.isdir(cachedir): + os.makedirs(cachedir) + return cachedir + + +def _do(*cmdline): + """A simple shortcut to execute the given command.""" + subprocess.check_call(cmdline) + + +def _bt(*cmdline): + """A simple shortbut to execute the command, returning stdout. + + "bt" is short for "backticks"; hopefully its use is obvious to shell + scripters and the like. + """ + p = subprocess.Popen(cmdline,stdout=subprocess.PIPE) + output = p.stdout.read() + retcode = p.wait() + if retcode != 0: + raise subprocess.CalledProcessError(retcode,cmdline) + return output + + +class TypeDB(object): + """PySide type database. + + A TypeDB instance encapsulates some basic information about the PySide API + and can be used to query e.g. what classes are available or what methods + are on a class. + + The current implementation gets this information in what might seem like + a very silly way - it pokes around in the online API documentation. This + has the advantage of being very quick to code up, and not requiring any + external dependencies. + + If PySide starts shipping with bindings for apiextractor, I'll write a + new version of this class to use those instead. + + Besides, parsing data out of the API docs isn't as fragile as it might + sound. The docs are themselves generated by parsing the source code, so + they have more than enough internal structure to support simple queries. + """ + + RE_CLASS_LINK=re.compile(r"<a href=\"(\w+).html\">([\w&;]+)</a>") + RE_METHOD_LINK=re.compile(r"<a href=\"(\w+).html\#([\w\-\.]+)\">(\w+)</a>") + + # These classes seem to be missing from the online docs. + # They are in the docs on the PySide website, but those don't seem + # to embed information about e.g. protected or virtual status. + MISSING_CLASSES = { + "QAudio": (), + "QPyTextObject": ("QTextObjectInterface","QObject",), + "QAbstractPageSetupDialog": ("QDialog",), + "QTextStreamManipulator": (), + "QFactoryInterface": (), + "QScriptExtensionInterface": ("QFactoryInterface",), + "QAudioEngineFactoryInterface": ("QFactoryInterface",), + "QAudioEnginePlugin": ("QAudioEngineFactoryInterface","QObject",), + "QAbstractAudioDeviceInfo": ("QObject",), + "QAbstractAudioOutput": ("QObject",), + "QAbstractAudioInput": ("QObject",), + "QDeclarativeExtensionInterface": (), + } + + # These methods seem to be missing from the online docs. + MISSING_METHODS = { + "QAbstractItemModel": { + "decodeData": (("QModelIndex","QList","QDataStream",), + True,False), + "encodeData": (("QModelIndex","QList","QDataStream",), + True,False), + }, + "QAbstractPageSetupDialog": { + "printer": (("QPrinter",), + True,False), + }, + "QScriptExtensionInterface": { + "initialize": (("QScriptEngine",), + True,False), + }, + "QAudioEngineFactoryInterface": { + "availableDevices": (("QAudio",), + True,False), + "createDeviceInfo": (("QByteArray","QAudio",), + True,False), + "createInput": (("QByteArray","QAudioFormat",), + True,False), + "createOutput": (("QByteArray","QAudioFormat",), + True,False), + }, + "QAbstractAudioDeviceInfo": { + "byteOrderList": ((),True,False), + "channelsList": ((),True,False), + "codecList": ((),True,False), + "deviceName": ((),True,False), + "frequencyList": ((),True,False), + "isFormatSupported": (("QAudioFormat",),True,False), + "nearestFormat": (("QAudioFormat",),True,False), + "preferredFormat": ((),True,False), + "sampleSizeList": ((),True,False), + "sampleTypeList": ((),True,False), + }, + "QAbstractAudioOutput": { + "bufferSize": ((),True,False), + "bytesFree": ((),True,False), + "elapsedUSecs": ((),True,False), + "error": ((),True,False), + "format": ((),True,False), + "notify": ((),True,False), + "notifyInterval": ((),True,False), + "periodSize": ((),True,False), + "processedUSecs": ((),True,False), + "reset": ((),True,False), + "resume": ((),True,False), + "setBufferSize": ((),True,False), + "setNotifyInterval": ((),True,False), + "start": (("QIODevice",),True,False), + "state": ((),True,False), + "stateChanged": ((),True,False), + "stop": ((),True,False), + "suspend": ((),True,False), + }, + "QAbstractAudioInput": { + "bufferSize": ((),True,False), + "bytesready": ((),True,False), + "elapsedUSecs": ((),True,False), + "error": ((),True,False), + "format": ((),True,False), + "notify": ((),True,False), + "notifyInterval": ((),True,False), + "periodSize": ((),True,False), + "processedUSecs": ((),True,False), + "reset": ((),True,False), + "resume": ((),True,False), + "setBufferSize": ((),True,False), + "setNotifyInterval": ((),True,False), + "start": (("QIODevice",),True,False), + "state": ((),True,False), + "stateChanged": ((),True,False), + "stop": ((),True,False), + "suspend": ((),True,False), + }, + "QDeclarativeExtensionInterface": { + "initializeEngine": (("QDeclarativeEngine",), + True,False), + "registerTypes": ((),True,False), + }, + } + + def __init__(self,root_url="http://doc.qt.nokia.com/4.7/",logger=None): + if not root_url.endswith("/"): + root_url += "/" + self.root_url = root_url + if logger is None: + logger = logging.getLogger("PySideKick.Hatchet") + self.logger = logger + + _url_cache = {} + def _read_url(self,url): + """Read the given URL, possibly using cached version.""" + url = urlparse.urljoin(self.root_url,url) + try: + return self._url_cache[url] + except KeyError: + pass + cachedir = get_cache_dir("Hatchet","QtDocTypeDB") + if cachedir is None: + cachefile = None + cachefile404 = None + else: + cachefile = os.path.join(cachedir,urllib.quote(url,"")) + cachefile404 = os.path.join(cachedir,"404_"+urllib.quote(url,"")) + if cachefile is not None: + try: + with open(cachefile,"rb") as f: + self._url_cache[url] = f.read() + return self._url_cache[url] + except EnvironmentError: + if os.path.exists(cachefile404): + msg = "not found: " + url + raise urllib2.HTTPError(url,"404",msg,{},None) + f = None + try: + self.logger.info("reading Qt API: %s",url) + f = urllib2.urlopen(url) + if f.geturl() != url: + msg = "not found: " + url + raise urllib2.HTTPError(url,"404",msg,{},None) + data = f.read() + except urllib2.HTTPError, e: + if "404" in str(e) and cachefile404 is not None: + open(cachefile404,"w").close() + raise + finally: + if f is not None: + f.close() + if cachefile is not None: + with open(cachefile,"wb") as f: + f.write(data) + self._url_cache[url] = data + return data + + def _get_linked_classes(self,data): + """Extract all class names linked to from the given HTML data.""" + for match in self.RE_CLASS_LINK.finditer(data): + # Careful now, it might inherit from an instantiated template + # type, e.g. QList<QItemSelectionRange>. We just yield both + # the template type and its argument. + if match.group(1) in match.group(2).lower(): + if "<" not in match.group(2): + yield match.group(2) + else: + yield match.group(2).split("<")[0] + yield match.group(2).split("<")[1].split(">")[0] + + def _get_linked_methods(self,data): + """Extract all method names linked to from the given HTML data.""" + for match in self.RE_METHOD_LINK.finditer(data): + if match.group(3) in match.group(2): + yield match.group(3) + + def _canonical_class_names(self,classnm): + """Get all canonical class names implied by the given identifier. + + This is a simple trick to decode common typedefs (e.g. QObjectList) + into their respective concrete classes (e.g. QObject and QList). + """ + if self.isclass(classnm): + yield classnm + else: + if classnm == "T": + # This appears as a generic template type variable + pass + elif classnm == "RawHeader": + yield "QPair" + yield "QByteArray" + elif classnm == "Event": + yield "QPair" + yield "QEvent" + yield "QWidget" + elif classnm.endswith("List"): + # These are usually typedefs for a QList<T> + found_classes = False + for cclassnm in self._canonical_class_names(classnm[:-4]): + found_classes = True + yield cclassnm + if found_classes: + yield "QList" + + def iterclasses(self): + """Iterator over all available class names.""" + for classnm in self.MISSING_CLASSES.iterkeys(): + yield classnm + # Everything else is conventiently listed on the "classes" page. + classlist = self._read_url("classes.html") + for ln in classlist.split("\n"): + ln = ln.strip() + if ln.startswith("<dd>"): + for classnm in self._get_linked_classes(ln): + yield classnm + break + + def isclass(self,classnm): + """Check whether the given name is indeed a class.""" + if classnm in self.MISSING_CLASSES: + return True + try: + self._read_url(classnm.lower()+".html") + except urllib2.HTTPError, e: + if "404" not in str(e) and "300" not in str(e): + raise + return False + else: + return True + + def superclasses(self,classnm): + """Get all superclasses for a given class.""" + yield classnm + if classnm in self.MISSING_CLASSES: + for bclassnm in self.MISSING_CLASSES[classnm]: + for sclassnm in self.superclasses(bclassnm): + yield sclassnm + return + docstr = self._read_url(classnm.lower()+".html") + for ln in docstr.split("\n"): + ln = ln.strip() + if "Inherits" in ln: + for supcls in self._get_linked_classes(ln): + for cname in self._canonical_class_names(supcls): + for supsupcls in self.superclasses(cname): + yield supsupcls + + def itermethods(self,classnm): + """Iterator over all methods on a given class.""" + # These methods are missing from the online docs. + if classnm in self.MISSING_METHODS: + for methnm in self.MISSING_METHODS[classnm]: + yield methnm + if classnm in self.MISSING_CLASSES: + for sclassnm in self.MISSING_CLASSES[classnm]: + for methnm in self.itermethods(sclassnm): + yield methnm + return + try: + docstr = self._read_url(classnm.lower()+"-members.html") + except urllib2.HTTPError, e: + if "404" not in str(e): + raise + assert self.isclass(classnm), "%r is not a class" % (classnm,) + else: + for ln in docstr.split("\n"): + ln = ln.strip() + if ln.startswith("<li class=\"fn\">"): + for methnm in self._get_linked_methods(ln): + yield methnm + + def relatedtypes(self,classnm,methnm): + """Get all possible return types for a method. + + Given a classname and methodname, this method returns the set of all + class names that are "related to" the specified method. Basically, + these are the classes that can be passed to the method as arguments + or returned from it as values. + """ + if classnm in self.MISSING_METHODS: + if methnm in self.MISSING_METHODS[classnm]: + for rtype in self.MISSING_METHODS[classnm][methnm][0]: + yield rtype + return + if classnm in self.MISSING_CLASSES: + for bclassnm in self.MISSING_CLASSES[classnm]: + for rtype in self.relatedtypes(bclassnm,methnm): + yield rtype + return + docstr = self._read_url(classnm.lower()+"-members.html") + for ln in docstr.split("\n"): + ln = ln.strip() + if ln.startswith("<li class=\"fn\">"): + if ">"+methnm+"<" not in ln: + continue + methsig = ln.rsplit("</b>",1)[-1][:-5] + # The method signature can contain plently of C++ + # junk, e.g. template instatiations and inner classes. + # We try our best to split them up into individual names. + for word in methsig.split(): + if word.endswith(","): + word = word[:-1] + word = word.split("::")[0] + if word.isalnum() and word[0].isupper(): + for cname in self._canonical_class_names(word): + yield cname + + def ispurevirtual(self,classnm,methnm): + """Check whether a given method is a pure virtual method.""" + if classnm in self.MISSING_METHODS: + if methnm in self.MISSING_METHODS[classnm]: + return self.MISSING_METHODS[classnm][methnm][1] + if classnm in self.MISSING_CLASSES: + for bclassnm in self.MISSING_CLASSES[classnm]: + if self.ispurevirtual(bclassnm,methnm): + return True + return False + # Pure virtual methods have a "= 0" at the end of their signature. + docstr = self._read_url(classnm.lower()+".html") + for ln in docstr.split("\n"): + ln = ln.strip() + if ln.startswith("<tr><td class=\"memItemLeft "): + if ">"+methnm+"<" not in ln: + continue + if "= 0</td>" in ln: + return True + return False + + + +def hack(appdir): + """Convenience function for hacking a frozen PySide app down to size. + + This function is a simple convenience wrapper that creates a Hatchet + instance and calls its main "hack" method. + """ + h = Hatchet(appdir) + h.hack() + + +if __name__ == "__main__": + import optparse + usage = "usage: Hatchet [options] /path/to/frozen/app [extra files]" + op = optparse.OptionParser(usage=usage) + op.add_option("-d","--debug",default="DEBUG", + help="set the logging debug level") + op.add_option("","--follow-imports", + action="store_true", + dest="follow_imports", + help="follow import when loading code", + default=True) + op.add_option("","--no-follow-imports", + action="store_false", + help="don't follow imports when loading code", + dest="follow_imports") + op.add_option("","--analyse-only", + action="store_true", + help="just analyse the code, don't hack it", + dest="analyse_only") + (opts,args) = op.parse_args() + try: + opts.debugs = int(opts.debug) + except ValueError: + try: + opts.debug = getattr(logging,opts.debug) + except AttributeError: + print >>sys.stderr, "unknown debug level:", opts.debug + sys.exit(1) + logging.basicConfig(level=opts.debug,format="%(name)-12s: %(message)s") + if len(args) < 1: + op.print_help() + sys.exit(1) + if not os.path.isdir(args[0]): + print >>sys.stderr, "error: not a directory:", args[0] + sys.exit(2) + h = Hatchet(args[0]) + for fnm in args[1:]: + if os.path.isdir(fnm): + h.add_directory(fnm,follow_imports=opts.follow_imports) + if fnm.endswith(".zip") or fnm.endswith(".exe"): + h.add_zipfile(fnm,follow_imports=opts.follow_imports) + else: + h.add_file(fnm,follow_imports=opts.follow_imports) + if not opts.analyse_only: + h.hack() + else: + logger = logging.getLogger("PySideKick.Hatchet") + if not h.mf.modules: + h.add_directory(h.appdir) + num_rejected_classes = 0 + num_rejected_methods = 0 + h.analyse_code() + for rej in h.find_rejections(): + if len(rej) == 1: + logger.debug("reject %s",rej[0]) + num_rejected_classes += 1 + else: + logger.debug("reject %s::%s",rej[0],rej[1]) + num_rejected_methods += 1 + logger.info("keeping %d classes",len(h.keep_classes)) + logger.info("rejecting %d classes, %d methods",num_rejected_classes, + num_rejected_methods) + + sys.exit(0) + + diff --git a/samples/dbg/PySideKick/__init__.py b/samples/dbg/PySideKick/__init__.py new file mode 100644 index 0000000..ec58456 --- /dev/null +++ b/samples/dbg/PySideKick/__init__.py @@ -0,0 +1,56 @@ +# Copyright (c) 2009-2010, Cloud Matrix Pty. Ltd. +# All rights reserved; available under the terms of the BSD License. +""" + +PySideKick: helpful utilities for working with PySide +====================================================== + + +This package is a rather ad-hoc collection of helpers, utilities and custom +widgets for building applications with PySide. So far we have: + + * PySideKick.Call: helpers for calling functions in a variety of ways, + e.g. qCallAfter, qCallInMainThread + + * PySideKick.Console: a simple interactive console to embed in your + application + + * PySideKick.Hatchet: a tool for hacking frozen PySide apps down to size, + by rebuilding PySide with a minimal set of classes + +""" + +__ver_major__ = 0 +__ver_minor__ = 2 +__ver_patch__ = 3 +__ver_sub__ = "" +__ver_tuple__ = (__ver_major__,__ver_minor__,__ver_patch__,__ver_sub__) +__version__ = "%d.%d.%d%s" % __ver_tuple__ + + +import thread + +from PySide import QtCore, QtGui +from PySide.QtCore import Qt + + +# Older versions of PySide don't expose the 'thread' attribute of QObject. +# In this case, assume the thread importing this module is the main thread. +if hasattr(QtCore.QCoreApplication,"thread"): + def qIsMainThread(): + app = QtCore.QCoreApplication.instance() + if app is None: + return False + return QtCore.QThread.currentThread() is app.thread() +else: + _MAIN_THREAD_ID = thread.get_ident() + def qIsMainThread(): + return thread.get_ident() == _MAIN_THREAD_ID + + +# PySideKick.Call needs to create a singleton object in the main gui thread. +# Since this is *usually* the thread that imports this module, loading it here +# will provide a small speedup in the common case. +import PySideKick.Call + + diff --git a/samples/dbg/commands/__init__.py b/samples/dbg/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/samples/dbg/commands/dbgcmd.py b/samples/dbg/commands/dbgcmd.py new file mode 100644 index 0000000..371bf64 --- /dev/null +++ b/samples/dbg/commands/dbgcmd.py @@ -0,0 +1,54 @@ +from widget import * + +class DebuggerController(BaseController): + + def __init__(self,dbgCore,mainWindow): + BaseController.__init__(self,dbgCore,mainWindow) + debugMenu = QMenu( "Debug" ) + + self.breakAction = QAction("Break", debugMenu ) + self.breakAction.triggered.connect( self.onBreak ) + self.breakAction.setDisabled(True) + debugMenu.addAction( self.breakAction ) + + self.goAction = QAction("Go", debugMenu ) + self.goAction.triggered.connect( self.onGo ) + self.goAction.setDisabled(True) + debugMenu.addAction( self.goAction ) + + self.stepAction = QAction("Step", debugMenu ) + self.stepAction.triggered.connect( self.onStep ) + self.stepAction.setDisabled(True) + debugMenu.addAction( self.stepAction ) + + mainWindow.menuBar().addMenu( debugMenu ) + + def onBreak( self ): + self.dbgCore.breakin() + + def onGo( self ): + self.dbgCore.go() + + def onStep( self ): + self.dbgCore.step() + + def onDbgBreak(self): + self.breakAction.setDisabled(True) + self.goAction.setDisabled(False) + self.stepAction.setDisabled(False) + + def onDbgRun(self): + self.breakAction.setDisabled(False) + self.goAction.setDisabled(True) + self.stepAction.setDisabled(True) + + def onDbgAttach(self): + self.breakAction.setDisabled(False) + self.goAction.setDisabled(False) + self.stepAction.setDisabled(False) + + def onDbgDetach(self): + self.breakAction.setDisabled(False) + self.goAction.setDisabled(False) + self.stepAction.setDisabled(False) + diff --git a/samples/dbg/commands/processcmd.py b/samples/dbg/commands/processcmd.py new file mode 100644 index 0000000..58aa135 --- /dev/null +++ b/samples/dbg/commands/processcmd.py @@ -0,0 +1,38 @@ + +from widget import * + +class ProcessController(BaseController): + + def __init__(self, dbgCore, mainWindow): + BaseController.__init__(self,dbgCore,mainWindow) + + self.openProcessAction = QAction( "Open process...", mainWindow.fileMenu ) + self.openProcessAction.triggered.connect(self.onOpenProcess) + mainWindow.fileMenu.addAction(self.openProcessAction) + + self.detachProcessAction = QAction( "Detach process", mainWindow.fileMenu ) + self.detachProcessAction.triggered.connect(self.onDetachProcess) + self.detachProcessAction.setDisabled(True) + mainWindow.fileMenu.addAction(self.detachProcessAction) + + def onOpenProcess(self): + fileDlg = QFileDialog( self.mainWnd ) + fileDlg.setNameFilter( "Executable (*.exe)" ) + self.dbgCore.openProcess( fileDlg.getOpenFileName()[0] ) + + def onDetachProcess(self): + self.dbgCore.detachProcess() + + def onDbgAttach(self): + self.openProcessAction.setDisabled(True) + self.detachProcessAction.setDisabled(True) + + def onDbgDetach(self): + self.openProcessAction.setDisabled(False) + self.detachProcessAction.setDisabled(True) + + def onDbgBreak(self): + self.detachProcessAction.setDisabled(False) + + def onDbgRun(self): + self.detachProcessAction.setDisabled(True) \ No newline at end of file diff --git a/samples/dbg/dbg.py b/samples/dbg/dbg.py new file mode 100644 index 0000000..d18fe2b --- /dev/null +++ b/samples/dbg/dbg.py @@ -0,0 +1,40 @@ + +from PySide.QtCore import * +from PySide.QtGui import * + + +from dbgcore import DbgCore +from settings import settings + +class MainForm(QMainWindow): + + def __init__(self): + QMainWindow.__init__(self, None) + self.resize( 800, 600 ) + self.setWindowTitle("Pykd Debugger Sample") + self.setDockNestingEnabled( True ) + + self.dbgCore = DbgCore() + + self.fileMenu = QMenu( "&File" ) + self.menuBar().addMenu( self.fileMenu ) + + self.viewMenu = QMenu( "View" ) + self.menuBar().addMenu( self.viewMenu ) + + self.widgets = settings["default"](self.dbgCore, self ) + + self.fileMenu.addAction( "Exit", self.onExit ) + + def onExit(self): + self.dbgCore.close() + self.close() + +def main(): + app = QApplication( [] ) + mainForm = MainForm() + mainForm.show() + exitres = app.exec_() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/samples/dbg/dbgcore.py b/samples/dbg/dbgcore.py new file mode 100644 index 0000000..075005d --- /dev/null +++ b/samples/dbg/dbgcore.py @@ -0,0 +1,70 @@ + +import pykd + +from PySide.QtCore import QThread +from PySide.QtCore import QObject +from PySide.QtCore import Signal + +class DbgThread( QThread ): + + def __init__(self, func): + QThread.__init__(self) + self.func = func + + def run(self): + self.func() + self.exit() + +class DbgCore( QObject ): + + targetBreak = Signal() + targetRunning = Signal() + targetAttached = Signal() + targetDetached = Signal() + + def close(self): + if self.processOpened: + if self.thread != None: + self.breakin() + + def openProcess( self, name ): + pykd.startProcess( name ) + self.processOpened = True + self.targetAttached.emit() + self.targetBreak.emit() + + def detachProcess(self): + pykd.detachProcess() + self.processOpened = False + self.targetDetached.emit() + + def killProcess(self): + pykd.killProcess() + self.processOpened = False + self.targetDetached.emit() + + + def breakin( self ): + pykd.breakin() + + def go( self ): + self.thread = DbgThread( pykd.go ) + self.thread.finished.connect( self.onDebugStop ) + self.targetRunning.emit() + self.thread.start() + + def step( self ): + self.thread = DbgThread( pykd.step ) + self.thread.finished.connect( self.onDebugStop ) + self.targetRunning.emit() + self.thread.start() + + def onDebugStop(self): + self.thread.wait(100) + self.thread = None + self.targetBreak.emit() + + def __init__(self): + QObject.__init__(self) + self.thread = None + self.processOpened = False \ No newline at end of file diff --git a/samples/dbg/settings.py b/samples/dbg/settings.py new file mode 100644 index 0000000..7418a69 --- /dev/null +++ b/samples/dbg/settings.py @@ -0,0 +1,18 @@ + +from commands.processcmd import ProcessController +from commands.dbgcmd import DebuggerController + +from widgets.registers import RegisterWidget +from widgets.cmd import CmdWidget + + +class DefaultSettings: + + def __init__( self, dbgcore, parentwnd ): + self.processCmd = ProcessController( dbgcore, parentwnd ) + self.debugCmd = DebuggerController( dbgcore, parentwnd ) + + self.regWidget = RegisterWidget( dbgcore, parentwnd ) + self.cmdWidget = CmdWidget( dbgcore, parentwnd ) + +settings = { "default" : DefaultSettings } diff --git a/samples/dbg/widget.py b/samples/dbg/widget.py new file mode 100644 index 0000000..79f4ec8 --- /dev/null +++ b/samples/dbg/widget.py @@ -0,0 +1,97 @@ + +from PySide.QtCore import * +from PySide.QtGui import * + +class BaseWidget( QDockWidget ): + + def __init__( self, dbgCore, mainWindow, title = "", visible = False ): + QDockWidget.__init__( self ) + self.setWindowTitle( title ) + self.setVisible( visible ) + mainWindow.addDockWidget( Qt.LeftDockWidgetArea, self ) + + self.dbgCore=dbgCore + self.mainWnd = mainWindow + dbgCore.targetBreak.connect( self.onDbgBreak ) + dbgCore.targetRunning.connect( self.onDbgRun ) + dbgCore.targetAttached.connect( self.onDbgAttach ) + dbgCore.targetDetached.connect( self.onDbgDetach ) + + def addMenuTriggerAction( self, actionName ): + self.action = QAction( actionName, self.mainWnd ) + self.action.triggered.connect(self.onTriggerAction) + self.action.setDisabled( True ) + self.mainWnd.viewMenu.addAction(self.action) + + def onDbgBreak(self): + pass + + def onDbgRun(self): + pass + + def onDbgAttach(self): + pass + + def onDbgDetach(self): + pass + + def onTriggerAction(self): + self.setVisible( not self.isVisible() ) + + +class DebugWidget( BaseWidget ): + + def __init__( self, dbgCore, mainWindow, title = "", visible = False ): + BaseWidget.__init__( self, dbgCore, mainWindow, title, visible ) + self.action = None + + def onDbgAttach(self): + if self.action != None: + self.action.setDisabled( True ) + self.setDisabled( True ) + + def onDbgDetach(self): + if self.action != None: + self.action.setDisabled( True ) + self.setDisabled( True ) + + def onDbgBreak(self): + if self.action != None: + self.action.setDisabled( False ) + self.setDisabled( False ) + self.updateView() + + def onDbgRun(self): + if self.action != None: + self.action.setDisabled( True ) + self.setDisabled( True ) + + def updateView(self): + pass + + +class BaseController(QObject): + + def __init__(self,dbgCore,mainWindow): + QObject.__init__(self,mainWindow) + self.dbgCore=dbgCore + self.mainWnd = mainWindow + dbgCore.targetBreak.connect( self.onDbgBreak ) + dbgCore.targetRunning.connect( self.onDbgRun ) + dbgCore.targetAttached.connect( self.onDbgAttach ) + dbgCore.targetDetached.connect( self.onDbgDetach ) + + def onDbgBreak(self): + self.onStateChange() + + def onDbgRun(self): + self.onStateChange() + + def onDbgAttach(self): + self.onStateChange() + + def onDbgDetach(self): + self.onStateChange() + + def onStateChange(self): + pass \ No newline at end of file diff --git a/samples/dbg/widgets/__init__.py b/samples/dbg/widgets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/samples/dbg/widgets/cmd.py b/samples/dbg/widgets/cmd.py new file mode 100644 index 0000000..22e383c --- /dev/null +++ b/samples/dbg/widgets/cmd.py @@ -0,0 +1,23 @@ + +from widget import * +from PySideKick.Console import QPythonConsole + +class CmdWidget( DebugWidget ): + + def __init__(self, dbgCore, mainWindow, visible = False ): + BaseWidget.__init__( self, dbgCore, mainWindow, "Commands", visible ) + + self.addMenuTriggerAction( "Commands" ) + self.console = QPythonConsole() + self.setWidget( self.console ) + self.console.interpreter.push("from pykd import *") + + + + + + + + + + diff --git a/samples/dbg/widgets/registers.py b/samples/dbg/widgets/registers.py new file mode 100644 index 0000000..7a55f58 --- /dev/null +++ b/samples/dbg/widgets/registers.py @@ -0,0 +1,40 @@ +from widget import * +import pykd + +class RegisterWidget( DebugWidget ): + + def __init__(self, dbgCore, mainWindow, visible = False ): + BaseWidget.__init__( self, dbgCore, mainWindow, "Registers", visible ) + + self.addMenuTriggerAction( "Registers" ) + + self.textArea = QTextEdit() + self.setWidget( self.textArea ) + + def updateView(self): + + s = "" + + try: + i = 0 + while True: + reg = pykd.reg(i) + s += "%s %x ( %d )\r\n" % ( reg.name(), reg, reg ) + i += 1 + + except pykd.BaseException: + pass + + self.textArea.setPlainText( s ) + + + + + + + + + + + +