From 03c7468fcc51888c8271b904e4d6400ed5c1cbb1 Mon Sep 17 00:00:00 2001 From: Thomas Grill Date: Wed, 7 Mar 2007 13:40:14 +0000 Subject: multiply inlets for py (hot and cold inlets) small optimizations and fixes use PyGILState_\*() functionality (enabled with PY_USE_GIL) updates for DSP processing __str__ method for pyext, to enable print self calls added message bundle functionality (pyext.Bundle class) enable compiled-only scripts (without .py) enable optimization of Python code in reease build let _inlets and _outlets default to 0 fix for numpy some ASSERTs for explicitly created pyext classes (should be runtime checks i guess) open editor for script under OS X fixing numpy initialization quirks enable symbol binding for all callables (not only functions and methods) _isthreaded is now a data member instead of a method fix for gcc4 added pyext._list and pyext._tuple to convert input lists to Python sequence objects enable module packages (module/__init__.py[co]), now also for Max python-like dotted module.function syntax cleaned up float vs. int pyext tags compiler flag to exclude DSP objects some optimizations and py reload fix more safety for calls where association python-pd has already been removed always run Python interpreter in the background svn path=/trunk/; revision=7474 --- externals/grill/py/source/pybase.cpp | 1793 +++++++++++++++++----------------- 1 file changed, 912 insertions(+), 881 deletions(-) (limited to 'externals/grill/py/source/pybase.cpp') diff --git a/externals/grill/py/source/pybase.cpp b/externals/grill/py/source/pybase.cpp index 5c9dd40c..a74df81e 100644 --- a/externals/grill/py/source/pybase.cpp +++ b/externals/grill/py/source/pybase.cpp @@ -1,881 +1,912 @@ -/* - -py/pyext - python external object for PD and MaxMSP - -Copyright (c)2002-2006 Thomas Grill (gr@grrrr.org) -For information on usage and redistribution, and for a DISCLAIMER OF ALL -WARRANTIES, see the file, "license.txt," in this distribution. - -*/ - -#include "pybase.h" -#include - -#if FLEXT_OS == FLEXT_OS_WIN -#include -#elif FLEXT_OS == FLEXT_OS_MAC -#include -#endif - -static PyMethodDef StdOut_Methods[] = -{ - { "write", pybase::StdOut_Write, 1 }, - { NULL, NULL, } -}; - -static PyObject *gcollect = NULL; - -#ifdef FLEXT_THREADS - -class ThrCmp -{ -public: - inline bool operator()(const flext::thrid_t &a,const flext::thrid_t &b) const - { - if(sizeof(a) == sizeof(size_t)) - return *(size_t *)&a < *(size_t *)&b; - else - return memcmp(&a,&b,sizeof(a)) < 0; - } -}; - -typedef std::map PyThrMap; - -static PyInterpreterState *pymain = NULL; -static PyThrMap pythrmap; -PyThreadState *pybase::pythrsys = NULL; - -int pybase::lockcount = 0; - -PyThreadState *pybase::FindThreadState() -{ - flext::thrid_t id = flext::GetThreadId(); - PyThrMap::iterator it = pythrmap.find(id); - if(it == pythrmap.end()) { - // Make new thread state - PyThreadState *st = PyThreadState_New(pymain); - pythrmap[id] = st; - return st; - } - else - return it->second; -} - -void pybase::FreeThreadState() -{ - flext::thrid_t id = flext::GetThreadId(); - PyThrMap::iterator it = pythrmap.find(id); - if(it != pythrmap.end()) { - // clear out any cruft from thread state object - PyThreadState_Clear(it->second); - // delete my thread state object - PyThreadState_Delete(it->second); - // delete from map - pythrmap.erase(it); - } -} - -PyFifo pybase::qufifo; -flext::ThrCond pybase::qucond; -#endif - - -PyObject *pybase::module_obj = NULL; -PyObject *pybase::module_dict = NULL; - -PyObject *pybase::builtins_obj = NULL; -PyObject *pybase::builtins_dict = NULL; - -const t_symbol *pybase::sym_fint = NULL; -const t_symbol *pybase::sym_response = NULL; - -// ----------------------------------------------------------------------------------------------------------- - - -void initsymbol(); -void initsamplebuffer(); -void initbundle(); - -void pybase::lib_setup() -{ - post(""); - post("------------------------------------------------"); - post("py/pyext %s - python script objects",PY__VERSION); - post("(C)2002-2006 Thomas Grill - http://grrrr.org/ext"); - post(""); - post("using Python %s",Py_GetVersion()); - -#ifdef FLEXT_DEBUG - post(""); - post("DEBUG version compiled on %s %s",__DATE__,__TIME__); -#endif - - // ------------------------------------------------------------- - - sym_response = flext::MakeSymbol("response"); - -#if FLEXT_SYS == FLEXT_SYS_PD - sym_fint = sym_float; -#else - sym_fint = sym_int; -#endif - - // ------------------------------------------------------------- - - Py_Initialize(); - -#ifdef FLEXT_DEBUG - Py_DebugFlag = 1; -// Py_VerboseFlag = 1; -#else - Py_OptimizeFlag = 1; -#endif - -#ifdef FLEXT_THREADS - // enable thread support and acquire the global thread lock - PyEval_InitThreads(); - - // get thread state - pythrsys = PyThreadState_Get(); - // get main interpreter state - pymain = pythrsys->interp; - - // add thread state of main thread to map - pythrmap[GetThreadId()] = pythrsys; -#endif - - // sys.argv must be set to empty tuple - char *nothing = ""; - PySys_SetArgv(0,¬hing); - - // register/initialize pyext module only once! - module_obj = Py_InitModule(PYEXT_MODULE, func_tbl); - module_dict = PyModule_GetDict(module_obj); // borrowed reference - - PyModule_AddStringConstant(module_obj,"__doc__",(char *)py_doc); - - // redirect stdout - PyObject* py_out; - py_out = Py_InitModule("stdout", StdOut_Methods); - PySys_SetObject("stdout", py_out); - py_out = Py_InitModule("stderr", StdOut_Methods); - PySys_SetObject("stderr", py_out); - - // get garbage collector function - PyObject *gcobj = PyImport_ImportModule("gc"); - if(gcobj) { - gcollect = PyObject_GetAttrString(gcobj,"collect"); - Py_DECREF(gcobj); - } - - builtins_obj = PyImport_ImportModule("__builtin__"); - builtins_dict = PyModule_GetDict(builtins_obj); // borrowed reference - - // add symbol type - initsymbol(); - PyModule_AddObject(module_obj,"Symbol",(PyObject *)&pySymbol_Type); - - // pre-defined symbols - PyModule_AddObject(module_obj,"_s_",(PyObject *)pySymbol__); - PyModule_AddObject(module_obj,"_s_bang",(PyObject *)pySymbol_bang); - PyModule_AddObject(module_obj,"_s_list",(PyObject *)pySymbol_list); - PyModule_AddObject(module_obj,"_s_symbol",(PyObject *)pySymbol_symbol); - PyModule_AddObject(module_obj,"_s_float",(PyObject *)pySymbol_float); - PyModule_AddObject(module_obj,"_s_int",(PyObject *)pySymbol_int); - - // add samplebuffer type - initsamplebuffer(); - PyModule_AddObject(module_obj,"Buffer",(PyObject *)&pySamplebuffer_Type); - - // add message bundle type - initbundle(); - PyModule_AddObject(module_obj,"Bundle",(PyObject *)&pyBundle_Type); - - // ------------------------------------------------------------- - - FLEXT_SETUP(pyobj); - FLEXT_SETUP(pymeth); - FLEXT_SETUP(pyext); -#ifndef PY_NODSP - FLEXT_DSP_SETUP(pydsp); -#endif - -#ifdef FLEXT_THREADS - // release global lock - PyEval_ReleaseLock(); - - // launch thread worker - LaunchThread(quworker,NULL); -#endif - - post("------------------------------------------------"); - post(""); -} - -FLEXT_LIB_SETUP(py,pybase::lib_setup) - - -// ----------------------------------------------------------------------------------------------------------- - - -pybase::pybase() - : module(NULL) - , dict(NULL) -#ifdef FLEXT_THREADS - , thrcount(0) - , shouldexit(false),stoptick(0) -#endif - , detach(0) - , pymsg(false) -{ - PyThreadState *state = PyLock(); - Py_INCREF(module_obj); - PyUnlock(state); -} - -pybase::~pybase() -{ - PyThreadState *state = PyLock(); - Py_XDECREF(module_obj); - PyUnlock(state); -} - -void pybase::Exit() -{ -#ifdef FLEXT_THREADS - erasethreads(); - - shouldexit = true; - qucond.Signal(); - - if(thrcount) { - // Wait for a certain time - for(int i = 0; i < (PY_STOP_WAIT/PY_STOP_TICK) && thrcount; ++i) - Sleep(PY_STOP_TICK*0.001f); - if(thrcount) { - // Wait forever - post("py/pyext - Waiting for thread termination!"); - while(thrcount) Sleep(PY_STOP_TICK*0.001f); - post("py/pyext - Okay, all threads have terminated"); - } - } -#endif -} - -void pybase::GetDir(PyObject *obj,AtomList &lst) -{ - if(obj) { - PyThreadState *state = PyLock(); - - PyObject *pvar = PyObject_Dir(obj); - if(!pvar) - PyErr_Print(); // no method found - else { - const t_symbol *sym = GetPyArgs(lst,pvar); - if(!sym) - post("py/pyext - Argument list could not be created"); - else - FLEXT_ASSERT(sym == sym_list); - Py_DECREF(pvar); - } - - PyUnlock(state); - } -} - -void pybase::m__dir(PyObject *obj) -{ - AtomList lst; - GetDir(obj,lst); - // dump dir to attribute outlet - DumpOut(NULL,lst.Count(),lst.Atoms()); -} - -void pybase::m__doc(PyObject *obj) -{ - if(obj) { - PyThreadState *state = PyLock(); - - PyObject *docf = PyDict_GetItemString(obj,"__doc__"); // borrowed!!! - if(docf && PyString_Check(docf)) { - post(""); - const char *s = PyString_AS_STRING(docf); - - // FIX: Python doc strings can easily be larger than 1k characters - // -> split into separate lines - for(;;) { - char buf[1024]; - char *nl = strchr((char *)s,'\n'); // the cast is for Borland C++ - if(!nl) { - // no more newline found - post(s); - break; - } - else { - // copy string before newline to temp buffer and post - unsigned int l = nl-s; - if(l >= sizeof(buf)) l = sizeof buf-1; - strncpy(buf,s,l); // copy all but newline - buf[l] = 0; - post(buf); - s = nl+1; // set after newline - } - } - } - - PyUnlock(state); - } -} - -void pybase::OpenEditor() -{ - if(!module) return; - const char *mname = PyModule_GetFilename(module); - if(!mname) { - PyErr_Clear(); - return; - } - - char fname[1024]; - strcpy(fname,mname); - - // replacing .pyc or .pyo for source file name - char *dt = strrchr(fname,'.'); - if(dt && !strncmp(dt,".py",2)) strcpy(dt,".py"); - - // this should open the editor.... -#if FLEXT_OS == FLEXT_OS_WIN - int err = (int)ShellExecute(NULL,"edit",fname,NULL,NULL,SW_SHOW); - if(err == SE_ERR_NOASSOC) { - // no association found - try notepad - err = (int)ShellExecute(NULL,NULL,"notepad.exe",fname,NULL,SW_SHOW); - } - else if(err == ERROR_FILE_NOT_FOUND || err == SE_ERR_FNF) - post("py/pyext - File not %s found",fname); - else if(err <= 32) - post("py/pyext - Unknown error opening %s",fname); - -#elif FLEXT_OS == FLEXT_OS_MAC - FSRef ref; - OSStatus err = FSPathMakeRef((unsigned char *)fname,&ref,NULL); - if(err) - post("py/pyext - Error interpreting path %s",fname); - else { - FSRef editor; - err = LSGetApplicationForItem(&ref,kLSRolesEditor,&editor,NULL); - if(err) { - // Can't find associated application... try Textedit - err = FSPathMakeRef((unsigned char *)"/Applications/TextEdit.app",&editor,NULL); - if(err) - post("py/pyext - Can't find Textedit application"); - } - - if(!err) { - LSLaunchFSRefSpec lspec; - lspec.appRef = &editor; - lspec.numDocs = 1; - lspec.itemRefs = &ref; - lspec.passThruParams = NULL; - lspec.launchFlags = kLSLaunchDefaults; - lspec.asyncRefCon = NULL; - err = LSOpenFromRefSpec(&lspec,NULL); - if(err) - post("py/pyext - Couldn't launch editor"); - } - } -#else - // thanks to Tim Blechmann - - char *editor = getenv("EDITOR"); - - if(!editor) { // || !strcmp(editor, "/usr/bin/nano") || !strcmp(editor, "/usr/bin/pico") || !strcmp(editor, "/usr/bin/vi")) { - // no environment variable or console text editor found ... use idle instead (should have come with Python) - editor = "idle"; - } - - pid_t child = fork(); - if(!child) { - char cmd[80]; - strcpy(cmd,editor); - strcat(cmd," "); - strcat(cmd,fname); - execl("/bin/sh", "sh", "-c", cmd, (char *) NULL); - } -#endif -} - -void pybase::SetArgs() -{ - // script arguments - int argc = args.Count(); - const t_atom *argv = args.Atoms(); - char **sargv = new char *[argc+1]; - for(int i = 0; i <= argc; ++i) { - sargv[i] = new char[256]; - if(!i) - strcpy(sargv[i],"py/pyext"); - else - GetAString(argv[i-1],sargv[i],255); - } - - // the arguments to the module are only recognized once! (at first use in a patcher) - PySys_SetArgv(argc+1,sargv); - - for(int j = 0; j <= argc; ++j) delete[] sargv[j]; - delete[] sargv; -} - -static bool getmodulesub(const char *mod,char *dir,int len,char *ext) -{ -#if FLEXT_SYS == FLEXT_SYS_PD - char *name; - int fd = open_via_path("",mod,ext,dir,&name,len,0); - if(fd > 0) { - FLEXT_ASSERT(name && *name); - close(fd); - } - else { - // look for mod/__init__.py - std::string tmp(mod); - int l = tmp.size(); - tmp += "/__init__"; - fd = open_via_path("",tmp.c_str(),ext,dir,&name,len,0); - if(fd > 0) { - FLEXT_ASSERT(name && *name); - close(fd); - // we must remove the module name from dir - char *t = dir+strlen(dir)-l; - FLEXT_ASSERT(!strcmp(mod,t) && t[-1] == '/'); - t[-1] = 0; - } - else - name = NULL; - } - - // if dir is current working directory... name points to dir - if(dir == name) strcpy(dir,"."); - return name != NULL; -#elif FLEXT_SYS == FLEXT_SYS_MAX - short path; - long type; - char smod[1024]; - strcpy(smod,mod); - strcat(smod,ext); - bool ok = !locatefile_extended(smod,&path,&type,&type,0); - if(ok) - // convert pathname to unix style - path_topathname(path,NULL,smod); - else { - // do path/file.ext combinations work at all under Max? - strcpy(smod,mod); - - short path; - type = 'fold'; - ok = !locatefile_extended(smod,&path,&type,&type,1); - if(ok) { - // convert pathname to unix style (including trailing slash) - path_topathname(path,NULL,smod); - char *end = smod+strlen(smod); - strcpy(end,mod); - strcat(end,"/__init__"); - strcat(end,ext); - - // check if file is really existing: try to open it - FILE *f = fopen(smod,"r"); - if(f) { - *end = 0; // clear module part ... we only need base path - fclose(f); - } - else - ok = false; - } - } - - if(ok) { - // convert path into slash style needed for Python -#if 0 - // Max API function uses Volume:/Path notation - path_nameconform(smod,dir,PATH_STYLE_SLASH,PATH_TYPE_ABSOLUTE); -#else -#if FLEXT_OS == FLEXT_OS_WIN - char *colon = NULL; -#else - char *colon = strchr(smod,':'); -#endif - if(colon) { - *colon = 0; - strcpy(dir,"/Volumes/"); - strcat(dir,smod); - strcat(dir,colon+1); - } - else - strcpy(dir,smod); -#endif - return true; - } - else - // not found - return false; -#else -#pragma message("Not implemented"); - return false; -#endif -} - -static bool getmodulepath(const char *mod,char *dir,int len) -{ - return - getmodulesub(mod,dir,len,".py") || - getmodulesub(mod,dir,len,".pyc") || - getmodulesub(mod,dir,len,".pyo"); -} - -bool pybase::ImportModule(const char *name) -{ - if(name) { - if(modname == name) { - // module with the same name is already loaded - if(module) return true; - } - else - modname = name; - } - else - modname.clear(); - - UnimportModule(); - return ReloadModule(); -} - -void pybase::UnimportModule() -{ - if(module) { - FLEXT_ASSERT(dict && module_obj && module_dict); - - Py_DECREF(module); - - // reference count to module is not 0 here, altough probably the last instance was unloaded - // Python retains one reference to the module all the time - // we don't care - - module = NULL; - dict = NULL; - } -} - -bool pybase::ReloadModule() -{ - SetArgs(); - PyObject *newmod; - - if(modname.length()) { - if(module) - newmod = PyImport_ReloadModule(module); - else { - // search in module path (TODO: check before if module is already present to avoid costly searching) - char dir[1024]; - if(!getmodulepath(modname.c_str(),dir,sizeof(dir))) - PyErr_SetString(PyExc_ImportError,"Module not found in path"); - else - AddToPath(dir); - - // module could also be loaded ok, even if it's not in the path (e.g. for internal stuff) - newmod = PyImport_ImportModule((char *)modname.c_str()); - } - } - else { - // if no module name given, take py module - newmod = module_obj; - Py_INCREF(newmod); - } - - if(!newmod) { - // unload faulty module - UnimportModule(); - return false; - } - else { - Py_XDECREF(module); - module = newmod; - dict = PyModule_GetDict(module); // borrowed - return true; - } -} - -void pybase::AddToPath(const char *dir) -{ - if(dir && *dir) { - PyObject *pobj = PySys_GetObject("path"); - if(pobj && PyList_Check(pobj)) { - PyObject *ps = PyString_FromString(dir); - if(!PySequence_Contains(pobj,ps)) - PyList_Append(pobj,ps); // makes new reference - Py_DECREF(ps); - } - PySys_SetObject("path",pobj); // steals reference to pobj - } -} - -void pybase::AddCurrentPath(flext_base *o) -{ - char dir[1024]; - - // add dir of current patch to path - o->GetCanvasDir(dir,sizeof(dir)); - if(*dir) AddToPath(dir); - - // add current dir to path -#if FLEXT_SYS == FLEXT_SYS_PD - AddToPath(GetString(canvas_getcurrentdir())); -#elif FLEXT_SYS == FLEXT_SYS_MAX - short path = path_getdefault(); - path_topathname(path,NULL,dir); - AddToPath(dir); -#endif -} - -bool pybase::OutObject(flext_base *ext,int o,PyObject *obj) -{ - flext::AtomListStatic<16> lst; - const t_symbol *sym = pymsg?GetPyAtom(lst,obj):GetPyArgs(lst,obj); - if(sym) { - // call to outlet _outside_ the Mutex lock! - // otherwise (if not detached) deadlock will occur - ext->ToOutAnything(o,sym,lst.Count(),lst.Atoms()); - return true; - } - else - return false; -} - -void pybase::Reload() -{ - PyThreadState *state = PyLock(); - - PyObject *reg = GetRegistry(REGNAME); - - if(reg) { - PyObject *key; - int pos = 0; - while(PyDict_Next(reg,&pos,&key,NULL)) { - pybase *th = (pybase *)PyLong_AsLong(key); - FLEXT_ASSERT(th); - th->Unload(); - } - - UnloadModule(); - } - - bool ok = ReloadModule(); - - if(ok) { - LoadModule(); - - if(reg) { - SetRegistry(REGNAME,reg); - - PyObject *key; - int pos = 0; - while(PyDict_Next(reg,&pos,&key,NULL)) { - pybase *th = (pybase *)PyLong_AsLong(key); - FLEXT_ASSERT(th); - th->Load(); - } - } - else - Load(); - } - - Report(); - PyUnlock(state); -} - -static PyObject *output = NULL; - -// post to the console -PyObject* pybase::StdOut_Write(PyObject* self, PyObject* args) -{ - // should always be a tuple - FLEXT_ASSERT(PyTuple_Check(args)); - - const int sz = PyTuple_GET_SIZE(args); - - for(int i = 0; i < sz; ++i) { - PyObject *val = PyTuple_GET_ITEM(args,i); // borrowed reference - PyObject *str = PyObject_Str(val); // new reference - char *cstr = PyString_AS_STRING(str); - char *lf = strchr(cstr,'\n'); - - // line feed in string - if(!lf) { - // no -> just append - if(output) - PyString_ConcatAndDel(&output,str); // str is decrefd - else - output = str; // take str reference - } - else { - // yes -> append up to line feed, reset output buffer to string remainder - PyObject *part = PyString_FromStringAndSize(cstr,lf-cstr); // new reference - if(output) - PyString_ConcatAndDel(&output,part); // str is decrefd - else - output = part; // take str reference - - // output concatenated string - post(PyString_AS_STRING(output)); - - Py_DECREF(output); - output = PyString_FromString(lf+1); // new reference - } - } - - Py_INCREF(Py_None); - return Py_None; -} - - -class work_data -{ -public: - work_data(PyObject *f,PyObject *a): fun(f),args(a) {} - ~work_data() { Py_DECREF(fun); Py_DECREF(args); } - - PyObject *fun,*args; -}; - -bool pybase::gencall(PyObject *pmeth,PyObject *pargs) -{ - bool ret = false; - - // Now call method - switch(detach) { - case 0: - ret = docall(pmeth,pargs); - Py_DECREF(pargs); - Py_DECREF(pmeth); - break; -#ifdef FLEXT_THREADS - case 1: - // put call into queue - ret = qucall(pmeth,pargs); - break; - case 2: - // each call a new thread - if(!shouldexit) { - thr_params *p = new thr_params; - p->cl = (flext_base *)this; - p->var->_ext = new work_data(pmeth,pargs); - ret = LaunchThread(thrworker,p); - if(!ret) post("py/pyext - Failed to launch thread!"); - } - break; -#endif - default: - post("py/pyext - Unknown detach mode"); - } - return ret; -} - -void pybase::exchandle() -{ -#if 0 - // want to use that, but exception keeps a reference to the object - // might be a Python bug! - PyErr_Print(); -#else - // must use that instead... clear the exception - PyObject *type,*value,*traceback; - PyErr_Fetch(&type,&value,&traceback); - PyErr_NormalizeException(&type,&value,&traceback); - PyErr_Display(type,value,traceback); - - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(traceback); -#endif -} - -#ifdef FLEXT_THREADS -void pybase::thrworker(thr_params *p) -{ - FLEXT_ASSERT(p); - pybase *th = (pybase *)p->cl; - work_data *w = (work_data *)p->var->_ext; - - ++th->thrcount; // \todo this should be atomic - PyThreadState *state = PyLock(); - - // call worker - th->docall(w->fun,w->args); - delete w; - - PyUnlock(state); - --th->thrcount; // \todo this should be atomic -} - -void pybase::quworker(thr_params *) -{ - FifoEl *el; - PyThreadState *my = FindThreadState(),*state; - - for(;;) { - while(el = qufifo.Get()) { - ++el->th->thrcount; // \todo this should be atomic - state = PyLock(my); - el->th->docall(el->fun,el->args); - Py_XDECREF(el->fun); - Py_XDECREF(el->args); - PyUnlock(state); - --el->th->thrcount; // \todo this should be atomic - qufifo.Free(el); - } - qucond.Wait(); - } - - // we never end -#if 0 - state = PyLock(my); - // unref remaining Python objects - while(el = qufifo.Get()) { - Py_XDECREF(el->fun); - Py_XDECREF(el->args); - qufifo.Free(el); - } - PyUnlock(state); -#endif -} - -void pybase::erasethreads() -{ - PyFifo tmp; - FifoEl *el; - while(el = qufifo.Get()) { - if(el->th == this) { - Py_XDECREF(el->fun); - Py_XDECREF(el->args); - qufifo.Free(el); - } - else - tmp.Put(el); - } - // Push back - while(el = tmp.Get()) qufifo.Put(el); -} -#endif - -bool pybase::collect() -{ - if(gcollect) { - PyObject *ret = PyObject_CallObject(gcollect,NULL); - if(ret) { -#ifdef FLEXT_DEBUG - int refs = PyInt_AsLong(ret); - if(refs) post("py/pyext - Garbage collector reports %i unreachable objects",refs); -#endif - Py_DECREF(ret); - return false; - } - } - return true; -} +/* + +py/pyext - python external object for PD and MaxMSP + +Copyright (c)2002-2007 Thomas Grill (gr@grrrr.org) +For information on usage and redistribution, and for a DISCLAIMER OF ALL +WARRANTIES, see the file, "license.txt," in this distribution. + +*/ + +#include "pybase.h" +#include + +#if FLEXT_OS == FLEXT_OS_WIN +#include +#elif FLEXT_OS == FLEXT_OS_MAC +#include +#endif + +static PyMethodDef StdOut_Methods[] = +{ + { "write", pybase::StdOut_Write, 1 }, + { NULL, NULL, } +}; + +static PyObject *gcollect = NULL; + +#ifdef FLEXT_THREADS + +class ThrCmp +{ +public: + inline bool operator()(const flext::thrid_t &a,const flext::thrid_t &b) const + { + if(sizeof(a) == sizeof(size_t)) + return *(size_t *)&a < *(size_t *)&b; + else + return memcmp(&a,&b,sizeof(a)) < 0; + } +}; + + +int pybase::lockcount = 0; + +#ifndef PY_USE_GIL +static PyInterpreterState *pymain = NULL; + +typedef std::map PyThrMap; + +static PyThrMap pythrmap; +ThrState pybase::pythrsys = NULL; + +ThrState pybase::FindThreadState() +{ + flext::thrid_t id = flext::GetThreadId(); + PyThrMap::iterator it = pythrmap.find(id); + if(it == pythrmap.end()) { + // Make new thread state + ThrState st = PyThreadState_New(pymain); + pythrmap[id] = st; + return st; + } + else + return it->second; +} + +void pybase::FreeThreadState() +{ + flext::thrid_t id = flext::GetThreadId(); + PyThrMap::iterator it = pythrmap.find(id); + if(it != pythrmap.end()) { + // clear out any cruft from thread state object + PyThreadState_Clear(it->second); + // delete my thread state object + PyThreadState_Delete(it->second); + // delete from map + pythrmap.erase(it); + } +} +#endif // PY_USE_GIL + +PyFifo pybase::qufifo; +flext::ThrCond pybase::qucond; +#endif + + +PyObject *pybase::module_obj = NULL; +PyObject *pybase::module_dict = NULL; + +PyObject *pybase::builtins_obj = NULL; +PyObject *pybase::builtins_dict = NULL; + +const t_symbol *pybase::sym_fint = NULL; +const t_symbol *pybase::sym_response = NULL; + +// ----------------------------------------------------------------------------------------------------------- + + +void initsymbol(); +void initsamplebuffer(); +void initbundle(); + +void pybase::lib_setup() +{ + post(""); + post("------------------------------------------------"); + post("py/pyext %s - python script objects",PY__VERSION); + post("(C)2002-2007 Thomas Grill - http://grrrr.org/ext"); + post(""); + post("using Python %s",Py_GetVersion()); + +#ifdef FLEXT_DEBUG + post(""); + post("DEBUG version compiled on %s %s",__DATE__,__TIME__); +#endif + + // ------------------------------------------------------------- + + sym_response = flext::MakeSymbol("response"); + +#if FLEXT_SYS == FLEXT_SYS_PD + sym_fint = sym_float; +#else + sym_fint = sym_int; +#endif + + // ------------------------------------------------------------- + + Py_Initialize(); + +#ifdef FLEXT_DEBUG + Py_DebugFlag = 1; +// Py_VerboseFlag = 1; +#else + Py_OptimizeFlag = 1; +#endif + +#ifdef FLEXT_THREADS + // enable thread support and acquire the global thread lock + PyEval_InitThreads(); + +#ifndef PY_USE_GIL + // get thread state + pythrsys = PyThreadState_Get(); + // get main interpreter state + pymain = pythrsys->interp; + + // add thread state of main thread to map + pythrmap[GetThreadId()] = pythrsys; +#endif // PY_USE_GIL + +#endif + + // sys.argv must be set to empty tuple + char *nothing = ""; + PySys_SetArgv(0,¬hing); + + // register/initialize pyext module only once! + module_obj = Py_InitModule(PYEXT_MODULE, func_tbl); + module_dict = PyModule_GetDict(module_obj); // borrowed reference + + PyModule_AddStringConstant(module_obj,"__doc__",(char *)py_doc); + + // redirect stdout + PyObject* py_out; + py_out = Py_InitModule("stdout", StdOut_Methods); + PySys_SetObject("stdout", py_out); + py_out = Py_InitModule("stderr", StdOut_Methods); + PySys_SetObject("stderr", py_out); + + // get garbage collector function + PyObject *gcobj = PyImport_ImportModule("gc"); + if(gcobj) { + gcollect = PyObject_GetAttrString(gcobj,"collect"); + Py_DECREF(gcobj); + } + + builtins_obj = PyImport_ImportModule("__builtin__"); + builtins_dict = PyModule_GetDict(builtins_obj); // borrowed reference + + // add symbol type + initsymbol(); + PyModule_AddObject(module_obj,"Symbol",(PyObject *)&pySymbol_Type); + + // pre-defined symbols + PyModule_AddObject(module_obj,"_s_",(PyObject *)pySymbol__); + PyModule_AddObject(module_obj,"_s_bang",(PyObject *)pySymbol_bang); + PyModule_AddObject(module_obj,"_s_list",(PyObject *)pySymbol_list); + PyModule_AddObject(module_obj,"_s_symbol",(PyObject *)pySymbol_symbol); + PyModule_AddObject(module_obj,"_s_float",(PyObject *)pySymbol_float); + PyModule_AddObject(module_obj,"_s_int",(PyObject *)pySymbol_int); + + // add samplebuffer type + initsamplebuffer(); + PyModule_AddObject(module_obj,"Buffer",(PyObject *)&pySamplebuffer_Type); + + // add message bundle type + initbundle(); + PyModule_AddObject(module_obj,"Bundle",(PyObject *)&pyBundle_Type); + + // ------------------------------------------------------------- + + FLEXT_SETUP(pyobj); + FLEXT_SETUP(pymeth); + FLEXT_SETUP(pyext); +#ifndef PY_NODSP + FLEXT_DSP_SETUP(pydsp); +#endif + +#ifdef FLEXT_THREADS + // release global lock + PyEval_ReleaseLock(); + + // launch thread worker + LaunchThread(quworker,NULL); + + // launch python worker + LaunchThread(pyworker,NULL); +#endif + + post("------------------------------------------------"); + post(""); +} + +FLEXT_LIB_SETUP(py,pybase::lib_setup) + + +// ----------------------------------------------------------------------------------------------------------- + + +pybase::pybase() + : module(NULL) + , dict(NULL) +#ifdef FLEXT_THREADS + , thrcount(0) + , shouldexit(false),stoptick(0) +#endif + , detach(0) + , pymsg(false) +{ + ThrState state = PyLock(); + Py_INCREF(module_obj); + PyUnlock(state); +} + +pybase::~pybase() +{ + ThrState state = PyLock(); + Py_XDECREF(module_obj); + PyUnlock(state); +} + +void pybase::Exit() +{ +#ifdef FLEXT_THREADS + erasethreads(); + + shouldexit = true; + qucond.Signal(); + + if(thrcount) { + // Wait for a certain time + for(int i = 0; i < (PY_STOP_WAIT/PY_STOP_TICK) && thrcount; ++i) + Sleep(PY_STOP_TICK*0.001f); + if(thrcount) { + // Wait forever + post("py/pyext - Waiting for thread termination!"); + while(thrcount) Sleep(PY_STOP_TICK*0.001f); + post("py/pyext - Okay, all threads have terminated"); + } + } +#endif +} + +void pybase::GetDir(PyObject *obj,AtomList &lst) +{ + if(obj) { + ThrState state = PyLock(); + + PyObject *pvar = PyObject_Dir(obj); + if(!pvar) + PyErr_Print(); // no method found + else { + const t_symbol *sym = GetPyArgs(lst,pvar); + if(!sym) + post("py/pyext - Argument list could not be created"); + else + FLEXT_ASSERT(sym == sym_list); + Py_DECREF(pvar); + } + + PyUnlock(state); + } +} + +void pybase::m__dir(PyObject *obj) +{ + AtomList lst; + GetDir(obj,lst); + // dump dir to attribute outlet + DumpOut(NULL,lst.Count(),lst.Atoms()); +} + +void pybase::m__doc(PyObject *obj) +{ + if(obj) { + ThrState state = PyLock(); + + PyObject *docf = PyDict_GetItemString(obj,"__doc__"); // borrowed!!! + if(docf && PyString_Check(docf)) { + post(""); + const char *s = PyString_AS_STRING(docf); + + // FIX: Python doc strings can easily be larger than 1k characters + // -> split into separate lines + for(;;) { + char buf[1024]; + char *nl = strchr((char *)s,'\n'); // the cast is for Borland C++ + if(!nl) { + // no more newline found + post(s); + break; + } + else { + // copy string before newline to temp buffer and post + unsigned int l = nl-s; + if(l >= sizeof(buf)) l = sizeof buf-1; + strncpy(buf,s,l); // copy all but newline + buf[l] = 0; + post(buf); + s = nl+1; // set after newline + } + } + } + + PyUnlock(state); + } +} + +void pybase::OpenEditor() +{ + if(!module) return; + const char *mname = PyModule_GetFilename(module); + if(!mname) { + PyErr_Clear(); + return; + } + + char fname[1024]; + strcpy(fname,mname); + + // replacing .pyc or .pyo for source file name + char *dt = strrchr(fname,'.'); + if(dt && !strncmp(dt,".py",2)) strcpy(dt,".py"); + + // this should open the editor.... +#if FLEXT_OS == FLEXT_OS_WIN + int err = (int)ShellExecute(NULL,"edit",fname,NULL,NULL,SW_SHOW); + if(err == SE_ERR_NOASSOC) { + // no association found - try notepad + err = (int)ShellExecute(NULL,NULL,"notepad.exe",fname,NULL,SW_SHOW); + } + else if(err == ERROR_FILE_NOT_FOUND || err == SE_ERR_FNF) + post("py/pyext - File not %s found",fname); + else if(err <= 32) + post("py/pyext - Unknown error opening %s",fname); + +#elif FLEXT_OS == FLEXT_OS_MAC + FSRef ref; + OSStatus err = FSPathMakeRef((unsigned char *)fname,&ref,NULL); + if(err) + post("py/pyext - Error interpreting path %s",fname); + else { + FSRef editor; + err = LSGetApplicationForItem(&ref,kLSRolesEditor,&editor,NULL); + if(err) { + // Can't find associated application... try Textedit + err = FSPathMakeRef((unsigned char *)"/Applications/TextEdit.app",&editor,NULL); + if(err) + post("py/pyext - Can't find Textedit application"); + } + + if(!err) { + LSLaunchFSRefSpec lspec; + lspec.appRef = &editor; + lspec.numDocs = 1; + lspec.itemRefs = &ref; + lspec.passThruParams = NULL; + lspec.launchFlags = kLSLaunchDefaults; + lspec.asyncRefCon = NULL; + err = LSOpenFromRefSpec(&lspec,NULL); + if(err) + post("py/pyext - Couldn't launch editor"); + } + } +#else + // thanks to Tim Blechmann + + char *editor = getenv("EDITOR"); + + if(!editor) { // || !strcmp(editor, "/usr/bin/nano") || !strcmp(editor, "/usr/bin/pico") || !strcmp(editor, "/usr/bin/vi")) { + // no environment variable or console text editor found ... use idle instead (should have come with Python) + editor = "idle"; + } + + pid_t child = fork(); + if(!child) { + char cmd[80]; + strcpy(cmd,editor); + strcat(cmd," "); + strcat(cmd,fname); + execl("/bin/sh", "sh", "-c", cmd, (char *) NULL); + } +#endif +} + +void pybase::SetArgs() +{ + // script arguments + int argc = args.Count(); + const t_atom *argv = args.Atoms(); + char **sargv = new char *[argc+1]; + for(int i = 0; i <= argc; ++i) { + sargv[i] = new char[256]; + if(!i) + strcpy(sargv[i],"py/pyext"); + else + GetAString(argv[i-1],sargv[i],255); + } + + // the arguments to the module are only recognized once! (at first use in a patcher) + PySys_SetArgv(argc+1,sargv); + + for(int j = 0; j <= argc; ++j) delete[] sargv[j]; + delete[] sargv; +} + +static bool getmodulesub(const char *mod,char *dir,int len,char *ext) +{ +#if FLEXT_SYS == FLEXT_SYS_PD + char *name; + int fd = open_via_path("",mod,ext,dir,&name,len,0); + if(fd > 0) { + FLEXT_ASSERT(name && *name); + close(fd); + } + else { + // look for mod/__init__.py + std::string tmp(mod); + int l = tmp.size(); + tmp += "/__init__"; + fd = open_via_path("",tmp.c_str(),ext,dir,&name,len,0); + if(fd > 0) { + FLEXT_ASSERT(name && *name); + close(fd); + // we must remove the module name from dir + char *t = dir+strlen(dir)-l; + FLEXT_ASSERT(!strcmp(mod,t) && t[-1] == '/'); + t[-1] = 0; + } + else + name = NULL; + } + + // if dir is current working directory... name points to dir + if(dir == name) strcpy(dir,"."); + return name != NULL; +#elif FLEXT_SYS == FLEXT_SYS_MAX + short path; + long type; + char smod[1024]; + strcpy(smod,mod); + strcat(smod,ext); + bool ok = !locatefile_extended(smod,&path,&type,&type,0); + if(ok) + // convert pathname to unix style + path_topathname(path,NULL,smod); + else { + // do path/file.ext combinations work at all under Max? + strcpy(smod,mod); + + short path; + type = 'fold'; + ok = !locatefile_extended(smod,&path,&type,&type,1); + if(ok) { + // convert pathname to unix style (including trailing slash) + path_topathname(path,NULL,smod); + char *end = smod+strlen(smod); + strcpy(end,mod); + strcat(end,"/__init__"); + strcat(end,ext); + + // check if file is really existing: try to open it + FILE *f = fopen(smod,"r"); + if(f) { + *end = 0; // clear module part ... we only need base path + fclose(f); + } + else + ok = false; + } + } + + if(ok) { + // convert path into slash style needed for Python +#if 0 + // Max API function uses Volume:/Path notation + path_nameconform(smod,dir,PATH_STYLE_SLASH,PATH_TYPE_ABSOLUTE); +#else +#if FLEXT_OS == FLEXT_OS_WIN + char *colon = NULL; +#else + char *colon = strchr(smod,':'); +#endif + if(colon) { + *colon = 0; + strcpy(dir,"/Volumes/"); + strcat(dir,smod); + strcat(dir,colon+1); + } + else + strcpy(dir,smod); +#endif + return true; + } + else + // not found + return false; +#else +#pragma message("Not implemented"); + return false; +#endif +} + +static bool getmodulepath(const char *mod,char *dir,int len) +{ + return + getmodulesub(mod,dir,len,".py") || + getmodulesub(mod,dir,len,".pyc") || + getmodulesub(mod,dir,len,".pyo"); +} + +bool pybase::ImportModule(const char *name) +{ + if(name) { + if(modname == name) { + // module with the same name is already loaded + if(module) return true; + } + else + modname = name; + } + else + modname.clear(); + + UnimportModule(); + return ReloadModule(); +} + +void pybase::UnimportModule() +{ + if(module) { + FLEXT_ASSERT(dict && module_obj && module_dict); + + Py_DECREF(module); + + // reference count to module is not 0 here, altough probably the last instance was unloaded + // Python retains one reference to the module all the time + // we don't care + + module = NULL; + dict = NULL; + } +} + +bool pybase::ReloadModule() +{ + SetArgs(); + PyObject *newmod; + + if(modname.length()) { + + if(module) + newmod = PyImport_ReloadModule(module); + else { + // search in module path (TODO: check before if module is already present to avoid costly searching) + char dir[1024]; + if(!getmodulepath(modname.c_str(),dir,sizeof(dir))) + PyErr_SetString(PyExc_ImportError,"Module not found in path"); + else + AddToPath(dir); + + // module could also be loaded ok, even if it's not in the path (e.g. for internal stuff) + newmod = PyImport_ImportModule((char *)modname.c_str()); + } + } + else { + // if no module name given, take py module + newmod = module_obj; + Py_INCREF(newmod); + } + + if(!newmod) { + // unload faulty module + UnimportModule(); + return false; + } + else { + Py_XDECREF(module); + module = newmod; + dict = PyModule_GetDict(module); // borrowed + return true; + } +} + +void pybase::AddToPath(const char *dir) +{ + if(dir && *dir) { + PyObject *pobj = PySys_GetObject("path"); + if(pobj && PyList_Check(pobj)) { + PyObject *ps = PyString_FromString(dir); + if(!PySequence_Contains(pobj,ps)) + PyList_Append(pobj,ps); // makes new reference + Py_DECREF(ps); + } + PySys_SetObject("path",pobj); // steals reference to pobj + } +} + +void pybase::AddCurrentPath(flext_base *o) +{ + char dir[1024]; + + // add dir of current patch to path + o->GetCanvasDir(dir,sizeof(dir)); + if(*dir) AddToPath(dir); + + // add current dir to path +#if FLEXT_SYS == FLEXT_SYS_PD + AddToPath(GetString(canvas_getcurrentdir())); +#elif FLEXT_SYS == FLEXT_SYS_MAX + short path = path_getdefault(); + path_topathname(path,NULL,dir); + AddToPath(dir); +#endif +} + +bool pybase::OutObject(flext_base *ext,int o,PyObject *obj) +{ + flext::AtomListStatic<16> lst; + const t_symbol *sym = pymsg?GetPyAtom(lst,obj):GetPyArgs(lst,obj); + if(sym) { + // call to outlet _outside_ the Mutex lock! + // otherwise (if not detached) deadlock will occur + ext->ToOutAnything(o,sym,lst.Count(),lst.Atoms()); + return true; + } + else + return false; +} + +void pybase::Reload() +{ + ThrState state = PyLock(); + + PyObject *reg = GetRegistry(REGNAME); + + if(reg) { + PyObject *key; + int pos = 0; + while(PyDict_Next(reg,&pos,&key,NULL)) { + pybase *th = (pybase *)PyLong_AsLong(key); + FLEXT_ASSERT(th); + th->Unload(); + } + + UnloadModule(); + } + + bool ok = ReloadModule(); + + if(ok) { + LoadModule(); + + if(reg) { + SetRegistry(REGNAME,reg); + + PyObject *key; + int pos = 0; + while(PyDict_Next(reg,&pos,&key,NULL)) { + pybase *th = (pybase *)PyLong_AsLong(key); + FLEXT_ASSERT(th); + th->Load(); + } + } + else + Load(); + } + + Report(); + PyUnlock(state); +} + +static PyObject *output = NULL; + +// post to the console +PyObject* pybase::StdOut_Write(PyObject* self, PyObject* args) +{ + // should always be a tuple + FLEXT_ASSERT(PyTuple_Check(args)); + + const int sz = PyTuple_GET_SIZE(args); + + for(int i = 0; i < sz; ++i) { + PyObject *val = PyTuple_GET_ITEM(args,i); // borrowed reference + PyObject *str = PyObject_Str(val); // new reference + char *cstr = PyString_AS_STRING(str); + char *lf = strchr(cstr,'\n'); + + // line feed in string + if(!lf) { + // no -> just append + if(output) + PyString_ConcatAndDel(&output,str); // str is decrefd + else + output = str; // take str reference + } + else { + // yes -> append up to line feed, reset output buffer to string remainder + PyObject *part = PyString_FromStringAndSize(cstr,lf-cstr); // new reference + if(output) + PyString_ConcatAndDel(&output,part); // str is decrefd + else + output = part; // take str reference + + // output concatenated string + post(PyString_AS_STRING(output)); + + Py_DECREF(output); + output = PyString_FromString(lf+1); // new reference + } + } + + Py_INCREF(Py_None); + return Py_None; +} + + +class work_data +{ +public: + work_data(PyObject *f,PyObject *a): fun(f),args(a) {} + ~work_data() { Py_DECREF(fun); Py_DECREF(args); } + + PyObject *fun,*args; +}; + +bool pybase::gencall(PyObject *pmeth,PyObject *pargs) +{ + bool ret = false; + + // Now call method + switch(detach) { + case 0: + ret = docall(pmeth,pargs); + Py_DECREF(pargs); + Py_DECREF(pmeth); + break; +#ifdef FLEXT_THREADS + case 1: + // put call into queue + ret = qucall(pmeth,pargs); + break; + case 2: + // each call a new thread + if(!shouldexit) { + thr_params *p = new thr_params; + p->cl = (flext_base *)this; + p->var->_ext = new work_data(pmeth,pargs); + ret = LaunchThread(thrworker,p); + if(!ret) post("py/pyext - Failed to launch thread!"); + } + break; +#endif + default: + post("py/pyext - Unknown detach mode"); + } + return ret; +} + +void pybase::exchandle() +{ +#if 0 + // want to use that, but exception keeps a reference to the object + // might be a Python bug! + PyErr_Print(); +#else + // must use that instead... clear the exception + PyObject *type,*value,*traceback; + PyErr_Fetch(&type,&value,&traceback); + PyErr_NormalizeException(&type,&value,&traceback); + PyErr_Display(type,value,traceback); + + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); +#endif +} + +#ifdef FLEXT_THREADS +void pybase::thrworker(thr_params *p) +{ + FLEXT_ASSERT(p); + pybase *th = (pybase *)p->cl; + work_data *w = (work_data *)p->var->_ext; + + ++th->thrcount; // \todo this should be atomic + ThrState state = PyLock(); + + // call worker + th->docall(w->fun,w->args); + delete w; + + PyUnlock(state); + --th->thrcount; // \todo this should be atomic +} + +/*! This thread function basically keeps alive the Python interpreter in the background + It's good for threads that have been started from scripted functions +*/ +void pybase::pyworker(thr_params *) +{ + ThrState state = PyLock(); + + PyObject *timemod = PyImport_ImportModule("time"); + PyObject *sleep = PyObject_GetAttrString(timemod,"sleep"); + PyObject *args = PyTuple_New(1); + PyTuple_SET_ITEM(args,0,PyFloat_FromDouble(1000000)); + + for(;;) { + PyObject *res = PyObject_CallObject(sleep,args); + Py_DECREF(res); + } + + PyUnlock(state); +} + +void pybase::quworker(thr_params *) +{ + FifoEl *el; + ThrState my = FindThreadState(),state; + + for(;;) { + while(el = qufifo.Get()) { + ++el->th->thrcount; // \todo this should be atomic + state = PyLock(my); + el->th->docall(el->fun,el->args); + Py_XDECREF(el->fun); + Py_XDECREF(el->args); + PyUnlock(state); + --el->th->thrcount; // \todo this should be atomic + qufifo.Free(el); + } + qucond.Wait(); + } + + // we never end +#if 0 + state = PyLock(my); + // unref remaining Python objects + while(el = qufifo.Get()) { + Py_XDECREF(el->fun); + Py_XDECREF(el->args); + qufifo.Free(el); + } + PyUnlock(state); +#endif +} + +void pybase::erasethreads() +{ + PyFifo tmp; + FifoEl *el; + while(el = qufifo.Get()) { + if(el->th == this) { + Py_XDECREF(el->fun); + Py_XDECREF(el->args); + qufifo.Free(el); + } + else + tmp.Put(el); + } + // Push back + while(el = tmp.Get()) qufifo.Put(el); +} +#endif + +bool pybase::collect() +{ + if(gcollect) { + PyObject *ret = PyObject_CallObject(gcollect,NULL); + if(ret) { +#ifdef FLEXT_DEBUG + int refs = PyInt_AsLong(ret); + if(refs) post("py/pyext - Garbage collector reports %i unreachable objects",refs); +#endif + Py_DECREF(ret); + return false; + } + } + return true; +} -- cgit v1.2.1