Package web2py :: Package gluon :: Module compileapp
[hide private]
[frames] | no frames]

Source Code for Module web2py.gluon.compileapp

  1  #!/usr/bin/env python 
  2  # -*- coding: utf-8 -*- 
  3   
  4  """ 
  5  This file is part of the web2py Web Framework 
  6  Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> 
  7  License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) 
  8   
  9  Functions required to execute app components 
 10  ============================================ 
 11   
 12  FOR INTERNAL USE ONLY 
 13  """ 
 14   
 15  import re 
 16  import sys 
 17  import fnmatch 
 18  import os 
 19  import copy 
 20  import random 
 21  import __builtin__ 
 22  from storage import Storage, List 
 23  from template import parse_template 
 24  from restricted import restricted, compile2 
 25  from fileutils import mktree, listdir, read_file, write_file 
 26  from myregex import regex_expose 
 27  from languages import translator 
 28  from dal import BaseAdapter, SQLDB, SQLField, DAL, Field 
 29  from sqlhtml import SQLFORM, SQLTABLE 
 30  from cache import Cache 
 31  from globals import current, Response 
 32  import settings 
 33  from cfs import getcfs 
 34  import html 
 35  import validators 
 36  from http import HTTP, redirect 
 37  import marshal 
 38  import shutil 
 39  import imp 
 40  import logging 
 41  logger = logging.getLogger("web2py") 
 42  import rewrite 
 43  import platform 
 44   
 45  try: 
 46      import py_compile 
 47  except: 
 48      logger.warning('unable to import py_compile') 
 49   
 50  is_pypy = hasattr(platform,'python_implementation') and \ 
 51      platform.python_implementation() == 'PyPy' 
 52  settings.global_settings.is_pypy = is_pypy 
 53  is_gae = settings.global_settings.web2py_runtime_gae 
 54  is_jython = settings.global_settings.is_jython = 'java' in sys.platform.lower() or hasattr(sys, 'JYTHON_JAR') or str(sys.copyright).find('Jython') > 0 
 55   
 56  TEST_CODE = \ 
 57      r""" 
 58  def _TEST(): 
 59      import doctest, sys, cStringIO, types, cgi, gluon.fileutils 
 60      if not gluon.fileutils.check_credentials(request): 
 61          raise HTTP(401, web2py_error='invalid credentials') 
 62      stdout = sys.stdout 
 63      html = '<h2>Testing controller "%s.py" ... done.</h2><br/>\n' \ 
 64          % request.controller 
 65      for key in sorted([key for key in globals() if not key in __symbols__+['_TEST']]): 
 66          eval_key = eval(key) 
 67          if type(eval_key) == types.FunctionType: 
 68              number_doctests = sum([len(ds.examples) for ds in doctest.DocTestFinder().find(eval_key)]) 
 69              if number_doctests>0: 
 70                  sys.stdout = cStringIO.StringIO() 
 71                  name = '%s/controllers/%s.py in %s.__doc__' \ 
 72                      % (request.folder, request.controller, key) 
 73                  doctest.run_docstring_examples(eval_key, 
 74                      globals(), False, name=name) 
 75                  report = sys.stdout.getvalue().strip() 
 76                  if report: 
 77                      pf = 'failed' 
 78                  else: 
 79                      pf = 'passed' 
 80                  html += '<h3 class="%s">Function %s [%s]</h3>\n' \ 
 81                      % (pf, key, pf) 
 82                  if report: 
 83                      html += CODE(report, language='web2py', \ 
 84                          link='/examples/global/vars/').xml() 
 85                  html += '<br/>\n' 
 86              else: 
 87                  html += \ 
 88                      '<h3 class="nodoctests">Function %s [no doctests]</h3><br/>\n' \ 
 89                      % (key) 
 90      response._vars = html 
 91      sys.stdout = stdout 
 92  _TEST() 
 93  """ 
 94   
95 -class mybuiltin(object):
96 """ 97 NOTE could simple use a dict and populate it, 98 NOTE not sure if this changes things though if monkey patching import..... 99 """ 100 #__builtins__
101 - def __getitem__(self, key):
102 try: 103 return getattr(__builtin__, key) 104 except AttributeError: 105 raise KeyError, key
106 - def __setitem__(self, key, value):
107 setattr(self, key, value)
108
109 -def LOAD(c=None, f='index', args=None, vars=None, 110 extension=None, target=None,ajax=False,ajax_trap=False, 111 url=None,user_signature=False, timeout=None, times=1, 112 content='loading...',**attr):
113 """ LOAD a component into the action's document 114 115 Timing options: 116 -times: An integer or string ("infinity"/"continuous") 117 specifies how many times the component is requested 118 -timeout (milliseconds): specifies the time to wait before 119 starting the request or the frequency if times is greater than 120 1 or "infinity". 121 Timing options default to the normal behavior. The component 122 is added on page loading without delay. 123 """ 124 from html import TAG, DIV, URL, SCRIPT, XML 125 if args is None: args = [] 126 vars = Storage(vars or {}) 127 target = target or 'c'+str(random.random())[2:] 128 attr['_id']=target 129 request = current.request 130 if '.' in f: 131 f, extension = f.split('.',1) 132 if url or ajax: 133 url = url or URL(request.application, c, f, r=request, 134 args=args, vars=vars, extension=extension, 135 user_signature=user_signature) 136 # timing options 137 if isinstance(times, basestring): 138 if times.upper() in ("INFINITY", "CONTINUOUS"): 139 times = "Infinity" 140 else: 141 raise TypeError("Unsupported times argument %s" % times) 142 elif isinstance(times, int): 143 if times <= 0: 144 raise ValueError("Times argument must be greater than zero, 'Infinity' or None") 145 else: 146 raise TypeError("Unsupported times argument type %s" % type(times)) 147 if timeout is not None: 148 if not isinstance(timeout, (int, long)): 149 raise ValueError("Timeout argument must be an integer or None") 150 elif timeout <= 0: 151 raise ValueError("Timeout argument must be greater than zero or None") 152 statement = "web2py_component('%s','%s', %s, %s);" \ 153 % (url, target, timeout, times) 154 else: 155 statement = "web2py_component('%s','%s');" % (url, target) 156 script = SCRIPT(statement, _type="text/javascript") 157 if not content is None: 158 return TAG[''](script, DIV(content,**attr)) 159 else: 160 return TAG[''](script) 161 162 else: 163 if not isinstance(args,(list,tuple)): 164 args = [args] 165 c = c or request.controller 166 other_request = Storage() 167 for key, value in request.items(): 168 other_request[key] = value 169 other_request['env'] = Storage() 170 for key, value in request.env.items(): 171 other_request.env['key'] = value 172 other_request.controller = c 173 other_request.function = f 174 other_request.extension = extension or request.extension 175 other_request.args = List(args) 176 other_request.vars = vars 177 other_request.get_vars = vars 178 other_request.post_vars = Storage() 179 other_response = Response() 180 other_request.env.path_info = '/' + \ 181 '/'.join([request.application,c,f] + \ 182 map(str, other_request.args)) 183 other_request.env.query_string = \ 184 vars and URL(vars=vars).split('?')[1] or '' 185 other_request.env.http_web2py_component_location = \ 186 request.env.path_info 187 other_request.cid = target 188 other_request.env.http_web2py_component_element = target 189 other_response.view = '%s/%s.%s' % (c,f, other_request.extension) 190 191 other_environment = copy.copy(current.globalenv) ### NASTY 192 193 other_response._view_environment = other_environment 194 other_response.generic_patterns = \ 195 copy.copy(current.response.generic_patterns) 196 other_environment['request'] = other_request 197 other_environment['response'] = other_response 198 199 ## some magic here because current are thread-locals 200 201 original_request, current.request = current.request, other_request 202 original_response, current.response = current.response, other_response 203 page = run_controller_in(c, f, other_environment) 204 if isinstance(page, dict): 205 other_response._vars = page 206 for key in page: 207 other_response._view_environment[key] = page[key] 208 run_view_in(other_response._view_environment) 209 page = other_response.body.getvalue() 210 current.request, current.response = original_request, original_response 211 js = None 212 if ajax_trap: 213 link = URL(request.application, c, f, r=request, 214 args=args, vars=vars, extension=extension, 215 user_signature=user_signature) 216 js = "web2py_trap_form('%s','%s');" % (link, target) 217 script = js and SCRIPT(js,_type="text/javascript") or '' 218 return TAG[''](DIV(XML(page),**attr),script)
219 220 221
222 -class LoadFactory(object):
223 """ 224 Attention: this helper is new and experimental 225 """
226 - def __init__(self,environment):
227 self.environment = environment
228 - def __call__(self, c=None, f='index', args=None, vars=None, 229 extension=None, target=None,ajax=False,ajax_trap=False, 230 url=None,user_signature=False, content='loading...',**attr):
231 if args is None: args = [] 232 vars = Storage(vars or {}) 233 import globals 234 target = target or 'c'+str(random.random())[2:] 235 attr['_id']=target 236 request = self.environment['request'] 237 if '.' in f: 238 f, extension = f.split('.',1) 239 if url or ajax: 240 url = url or html.URL(request.application, c, f, r=request, 241 args=args, vars=vars, extension=extension, 242 user_signature=user_signature) 243 script = html.SCRIPT('web2py_component("%s","%s")' % (url, target), 244 _type="text/javascript") 245 return html.TAG[''](script, html.DIV(content,**attr)) 246 else: 247 if not isinstance(args,(list,tuple)): 248 args = [args] 249 c = c or request.controller 250 251 other_request = Storage() 252 for key, value in request.items(): 253 other_request[key] = value 254 other_request['env'] = Storage() 255 for key, value in request.env.items(): 256 other_request.env['key'] = value 257 other_request.controller = c 258 other_request.function = f 259 other_request.extension = extension or request.extension 260 other_request.args = List(args) 261 other_request.vars = vars 262 other_request.get_vars = vars 263 other_request.post_vars = Storage() 264 other_response = globals.Response() 265 other_request.env.path_info = '/' + \ 266 '/'.join([request.application,c,f] + \ 267 map(str, other_request.args)) 268 other_request.env.query_string = \ 269 vars and html.URL(vars=vars).split('?')[1] or '' 270 other_request.env.http_web2py_component_location = \ 271 request.env.path_info 272 other_request.cid = target 273 other_request.env.http_web2py_component_element = target 274 other_response.view = '%s/%s.%s' % (c,f, other_request.extension) 275 other_environment = copy.copy(self.environment) 276 other_response._view_environment = other_environment 277 other_response.generic_patterns = \ 278 copy.copy(current.response.generic_patterns) 279 other_environment['request'] = other_request 280 other_environment['response'] = other_response 281 282 ## some magic here because current are thread-locals 283 284 original_request, current.request = current.request, other_request 285 original_response, current.response = current.response, other_response 286 page = run_controller_in(c, f, other_environment) 287 if isinstance(page, dict): 288 other_response._vars = page 289 for key in page: 290 other_response._view_environment[key] = page[key] 291 run_view_in(other_response._view_environment) 292 page = other_response.body.getvalue() 293 current.request, current.response = original_request, original_response 294 js = None 295 if ajax_trap: 296 link = html.URL(request.application, c, f, r=request, 297 args=args, vars=vars, extension=extension, 298 user_signature=user_signature) 299 js = "web2py_trap_form('%s','%s');" % (link, target) 300 script = js and html.SCRIPT(js,_type="text/javascript") or '' 301 return html.TAG[''](html.DIV(html.XML(page),**attr),script)
302 303
304 -def local_import_aux(name, reload_force=False, app='welcome'):
305 """ 306 In apps, instead of importing a local module 307 (in applications/app/modules) with:: 308 309 import a.b.c as d 310 311 you should do:: 312 313 d = local_import('a.b.c') 314 315 or (to force a reload): 316 317 d = local_import('a.b.c', reload=True) 318 319 This prevents conflict between applications and un-necessary execs. 320 It can be used to import any module, including regular Python modules. 321 """ 322 items = name.replace('/','.') 323 name = "applications.%s.modules.%s" % (app, items) 324 module = __import__(name) 325 for item in name.split(".")[1:]: 326 module = getattr(module, item) 327 if reload_force: 328 reload(module) 329 return module
330 331 332 """ 333 OLD IMPLEMENTATION: 334 items = name.replace('/','.').split('.') 335 filename, modulepath = items[-1], os.path.join(apath,'modules',*items[:-1]) 336 imp.acquire_lock() 337 try: 338 file=None 339 (file,path,desc) = imp.find_module(filename,[modulepath]+sys.path) 340 if not path in sys.modules or reload: 341 if is_gae: 342 module={} 343 execfile(path,{},module) 344 module=Storage(module) 345 else: 346 module = imp.load_module(path,file,path,desc) 347 sys.modules[path] = module 348 else: 349 module = sys.modules[path] 350 except Exception, e: 351 module = None 352 if file: 353 file.close() 354 imp.release_lock() 355 if not module: 356 raise ImportError, "cannot find module %s in %s" % (filename, modulepath) 357 return module 358 """ 359
360 -def build_environment(request, response, session, store_current=True):
361 """ 362 Build the environment dictionary into which web2py files are executed. 363 """ 364 365 environment = {} 366 for key in html.__all__: 367 environment[key] = getattr(html, key) 368 for key in validators.__all__: 369 environment[key] = getattr(validators, key) 370 if not request.env: 371 request.env = Storage() 372 373 t = environment['T'] = translator(request) 374 c = environment['cache'] = Cache(request) 375 if store_current: 376 current.globalenv = environment 377 current.request = request 378 current.response = response 379 current.session = session 380 current.T = t 381 current.cache = c 382 383 global __builtins__ 384 if is_jython: # jython hack 385 __builtins__ = mybuiltin() 386 elif is_pypy: # apply the same hack to pypy too 387 __builtins__ = mybuiltin() 388 else: 389 __builtins__['__import__'] = __builtin__.__import__ ### WHY? 390 environment['__builtins__'] = __builtins__ 391 environment['HTTP'] = HTTP 392 environment['redirect'] = redirect 393 environment['request'] = request 394 environment['response'] = response 395 environment['session'] = session 396 environment['DAL'] = DAL 397 environment['Field'] = Field 398 environment['SQLDB'] = SQLDB # for backward compatibility 399 environment['SQLField'] = SQLField # for backward compatibility 400 environment['SQLFORM'] = SQLFORM 401 environment['SQLTABLE'] = SQLTABLE 402 environment['LOAD'] = LOAD 403 environment['local_import'] = \ 404 lambda name, reload=False, app=request.application:\ 405 local_import_aux(name,reload,app) 406 BaseAdapter.set_folder(os.path.join(request.folder, 'databases')) 407 response._view_environment = copy.copy(environment) 408 return environment
409 410
411 -def save_pyc(filename):
412 """ 413 Bytecode compiles the file `filename` 414 """ 415 py_compile.compile(filename)
416 417
418 -def read_pyc(filename):
419 """ 420 Read the code inside a bytecode compiled file if the MAGIC number is 421 compatible 422 423 :returns: a code object 424 """ 425 data = read_file(filename, 'rb') 426 if not is_gae and data[:4] != imp.get_magic(): 427 raise SystemError, 'compiled code is incompatible' 428 return marshal.loads(data[8:])
429 430
431 -def compile_views(folder):
432 """ 433 Compiles all the views in the application specified by `folder` 434 """ 435 436 path = os.path.join(folder, 'views') 437 for file in listdir(path, '^[\w/\-]+(\.\w+)+$'): 438 data = parse_template(file, path) 439 filename = ('views/%s.py' % file).replace('/', '_').replace('\\', '_') 440 filename = os.path.join(folder, 'compiled', filename) 441 write_file(filename, data) 442 save_pyc(filename) 443 os.unlink(filename)
444 445
446 -def compile_models(folder):
447 """ 448 Compiles all the models in the application specified by `folder` 449 """ 450 451 path = os.path.join(folder, 'models') 452 for file in listdir(path, '.+\.py$'): 453 data = read_file(os.path.join(path, file)) 454 filename = os.path.join(folder, 'compiled','models',file) 455 mktree(filename) 456 write_file(filename, data) 457 save_pyc(filename) 458 os.unlink(filename)
459 460
461 -def compile_controllers(folder):
462 """ 463 Compiles all the controllers in the application specified by `folder` 464 """ 465 466 path = os.path.join(folder, 'controllers') 467 for file in listdir(path, '.+\.py$'): 468 ### why is this here? save_pyc(os.path.join(path, file)) 469 data = read_file(os.path.join(path,file)) 470 exposed = regex_expose.findall(data) 471 for function in exposed: 472 command = data + "\nresponse._vars=response._caller(%s)\n" % \ 473 function 474 filename = os.path.join(folder, 'compiled', ('controllers/' 475 + file[:-3]).replace('/', '_') 476 + '_' + function + '.py') 477 write_file(filename, command) 478 save_pyc(filename) 479 os.unlink(filename)
480 481
482 -def run_models_in(environment):
483 """ 484 Runs all models (in the app specified by the current folder) 485 It tries pre-compiled models first before compiling them. 486 """ 487 488 folder = environment['request'].folder 489 c = environment['request'].controller 490 f = environment['request'].function 491 cpath = os.path.join(folder, 'compiled') 492 if os.path.exists(cpath): 493 for model in listdir(cpath, '^models_\w+\.pyc$', 0): 494 restricted(read_pyc(model), environment, layer=model) 495 path = os.path.join(cpath, 'models') 496 models = listdir(path, '^\w+\.pyc$',0,sort=False) 497 compiled=True 498 else: 499 path = os.path.join(folder, 'models') 500 models = listdir(path, '^\w+\.py$',0,sort=False) 501 compiled=False 502 paths = (path, os.path.join(path,c), os.path.join(path,c,f)) 503 for model in models: 504 if not os.path.split(model)[0] in paths and c!='appadmin': 505 continue 506 elif compiled: 507 code = read_pyc(model) 508 elif is_gae: 509 code = getcfs(model, model, 510 lambda: compile2(read_file(model), model)) 511 else: 512 code = getcfs(model, model, None) 513 restricted(code, environment, layer=model)
514 515
516 -def run_controller_in(controller, function, environment):
517 """ 518 Runs the controller.function() (for the app specified by 519 the current folder). 520 It tries pre-compiled controller_function.pyc first before compiling it. 521 """ 522 523 # if compiled should run compiled! 524 525 folder = environment['request'].folder 526 path = os.path.join(folder, 'compiled') 527 badc = 'invalid controller (%s/%s)' % (controller, function) 528 badf = 'invalid function (%s/%s)' % (controller, function) 529 if os.path.exists(path): 530 filename = os.path.join(path, 'controllers_%s_%s.pyc' 531 % (controller, function)) 532 if not os.path.exists(filename): 533 raise HTTP(404, 534 rewrite.thread.routes.error_message % badf, 535 web2py_error=badf) 536 restricted(read_pyc(filename), environment, layer=filename) 537 elif function == '_TEST': 538 # TESTING: adjust the path to include site packages 539 from settings import global_settings 540 from admin import abspath, add_path_first 541 paths = (global_settings.gluon_parent, abspath('site-packages', gluon=True), abspath('gluon', gluon=True), '') 542 [add_path_first(path) for path in paths] 543 # TESTING END 544 545 filename = os.path.join(folder, 'controllers/%s.py' 546 % controller) 547 if not os.path.exists(filename): 548 raise HTTP(404, 549 rewrite.thread.routes.error_message % badc, 550 web2py_error=badc) 551 environment['__symbols__'] = environment.keys() 552 code = read_file(filename) 553 code += TEST_CODE 554 restricted(code, environment, layer=filename) 555 else: 556 filename = os.path.join(folder, 'controllers/%s.py' 557 % controller) 558 if not os.path.exists(filename): 559 raise HTTP(404, 560 rewrite.thread.routes.error_message % badc, 561 web2py_error=badc) 562 code = read_file(filename) 563 exposed = regex_expose.findall(code) 564 if not function in exposed: 565 raise HTTP(404, 566 rewrite.thread.routes.error_message % badf, 567 web2py_error=badf) 568 code = "%s\nresponse._vars=response._caller(%s)\n" % (code, function) 569 if is_gae: 570 layer = filename + ':' + function 571 code = getcfs(layer, filename, lambda: compile2(code,layer)) 572 restricted(code, environment, filename) 573 response = environment['response'] 574 vars=response._vars 575 if response.postprocessing: 576 for p in response.postprocessing: 577 vars = p(vars) 578 if isinstance(vars,unicode): 579 vars = vars.encode('utf8') 580 if hasattr(vars,'xml'): 581 vars = vars.xml() 582 return vars
583
584 -def run_view_in(environment):
585 """ 586 Executes the view for the requested action. 587 The view is the one specified in `response.view` or determined by the url 588 or `view/generic.extension` 589 It tries the pre-compiled views_controller_function.pyc before compiling it. 590 """ 591 592 request = environment['request'] 593 response = environment['response'] 594 folder = request.folder 595 path = os.path.join(folder, 'compiled') 596 badv = 'invalid view (%s)' % response.view 597 patterns = response.generic_patterns or [] 598 regex = re.compile('|'.join(fnmatch.translate(r) for r in patterns)) 599 short_action = '%(controller)s/%(function)s.%(extension)s' % request 600 allow_generic = patterns and regex.search(short_action) 601 if not isinstance(response.view, str): 602 ccode = parse_template(response.view, os.path.join(folder, 'views'), 603 context=environment) 604 restricted(ccode, environment, 'file stream') 605 elif os.path.exists(path): 606 x = response.view.replace('/', '_') 607 files = ['views_%s.pyc' % x] 608 if allow_generic: 609 files.append('views_generic.%s.pyc' % request.extension) 610 # for backward compatibility 611 if request.extension == 'html': 612 files.append('views_%s.pyc' % x[:-5]) 613 if allow_generic: 614 files.append('views_generic.pyc') 615 # end backward compatibility code 616 for f in files: 617 filename = os.path.join(path,f) 618 if os.path.exists(filename): 619 code = read_pyc(filename) 620 restricted(code, environment, layer=filename) 621 return 622 raise HTTP(404, 623 rewrite.thread.routes.error_message % badv, 624 web2py_error=badv) 625 else: 626 filename = os.path.join(folder, 'views', response.view) 627 if not os.path.exists(filename) and allow_generic: 628 response.view = 'generic.' + request.extension 629 filename = os.path.join(folder, 'views', response.view) 630 if not os.path.exists(filename): 631 raise HTTP(404, 632 rewrite.thread.routes.error_message % badv, 633 web2py_error=badv) 634 layer = filename 635 if is_gae: 636 ccode = getcfs(layer, filename, 637 lambda: compile2(parse_template(response.view, 638 os.path.join(folder, 'views'), 639 context=environment),layer)) 640 else: 641 ccode = parse_template(response.view, 642 os.path.join(folder, 'views'), 643 context=environment) 644 restricted(ccode, environment, layer)
645
646 -def remove_compiled_application(folder):
647 """ 648 Deletes the folder `compiled` containing the compiled application. 649 """ 650 try: 651 shutil.rmtree(os.path.join(folder, 'compiled')) 652 path = os.path.join(folder, 'controllers') 653 for file in listdir(path,'.*\.pyc$',drop=False): 654 os.unlink(file) 655 except OSError: 656 pass
657 658
659 -def compile_application(folder):
660 """ 661 Compiles all models, views, controller for the application in `folder`. 662 """ 663 remove_compiled_application(folder) 664 os.mkdir(os.path.join(folder, 'compiled')) 665 compile_models(folder) 666 compile_controllers(folder) 667 compile_views(folder)
668 669
670 -def test():
671 """ 672 Example:: 673 674 >>> import traceback, types 675 >>> environment={'x':1} 676 >>> open('a.py', 'w').write('print 1/x') 677 >>> save_pyc('a.py') 678 >>> os.unlink('a.py') 679 >>> if type(read_pyc('a.pyc'))==types.CodeType: print 'code' 680 code 681 >>> exec read_pyc('a.pyc') in environment 682 1 683 """ 684 685 return
686 687 688 if __name__ == '__main__': 689 import doctest 690 doctest.testmod() 691