1
2
3
4 """
5 This file is part of the web2py Web Framework
6 Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7 License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8
9 Contains:
10
11 - wsgibase: the gluon wsgi application
12
13 """
14
15 import gc
16 import cgi
17 import cStringIO
18 import Cookie
19 import os
20 import re
21 import copy
22 import sys
23 import time
24 import thread
25 import datetime
26 import signal
27 import socket
28 import tempfile
29 import random
30 import string
31 from fileutils import abspath, write_file, parse_version
32 from settings import global_settings
33 from admin import add_path_first, create_missing_folders, create_missing_app_folders
34 from globals import current
35
36 from custom_import import custom_import_install
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54 if not hasattr(os, 'mkdir'):
55 global_settings.db_sessions = True
56 if global_settings.db_sessions is not True:
57 global_settings.db_sessions = set()
58 global_settings.gluon_parent = os.environ.get('web2py_path', os.getcwd())
59 global_settings.applications_parent = global_settings.gluon_parent
60 web2py_path = global_settings.applications_parent
61 global_settings.app_folders = set()
62 global_settings.debugging = False
63
64 custom_import_install(web2py_path)
65
66 create_missing_folders()
67
68
69 import logging
70 import logging.config
71 logpath = abspath("logging.conf")
72 if os.path.exists(logpath):
73 logging.config.fileConfig(abspath("logging.conf"))
74 else:
75 logging.basicConfig()
76 logger = logging.getLogger("web2py")
77
78 from restricted import RestrictedError
79 from http import HTTP, redirect
80 from globals import Request, Response, Session
81 from compileapp import build_environment, run_models_in, \
82 run_controller_in, run_view_in
83 from fileutils import copystream
84 from contenttype import contenttype
85 from dal import BaseAdapter
86 from settings import global_settings
87 from validators import CRYPT
88 from cache import Cache
89 from html import URL as Url
90 import newcron
91 import rewrite
92
93 __all__ = ['wsgibase', 'save_password', 'appfactory', 'HttpServer']
94
95 requests = 0
96
97
98
99
100
101 regex_client = re.compile('[\w\-:]+(\.[\w\-]+)*\.?')
102
103 version_info = open(abspath('VERSION', gluon=True), 'r')
104 web2py_version = parse_version(version_info.read().strip())
105 version_info.close()
106 global_settings.web2py_version = web2py_version
107
108 try:
109 import rocket
110 except:
111 if not global_settings.web2py_runtime_gae:
112 logger.warn('unable to import Rocket')
113
114 rewrite.load()
115
117 """
118 guess the client address from the environment variables
119
120 first tries 'http_x_forwarded_for', secondly 'remote_addr'
121 if all fails assume '127.0.0.1' (running locally)
122 """
123 g = regex_client.search(env.get('http_x_forwarded_for', ''))
124 if g:
125 return g.group()
126 g = regex_client.search(env.get('remote_addr', ''))
127 if g:
128 return g.group()
129 return '127.0.0.1'
130
132 """
133 copies request.env.wsgi_input into request.body
134 and stores progress upload status in cache.ram
135 X-Progress-ID:length and X-Progress-ID:uploaded
136 """
137 if not request.env.content_length:
138 return cStringIO.StringIO()
139 source = request.env.wsgi_input
140 size = int(request.env.content_length)
141 dest = tempfile.TemporaryFile()
142 if not 'X-Progress-ID' in request.vars:
143 copystream(source, dest, size, chunk_size)
144 return dest
145 cache_key = 'X-Progress-ID:'+request.vars['X-Progress-ID']
146 cache = Cache(request)
147 cache.ram(cache_key+':length', lambda: size, 0)
148 cache.ram(cache_key+':uploaded', lambda: 0, 0)
149 while size > 0:
150 if size < chunk_size:
151 data = source.read(size)
152 cache.ram.increment(cache_key+':uploaded', size)
153 else:
154 data = source.read(chunk_size)
155 cache.ram.increment(cache_key+':uploaded', chunk_size)
156 length = len(data)
157 if length > size:
158 (data, length) = (data[:size], size)
159 size -= length
160 if length == 0:
161 break
162 dest.write(data)
163 if length < chunk_size:
164 break
165 dest.seek(0)
166 cache.ram(cache_key+':length', None)
167 cache.ram(cache_key+':uploaded', None)
168 return dest
169
170
172 """
173 this function is used to generate a dynamic page.
174 It first runs all models, then runs the function in the controller,
175 and then tries to render the output using a view/template.
176 this function must run from the [application] folder.
177 A typical example would be the call to the url
178 /[application]/[controller]/[function] that would result in a call
179 to [function]() in applications/[application]/[controller].py
180 rendered by applications/[application]/views/[controller]/[function].html
181 """
182
183
184
185
186
187 environment = build_environment(request, response, session)
188
189
190
191 response.view = '%s/%s.%s' % (request.controller,
192 request.function,
193 request.extension)
194
195
196
197
198
199
200 run_models_in(environment)
201 response._view_environment = copy.copy(environment)
202 page = run_controller_in(request.controller, request.function, environment)
203 if isinstance(page, dict):
204 response._vars = page
205 for key in page:
206 response._view_environment[key] = page[key]
207 run_view_in(response._view_environment)
208 page = response.body.getvalue()
209
210 global requests
211 requests = ('requests' in globals()) and (requests+1) % 100 or 0
212 if not requests: gc.collect()
213
214 raise HTTP(response.status, page, **response.headers)
215
216
218 """
219 in controller you can use::
220
221 - request.wsgi.environ
222 - request.wsgi.start_response
223
224 to call third party WSGI applications
225 """
226 response.status = str(status).split(' ',1)[0]
227 response.headers = dict(headers)
228 return lambda *args, **kargs: response.write(escape=False,*args,**kargs)
229
230
232 """
233 In you controller use::
234
235 @request.wsgi.middleware(middleware1, middleware2, ...)
236
237 to decorate actions with WSGI middleware. actions must return strings.
238 uses a simulated environment so it may have weird behavior in some cases
239 """
240 def middleware(f):
241 def app(environ, start_response):
242 data = f()
243 start_response(response.status,response.headers.items())
244 if isinstance(data,list):
245 return data
246 return [data]
247 for item in middleware_apps:
248 app=item(app)
249 def caller(app):
250 return app(request.wsgi.environ,request.wsgi.start_response)
251 return lambda caller=caller, app=app: caller(app)
252 return middleware
253
255 new_environ = copy.copy(environ)
256 new_environ['wsgi.input'] = request.body
257 new_environ['wsgi.version'] = 1
258 return new_environ
259
260 -def parse_get_post_vars(request, environ):
261
262
263 dget = cgi.parse_qsl(request.env.query_string or '', keep_blank_values=1)
264 for (key, value) in dget:
265 if key in request.get_vars:
266 if isinstance(request.get_vars[key], list):
267 request.get_vars[key] += [value]
268 else:
269 request.get_vars[key] = [request.get_vars[key]] + [value]
270 else:
271 request.get_vars[key] = value
272 request.vars[key] = request.get_vars[key]
273
274
275 request.body = copystream_progress(request)
276 if (request.body and request.env.request_method in ('POST', 'PUT', 'BOTH')):
277 dpost = cgi.FieldStorage(fp=request.body,environ=environ,keep_blank_values=1)
278
279 is_multipart = dpost.type[:10] == 'multipart/'
280 request.body.seek(0)
281 isle25 = sys.version_info[1] <= 5
282
283 def listify(a):
284 return (not isinstance(a,list) and [a]) or a
285 try:
286 keys = sorted(dpost)
287 except TypeError:
288 keys = []
289 for key in keys:
290 dpk = dpost[key]
291
292 if isinstance(dpk, list):
293 if not dpk[0].filename:
294 value = [x.value for x in dpk]
295 else:
296 value = [x for x in dpk]
297 elif not dpk.filename:
298 value = dpk.value
299 else:
300 value = dpk
301 pvalue = listify(value)
302 if key in request.vars:
303 gvalue = listify(request.vars[key])
304 if isle25:
305 value = pvalue + gvalue
306 elif is_multipart:
307 pvalue = pvalue[len(gvalue):]
308 else:
309 pvalue = pvalue[:-len(gvalue)]
310 request.vars[key] = value
311 if len(pvalue):
312 request.post_vars[key] = (len(pvalue)>1 and pvalue) or pvalue[0]
313
314
316 """
317 this is the gluon wsgi application. the first function called when a page
318 is requested (static or dynamic). it can be called by paste.httpserver
319 or by apache mod_wsgi.
320
321 - fills request with info
322 - the environment variables, replacing '.' with '_'
323 - adds web2py path and version info
324 - compensates for fcgi missing path_info and query_string
325 - validates the path in url
326
327 The url path must be either:
328
329 1. for static pages:
330
331 - /<application>/static/<file>
332
333 2. for dynamic pages:
334
335 - /<application>[/<controller>[/<function>[/<sub>]]][.<extension>]
336 - (sub may go several levels deep, currently 3 levels are supported:
337 sub1/sub2/sub3)
338
339 The naming conventions are:
340
341 - application, controller, function and extension may only contain
342 [a-zA-Z0-9_]
343 - file and sub may also contain '-', '=', '.' and '/'
344 """
345
346 current.__dict__.clear()
347 request = Request()
348 response = Response()
349 session = Session()
350 request.env.web2py_path = global_settings.applications_parent
351 request.env.web2py_version = web2py_version
352 request.env.update(global_settings)
353 static_file = False
354 try:
355 try:
356 try:
357
358
359
360
361
362
363
364
365
366 if not environ.get('PATH_INFO',None) and \
367 environ.get('REQUEST_URI',None):
368
369 items = environ['REQUEST_URI'].split('?')
370 environ['PATH_INFO'] = items[0]
371 if len(items) > 1:
372 environ['QUERY_STRING'] = items[1]
373 else:
374 environ['QUERY_STRING'] = ''
375 if not environ.get('HTTP_HOST',None):
376 environ['HTTP_HOST'] = '%s:%s' % (environ.get('SERVER_NAME'),
377 environ.get('SERVER_PORT'))
378
379 (static_file, environ) = rewrite.url_in(request, environ)
380 if static_file:
381 if environ.get('QUERY_STRING', '')[:10] == 'attachment':
382 response.headers['Content-Disposition'] = 'attachment'
383 response.stream(static_file, request=request)
384
385
386
387
388
389 http_host = request.env.http_host.split(':',1)[0]
390
391 local_hosts = [http_host,'::1','127.0.0.1','::ffff:127.0.0.1']
392 if not global_settings.web2py_runtime_gae:
393 local_hosts += [socket.gethostname(),
394 socket.gethostbyname(http_host)]
395 request.client = get_client(request.env)
396 request.folder = abspath('applications',
397 request.application) + os.sep
398 x_req_with = str(request.env.http_x_requested_with).lower()
399 request.ajax = x_req_with == 'xmlhttprequest'
400 request.cid = request.env.http_web2py_component_element
401 request.is_local = request.env.remote_addr in local_hosts
402 request.is_https = request.env.wsgi_url_scheme \
403 in ['https', 'HTTPS'] or request.env.https == 'on'
404
405
406
407
408
409 response.uuid = request.compute_uuid()
410
411
412
413
414
415 if not os.path.exists(request.folder):
416 if request.application == \
417 rewrite.thread.routes.default_application \
418 and request.application != 'welcome':
419 request.application = 'welcome'
420 redirect(Url(r=request))
421 elif rewrite.thread.routes.error_handler:
422 _handler = rewrite.thread.routes.error_handler
423 redirect(Url(_handler['application'],
424 _handler['controller'],
425 _handler['function'],
426 args=request.application))
427 else:
428 raise HTTP(404, rewrite.thread.routes.error_message \
429 % 'invalid request',
430 web2py_error='invalid application')
431 elif not request.is_local and \
432 os.path.exists(os.path.join(request.folder,'DISABLED')):
433 raise HTTP(200, "<html><body><h1>Down for maintenance</h1></body></html>")
434 request.url = Url(r=request, args=request.args,
435 extension=request.raw_extension)
436
437
438
439
440
441 create_missing_app_folders(request)
442
443
444
445
446
447 parse_get_post_vars(request, environ)
448
449
450
451
452
453 request.wsgi.environ = environ_aux(environ,request)
454 request.wsgi.start_response = \
455 lambda status='200', headers=[], \
456 exec_info=None, response=response: \
457 start_response_aux(status, headers, exec_info, response)
458 request.wsgi.middleware = \
459 lambda *a: middleware_aux(request,response,*a)
460
461
462
463
464
465 if request.env.http_cookie:
466 try:
467 request.cookies.load(request.env.http_cookie)
468 except Cookie.CookieError, e:
469 pass
470
471
472
473
474
475 session.connect(request, response)
476
477
478
479
480
481 response.headers['Content-Type'] = \
482 contenttype('.'+request.extension)
483 response.headers['Cache-Control'] = \
484 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0'
485 response.headers['Expires'] = \
486 time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime())
487 response.headers['Pragma'] = 'no-cache'
488
489
490
491
492
493 if global_settings.debugging and request.application != "admin":
494 import gluon.debug
495
496 gluon.debug.dbg.do_debug(mainpyfile=request.folder)
497
498 serve_controller(request, response, session)
499
500 except HTTP, http_response:
501 if static_file:
502 return http_response.to(responder)
503
504 if request.body:
505 request.body.close()
506
507
508
509
510 session._try_store_in_db(request, response)
511
512
513
514
515
516 if response.do_not_commit is True:
517 BaseAdapter.close_all_instances(None)
518 elif response._custom_commit:
519 response._custom_commit()
520 else:
521 BaseAdapter.close_all_instances('commit')
522
523
524
525
526
527
528 session._try_store_on_disk(request, response)
529
530
531
532
533
534 if request.cid:
535
536 if response.flash and not 'web2py-component-flash' in http_response.headers:
537 http_response.headers['web2py-component-flash'] = \
538 str(response.flash).replace('\n','')
539 if response.js and not 'web2py-component-command' in http_response.headers:
540 http_response.headers['web2py-component-command'] = \
541 response.js.replace('\n','')
542 if session._forget and \
543 response.session_id_name in response.cookies:
544 del response.cookies[response.session_id_name]
545 elif session._secure:
546 response.cookies[response.session_id_name]['secure'] = True
547 if len(response.cookies)>0:
548 http_response.headers['Set-Cookie'] = \
549 [str(cookie)[11:] for cookie in response.cookies.values()]
550 ticket=None
551
552 except RestrictedError, e:
553
554 if request.body:
555 request.body.close()
556
557
558
559
560
561 ticket = e.log(request) or 'unknown'
562 if response._custom_rollback:
563 response._custom_rollback()
564 else:
565 BaseAdapter.close_all_instances('rollback')
566
567 http_response = \
568 HTTP(500, rewrite.thread.routes.error_message_ticket % \
569 dict(ticket=ticket),
570 web2py_error='ticket %s' % ticket)
571
572 except:
573
574 if request.body:
575 request.body.close()
576
577
578
579
580
581 try:
582 if response._custom_rollback:
583 response._custom_rollback()
584 else:
585 BaseAdapter.close_all_instances('rollback')
586 except:
587 pass
588 e = RestrictedError('Framework', '', '', locals())
589 ticket = e.log(request) or 'unrecoverable'
590 http_response = \
591 HTTP(500, rewrite.thread.routes.error_message_ticket \
592 % dict(ticket=ticket),
593 web2py_error='ticket %s' % ticket)
594
595 finally:
596 if response and hasattr(response, 'session_file') \
597 and response.session_file:
598 response.session_file.close()
599
600 session._unlock(response)
601 http_response, new_environ = rewrite.try_rewrite_on_error(
602 http_response, request, environ, ticket)
603 if not http_response:
604 return wsgibase(new_environ,responder)
605 if global_settings.web2py_crontype == 'soft':
606 newcron.softcron(global_settings.applications_parent).start()
607 return http_response.to(responder)
608
609
611 """
612 used by main() to save the password in the parameters_port.py file.
613 """
614
615 password_file = abspath('parameters_%i.py' % port)
616 if password == '<random>':
617
618 chars = string.letters + string.digits
619 password = ''.join([random.choice(chars) for i in range(8)])
620 cpassword = CRYPT()(password)[0]
621 print '******************* IMPORTANT!!! ************************'
622 print 'your admin password is "%s"' % password
623 print '*********************************************************'
624 elif password == '<recycle>':
625
626 if os.path.exists(password_file):
627 return
628 else:
629 password = ''
630 elif password.startswith('<pam_user:'):
631
632 cpassword = password[1:-1]
633 else:
634
635 cpassword = CRYPT()(password)[0]
636 fp = open(password_file, 'w')
637 if password:
638 fp.write('password="%s"\n' % cpassword)
639 else:
640 fp.write('password=None\n')
641 fp.close()
642
643
644 -def appfactory(wsgiapp=wsgibase,
645 logfilename='httpserver.log',
646 profilerfilename='profiler.log'):
647 """
648 generates a wsgi application that does logging and profiling and calls
649 wsgibase
650
651 .. function:: gluon.main.appfactory(
652 [wsgiapp=wsgibase
653 [, logfilename='httpserver.log'
654 [, profilerfilename='profiler.log']]])
655
656 """
657 if profilerfilename and os.path.exists(profilerfilename):
658 os.unlink(profilerfilename)
659 locker = thread.allocate_lock()
660
661 def app_with_logging(environ, responder):
662 """
663 a wsgi app that does logging and profiling and calls wsgibase
664 """
665 status_headers = []
666
667 def responder2(s, h):
668 """
669 wsgi responder app
670 """
671 status_headers.append(s)
672 status_headers.append(h)
673 return responder(s, h)
674
675 time_in = time.time()
676 ret = [0]
677 if not profilerfilename:
678 ret[0] = wsgiapp(environ, responder2)
679 else:
680 import cProfile
681 import pstats
682 logger.warn('profiler is on. this makes web2py slower and serial')
683
684 locker.acquire()
685 cProfile.runctx('ret[0] = wsgiapp(environ, responder2)',
686 globals(), locals(), profilerfilename+'.tmp')
687 stat = pstats.Stats(profilerfilename+'.tmp')
688 stat.stream = cStringIO.StringIO()
689 stat.strip_dirs().sort_stats("time").print_stats(80)
690 profile_out = stat.stream.getvalue()
691 profile_file = open(profilerfilename, 'a')
692 profile_file.write('%s\n%s\n%s\n%s\n\n' % \
693 ('='*60, environ['PATH_INFO'], '='*60, profile_out))
694 profile_file.close()
695 locker.release()
696 try:
697 line = '%s, %s, %s, %s, %s, %s, %f\n' % (
698 environ['REMOTE_ADDR'],
699 datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
700 environ['REQUEST_METHOD'],
701 environ['PATH_INFO'].replace(',', '%2C'),
702 environ['SERVER_PROTOCOL'],
703 (status_headers[0])[:3],
704 time.time() - time_in,
705 )
706 if not logfilename:
707 sys.stdout.write(line)
708 elif isinstance(logfilename, str):
709 write_file(logfilename, line, 'a')
710 else:
711 logfilename.write(line)
712 except:
713 pass
714 return ret[0]
715
716 return app_with_logging
717
718
720 """
721 the web2py web server (Rocket)
722 """
723
724 - def __init__(
725 self,
726 ip='127.0.0.1',
727 port=8000,
728 password='',
729 pid_filename='httpserver.pid',
730 log_filename='httpserver.log',
731 profiler_filename=None,
732 ssl_certificate=None,
733 ssl_private_key=None,
734 ssl_ca_certificate=None,
735 min_threads=None,
736 max_threads=None,
737 server_name=None,
738 request_queue_size=5,
739 timeout=10,
740 socket_timeout = 1,
741 shutdown_timeout=None,
742 path=None,
743 interfaces=None
744 ):
745 """
746 starts the web server.
747 """
748
749 if interfaces:
750
751
752 import types
753 if isinstance(interfaces,types.ListType):
754 for i in interfaces:
755 if not isinstance(i,types.TupleType):
756 raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/"
757 else:
758 raise "Wrong format for rocket interfaces parameter - see http://packages.python.org/rocket/"
759
760 if path:
761
762
763 global web2py_path
764 path = os.path.normpath(path)
765 web2py_path = path
766 global_settings.applications_parent = path
767 os.chdir(path)
768 [add_path_first(p) for p in (path, abspath('site-packages'), "")]
769 custom_import_install(web2py_path)
770 if os.path.exists("logging.conf"):
771 logging.config.fileConfig("logging.conf")
772
773 save_password(password, port)
774 self.pid_filename = pid_filename
775 if not server_name:
776 server_name = socket.gethostname()
777 logger.info('starting web server...')
778 rocket.SERVER_NAME = server_name
779 rocket.SOCKET_TIMEOUT = socket_timeout
780 sock_list = [ip, port]
781 if not ssl_certificate or not ssl_private_key:
782 logger.info('SSL is off')
783 elif not rocket.ssl:
784 logger.warning('Python "ssl" module unavailable. SSL is OFF')
785 elif not os.path.exists(ssl_certificate):
786 logger.warning('unable to open SSL certificate. SSL is OFF')
787 elif not os.path.exists(ssl_private_key):
788 logger.warning('unable to open SSL private key. SSL is OFF')
789 else:
790 sock_list.extend([ssl_private_key, ssl_certificate])
791 if ssl_ca_certificate:
792 sock_list.append(ssl_ca_certificate)
793
794 logger.info('SSL is ON')
795 app_info = {'wsgi_app': appfactory(wsgibase,
796 log_filename,
797 profiler_filename) }
798
799 self.server = rocket.Rocket(interfaces or tuple(sock_list),
800 method='wsgi',
801 app_info=app_info,
802 min_threads=min_threads,
803 max_threads=max_threads,
804 queue_size=int(request_queue_size),
805 timeout=int(timeout),
806 handle_signals=False,
807 )
808
809
811 """
812 start the web server
813 """
814 try:
815 signal.signal(signal.SIGTERM, lambda a, b, s=self: s.stop())
816 signal.signal(signal.SIGINT, lambda a, b, s=self: s.stop())
817 except:
818 pass
819 write_file(self.pid_filename, str(os.getpid()))
820 self.server.start()
821
822 - def stop(self, stoplogging=False):
823 """
824 stop cron and the web server
825 """
826 newcron.stopcron()
827 self.server.stop(stoplogging)
828 try:
829 os.unlink(self.pid_filename)
830 except:
831 pass
832