source: vendor/python/2.5/Lib/httplib.py@ 3225

Last change on this file since 3225 was 3225, checked in by bird, 19 years ago

Python 2.5
File size: 46.0 KB
Line 
1"""HTTP/1.1 client library
2
3<intro stuff goes here>
4<other stuff, too>
5
6HTTPConnection goes through a number of "states", which define when a client
7may legally make another request or fetch the response for a particular
8request. This diagram details these state transitions:
9
10 (null)
11 |
12 | HTTPConnection()
13 v
14 Idle
15 |
16 | putrequest()
17 v
18 Request-started
19 |
20 | ( putheader() )* endheaders()
21 v
22 Request-sent
23 |
24 | response = getresponse()
25 v
26 Unread-response [Response-headers-read]
27 |\____________________
28 | |
29 | response.read() | putrequest()
30 v v
31 Idle Req-started-unread-response
32 ______/|
33 / |
34 response.read() | | ( putheader() )* endheaders()
35 v v
36 Request-started Req-sent-unread-response
37 |
38 | response.read()
39 v
40 Request-sent
41
42This diagram presents the following rules:
43 -- a second request may not be started until {response-headers-read}
44 -- a response [object] cannot be retrieved until {request-sent}
45 -- there is no differentiation between an unread response body and a
46 partially read response body
47
48Note: this enforcement is applied by the HTTPConnection class. The
49 HTTPResponse class does not enforce this state machine, which
50 implies sophisticated clients may accelerate the request/response
51 pipeline. Caution should be taken, though: accelerating the states
52 beyond the above pattern may imply knowledge of the server's
53 connection-close behavior for certain requests. For example, it
54 is impossible to tell whether the server will close the connection
55 UNTIL the response headers have been read; this means that further
56 requests cannot be placed into the pipeline until it is known that
57 the server will NOT be closing the connection.
58
59Logical State __state __response
60------------- ------- ----------
61Idle _CS_IDLE None
62Request-started _CS_REQ_STARTED None
63Request-sent _CS_REQ_SENT None
64Unread-response _CS_IDLE <response_class>
65Req-started-unread-response _CS_REQ_STARTED <response_class>
66Req-sent-unread-response _CS_REQ_SENT <response_class>
67"""
68
69import errno
70import mimetools
71import socket
72from urlparse import urlsplit
73
74try:
75 from cStringIO import StringIO
76except ImportError:
77 from StringIO import StringIO
78
79__all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
80 "HTTPException", "NotConnected", "UnknownProtocol",
81 "UnknownTransferEncoding", "UnimplementedFileMode",
82 "IncompleteRead", "InvalidURL", "ImproperConnectionState",
83 "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
84 "BadStatusLine", "error", "responses"]
85
86HTTP_PORT = 80
87HTTPS_PORT = 443
88
89_UNKNOWN = 'UNKNOWN'
90
91# connection states
92_CS_IDLE = 'Idle'
93_CS_REQ_STARTED = 'Request-started'
94_CS_REQ_SENT = 'Request-sent'
95
96# status codes
97# informational
98CONTINUE = 100
99SWITCHING_PROTOCOLS = 101
100PROCESSING = 102
101
102# successful
103OK = 200
104CREATED = 201
105ACCEPTED = 202
106NON_AUTHORITATIVE_INFORMATION = 203
107NO_CONTENT = 204
108RESET_CONTENT = 205
109PARTIAL_CONTENT = 206
110MULTI_STATUS = 207
111IM_USED = 226
112
113# redirection
114MULTIPLE_CHOICES = 300
115MOVED_PERMANENTLY = 301
116FOUND = 302
117SEE_OTHER = 303
118NOT_MODIFIED = 304
119USE_PROXY = 305
120TEMPORARY_REDIRECT = 307
121
122# client error
123BAD_REQUEST = 400
124UNAUTHORIZED = 401
125PAYMENT_REQUIRED = 402
126FORBIDDEN = 403
127NOT_FOUND = 404
128METHOD_NOT_ALLOWED = 405
129NOT_ACCEPTABLE = 406
130PROXY_AUTHENTICATION_REQUIRED = 407
131REQUEST_TIMEOUT = 408
132CONFLICT = 409
133GONE = 410
134LENGTH_REQUIRED = 411
135PRECONDITION_FAILED = 412
136REQUEST_ENTITY_TOO_LARGE = 413
137REQUEST_URI_TOO_LONG = 414
138UNSUPPORTED_MEDIA_TYPE = 415
139REQUESTED_RANGE_NOT_SATISFIABLE = 416
140EXPECTATION_FAILED = 417
141UNPROCESSABLE_ENTITY = 422
142LOCKED = 423
143FAILED_DEPENDENCY = 424
144UPGRADE_REQUIRED = 426
145
146# server error
147INTERNAL_SERVER_ERROR = 500
148NOT_IMPLEMENTED = 501
149BAD_GATEWAY = 502
150SERVICE_UNAVAILABLE = 503
151GATEWAY_TIMEOUT = 504
152HTTP_VERSION_NOT_SUPPORTED = 505
153INSUFFICIENT_STORAGE = 507
154NOT_EXTENDED = 510
155
156# Mapping status codes to official W3C names
157responses = {
158 100: 'Continue',
159 101: 'Switching Protocols',
160
161 200: 'OK',
162 201: 'Created',
163 202: 'Accepted',
164 203: 'Non-Authoritative Information',
165 204: 'No Content',
166 205: 'Reset Content',
167 206: 'Partial Content',
168
169 300: 'Multiple Choices',
170 301: 'Moved Permanently',
171 302: 'Found',
172 303: 'See Other',
173 304: 'Not Modified',
174 305: 'Use Proxy',
175 306: '(Unused)',
176 307: 'Temporary Redirect',
177
178 400: 'Bad Request',
179 401: 'Unauthorized',
180 402: 'Payment Required',
181 403: 'Forbidden',
182 404: 'Not Found',
183 405: 'Method Not Allowed',
184 406: 'Not Acceptable',
185 407: 'Proxy Authentication Required',
186 408: 'Request Timeout',
187 409: 'Conflict',
188 410: 'Gone',
189 411: 'Length Required',
190 412: 'Precondition Failed',
191 413: 'Request Entity Too Large',
192 414: 'Request-URI Too Long',
193 415: 'Unsupported Media Type',
194 416: 'Requested Range Not Satisfiable',
195 417: 'Expectation Failed',
196
197 500: 'Internal Server Error',
198 501: 'Not Implemented',
199 502: 'Bad Gateway',
200 503: 'Service Unavailable',
201 504: 'Gateway Timeout',
202 505: 'HTTP Version Not Supported',
203}
204
205# maximal amount of data to read at one time in _safe_read
206MAXAMOUNT = 1048576
207
208class HTTPMessage(mimetools.Message):
209
210 def addheader(self, key, value):
211 """Add header for field key handling repeats."""
212 prev = self.dict.get(key)
213 if prev is None:
214 self.dict[key] = value
215 else:
216 combined = ", ".join((prev, value))
217 self.dict[key] = combined
218
219 def addcontinue(self, key, more):
220 """Add more field data from a continuation line."""
221 prev = self.dict[key]
222 self.dict[key] = prev + "\n " + more
223
224 def readheaders(self):
225 """Read header lines.
226
227 Read header lines up to the entirely blank line that terminates them.
228 The (normally blank) line that ends the headers is skipped, but not
229 included in the returned list. If a non-header line ends the headers,
230 (which is an error), an attempt is made to backspace over it; it is
231 never included in the returned list.
232
233 The variable self.status is set to the empty string if all went well,
234 otherwise it is an error message. The variable self.headers is a
235 completely uninterpreted list of lines contained in the header (so
236 printing them will reproduce the header exactly as it appears in the
237 file).
238
239 If multiple header fields with the same name occur, they are combined
240 according to the rules in RFC 2616 sec 4.2:
241
242 Appending each subsequent field-value to the first, each separated
243 by a comma. The order in which header fields with the same field-name
244 are received is significant to the interpretation of the combined
245 field value.
246 """
247 # XXX The implementation overrides the readheaders() method of
248 # rfc822.Message. The base class design isn't amenable to
249 # customized behavior here so the method here is a copy of the
250 # base class code with a few small changes.
251
252 self.dict = {}
253 self.unixfrom = ''
254 self.headers = hlist = []
255 self.status = ''
256 headerseen = ""
257 firstline = 1
258 startofline = unread = tell = None
259 if hasattr(self.fp, 'unread'):
260 unread = self.fp.unread
261 elif self.seekable:
262 tell = self.fp.tell
263 while True:
264 if tell:
265 try:
266 startofline = tell()
267 except IOError:
268 startofline = tell = None
269 self.seekable = 0
270 line = self.fp.readline()
271 if not line:
272 self.status = 'EOF in headers'
273 break
274 # Skip unix From name time lines
275 if firstline and line.startswith('From '):
276 self.unixfrom = self.unixfrom + line
277 continue
278 firstline = 0
279 if headerseen and line[0] in ' \t':
280 # XXX Not sure if continuation lines are handled properly
281 # for http and/or for repeating headers
282 # It's a continuation line.
283 hlist.append(line)
284 self.addcontinue(headerseen, line.strip())
285 continue
286 elif self.iscomment(line):
287 # It's a comment. Ignore it.
288 continue
289 elif self.islast(line):
290 # Note! No pushback here! The delimiter line gets eaten.
291 break
292 headerseen = self.isheader(line)
293 if headerseen:
294 # It's a legal header line, save it.
295 hlist.append(line)
296 self.addheader(headerseen, line[len(headerseen)+1:].strip())
297 continue
298 else:
299 # It's not a header line; throw it back and stop here.
300 if not self.dict:
301 self.status = 'No headers'
302 else:
303 self.status = 'Non-header line where header expected'
304 # Try to undo the read.
305 if unread:
306 unread(line)
307 elif tell:
308 self.fp.seek(startofline)
309 else:
310 self.status = self.status + '; bad seek'
311 break
312
313class HTTPResponse:
314
315 # strict: If true, raise BadStatusLine if the status line can't be
316 # parsed as a valid HTTP/1.0 or 1.1 status line. By default it is
317 # false because it prevents clients from talking to HTTP/0.9
318 # servers. Note that a response with a sufficiently corrupted
319 # status line will look like an HTTP/0.9 response.
320
321 # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
322
323 def __init__(self, sock, debuglevel=0, strict=0, method=None):
324 self.fp = sock.makefile('rb', 0)
325 self.debuglevel = debuglevel
326 self.strict = strict
327 self._method = method
328
329 self.msg = None
330
331 # from the Status-Line of the response
332 self.version = _UNKNOWN # HTTP-Version
333 self.status = _UNKNOWN # Status-Code
334 self.reason = _UNKNOWN # Reason-Phrase
335
336 self.chunked = _UNKNOWN # is "chunked" being used?
337 self.chunk_left = _UNKNOWN # bytes left to read in current chunk
338 self.length = _UNKNOWN # number of bytes left in response
339 self.will_close = _UNKNOWN # conn will close at end of response
340
341 def _read_status(self):
342 # Initialize with Simple-Response defaults
343 line = self.fp.readline()
344 if self.debuglevel > 0:
345 print "reply:", repr(line)
346 if not line:
347 # Presumably, the server closed the connection before
348 # sending a valid response.
349 raise BadStatusLine(line)
350 try:
351 [version, status, reason] = line.split(None, 2)
352 except ValueError:
353 try:
354 [version, status] = line.split(None, 1)
355 reason = ""
356 except ValueError:
357 # empty version will cause next test to fail and status
358 # will be treated as 0.9 response.
359 version = ""
360 if not version.startswith('HTTP/'):
361 if self.strict:
362 self.close()
363 raise BadStatusLine(line)
364 else:
365 # assume it's a Simple-Response from an 0.9 server
366 self.fp = LineAndFileWrapper(line, self.fp)
367 return "HTTP/0.9", 200, ""
368
369 # The status code is a three-digit number
370 try:
371 status = int(status)
372 if status < 100 or status > 999:
373 raise BadStatusLine(line)
374 except ValueError:
375 raise BadStatusLine(line)
376 return version, status, reason
377
378 def begin(self):
379 if self.msg is not None:
380 # we've already started reading the response
381 return
382
383 # read until we get a non-100 response
384 while True:
385 version, status, reason = self._read_status()
386 if status != CONTINUE:
387 break
388 # skip the header from the 100 response
389 while True:
390 skip = self.fp.readline().strip()
391 if not skip:
392 break
393 if self.debuglevel > 0:
394 print "header:", skip
395
396 self.status = status
397 self.reason = reason.strip()
398 if version == 'HTTP/1.0':
399 self.version = 10
400 elif version.startswith('HTTP/1.'):
401 self.version = 11 # use HTTP/1.1 code for HTTP/1.x where x>=1
402 elif version == 'HTTP/0.9':
403 self.version = 9
404 else:
405 raise UnknownProtocol(version)
406
407 if self.version == 9:
408 self.length = None
409 self.chunked = 0
410 self.will_close = 1
411 self.msg = HTTPMessage(StringIO())
412 return
413
414 self.msg = HTTPMessage(self.fp, 0)
415 if self.debuglevel > 0:
416 for hdr in self.msg.headers:
417 print "header:", hdr,
418
419 # don't let the msg keep an fp
420 self.msg.fp = None
421
422 # are we using the chunked-style of transfer encoding?
423 tr_enc = self.msg.getheader('transfer-encoding')
424 if tr_enc and tr_enc.lower() == "chunked":
425 self.chunked = 1
426 self.chunk_left = None
427 else:
428 self.chunked = 0
429
430 # will the connection close at the end of the response?
431 self.will_close = self._check_close()
432
433 # do we have a Content-Length?
434 # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
435 length = self.msg.getheader('content-length')
436 if length and not self.chunked:
437 try:
438 self.length = int(length)
439 except ValueError:
440 self.length = None
441 else:
442 self.length = None
443
444 # does the body have a fixed length? (of zero)
445 if (status == NO_CONTENT or status == NOT_MODIFIED or
446 100 <= status < 200 or # 1xx codes
447 self._method == 'HEAD'):
448 self.length = 0
449
450 # if the connection remains open, and we aren't using chunked, and
451 # a content-length was not provided, then assume that the connection
452 # WILL close.
453 if not self.will_close and \
454 not self.chunked and \
455 self.length is None:
456 self.will_close = 1
457
458 def _check_close(self):
459 conn = self.msg.getheader('connection')
460 if self.version == 11:
461 # An HTTP/1.1 proxy is assumed to stay open unless
462 # explicitly closed.
463 conn = self.msg.getheader('connection')
464 if conn and "close" in conn.lower():
465 return True
466 return False
467
468 # Some HTTP/1.0 implementations have support for persistent
469 # connections, using rules different than HTTP/1.1.
470
471 # For older HTTP, Keep-Alive indiciates persistent connection.
472 if self.msg.getheader('keep-alive'):
473 return False
474
475 # At least Akamai returns a "Connection: Keep-Alive" header,
476 # which was supposed to be sent by the client.
477 if conn and "keep-alive" in conn.lower():
478 return False
479
480 # Proxy-Connection is a netscape hack.
481 pconn = self.msg.getheader('proxy-connection')
482 if pconn and "keep-alive" in pconn.lower():
483 return False
484
485 # otherwise, assume it will close
486 return True
487
488 def close(self):
489 if self.fp:
490 self.fp.close()
491 self.fp = None
492
493 def isclosed(self):
494 # NOTE: it is possible that we will not ever call self.close(). This
495 # case occurs when will_close is TRUE, length is None, and we
496 # read up to the last byte, but NOT past it.
497 #
498 # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
499 # called, meaning self.isclosed() is meaningful.
500 return self.fp is None
501
502 # XXX It would be nice to have readline and __iter__ for this, too.
503
504 def read(self, amt=None):
505 if self.fp is None:
506 return ''
507
508 if self.chunked:
509 return self._read_chunked(amt)
510
511 if amt is None:
512 # unbounded read
513 if self.length is None:
514 s = self.fp.read()
515 else:
516 s = self._safe_read(self.length)
517 self.length = 0
518 self.close() # we read everything
519 return s
520
521 if self.length is not None:
522 if amt > self.length:
523 # clip the read to the "end of response"
524 amt = self.length
525
526 # we do not use _safe_read() here because this may be a .will_close
527 # connection, and the user is reading more bytes than will be provided
528 # (for example, reading in 1k chunks)
529 s = self.fp.read(amt)
530 if self.length is not None:
531 self.length -= len(s)
532
533 return s
534
535 def _read_chunked(self, amt):
536 assert self.chunked != _UNKNOWN
537 chunk_left = self.chunk_left
538 value = ''
539
540 # XXX This accumulates chunks by repeated string concatenation,
541 # which is not efficient as the number or size of chunks gets big.
542 while True:
543 if chunk_left is None:
544 line = self.fp.readline()
545 i = line.find(';')
546 if i >= 0:
547 line = line[:i] # strip chunk-extensions
548 chunk_left = int(line, 16)
549 if chunk_left == 0:
550 break
551 if amt is None:
552 value += self._safe_read(chunk_left)
553 elif amt < chunk_left:
554 value += self._safe_read(amt)
555 self.chunk_left = chunk_left - amt
556 return value
557 elif amt == chunk_left:
558 value += self._safe_read(amt)
559 self._safe_read(2) # toss the CRLF at the end of the chunk
560 self.chunk_left = None
561 return value
562 else:
563 value += self._safe_read(chunk_left)
564 amt -= chunk_left
565
566 # we read the whole chunk, get another
567 self._safe_read(2) # toss the CRLF at the end of the chunk
568 chunk_left = None
569
570 # read and discard trailer up to the CRLF terminator
571 ### note: we shouldn't have any trailers!
572 while True:
573 line = self.fp.readline()
574 if line == '\r\n':
575 break
576
577 # we read everything; close the "file"
578 self.close()
579
580 return value
581
582 def _safe_read(self, amt):
583 """Read the number of bytes requested, compensating for partial reads.
584
585 Normally, we have a blocking socket, but a read() can be interrupted
586 by a signal (resulting in a partial read).
587
588 Note that we cannot distinguish between EOF and an interrupt when zero
589 bytes have been read. IncompleteRead() will be raised in this
590 situation.
591
592 This function should be used when <amt> bytes "should" be present for
593 reading. If the bytes are truly not available (due to EOF), then the
594 IncompleteRead exception can be used to detect the problem.
595 """
596 s = []
597 while amt > 0:
598 chunk = self.fp.read(min(amt, MAXAMOUNT))
599 if not chunk:
600 raise IncompleteRead(s)
601 s.append(chunk)
602 amt -= len(chunk)
603 return ''.join(s)
604
605 def getheader(self, name, default=None):
606 if self.msg is None:
607 raise ResponseNotReady()
608 return self.msg.getheader(name, default)
609
610 def getheaders(self):
611 """Return list of (header, value) tuples."""
612 if self.msg is None:
613 raise ResponseNotReady()
614 return self.msg.items()
615
616
617class HTTPConnection:
618
619 _http_vsn = 11
620 _http_vsn_str = 'HTTP/1.1'
621
622 response_class = HTTPResponse
623 default_port = HTTP_PORT
624 auto_open = 1
625 debuglevel = 0
626 strict = 0
627
628 def __init__(self, host, port=None, strict=None):
629 self.sock = None
630 self._buffer = []
631 self.__response = None
632 self.__state = _CS_IDLE
633 self._method = None
634
635 self._set_hostport(host, port)
636 if strict is not None:
637 self.strict = strict
638
639 def _set_hostport(self, host, port):
640 if port is None:
641 i = host.rfind(':')
642 j = host.rfind(']') # ipv6 addresses have [...]
643 if i > j:
644 try:
645 port = int(host[i+1:])
646 except ValueError:
647 raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
648 host = host[:i]
649 else:
650 port = self.default_port
651 if host and host[0] == '[' and host[-1] == ']':
652 host = host[1:-1]
653 self.host = host
654 self.port = port
655
656 def set_debuglevel(self, level):
657 self.debuglevel = level
658
659 def connect(self):
660 """Connect to the host and port specified in __init__."""
661 msg = "getaddrinfo returns an empty list"
662 for res in socket.getaddrinfo(self.host, self.port, 0,
663 socket.SOCK_STREAM):
664 af, socktype, proto, canonname, sa = res
665 try:
666 self.sock = socket.socket(af, socktype, proto)
667 if self.debuglevel > 0:
668 print "connect: (%s, %s)" % (self.host, self.port)
669 self.sock.connect(sa)
670 except socket.error, msg:
671 if self.debuglevel > 0:
672 print 'connect fail:', (self.host, self.port)
673 if self.sock:
674 self.sock.close()
675 self.sock = None
676 continue
677 break
678 if not self.sock:
679 raise socket.error, msg
680
681 def close(self):
682 """Close the connection to the HTTP server."""
683 if self.sock:
684 self.sock.close() # close it manually... there may be other refs
685 self.sock = None
686 if self.__response:
687 self.__response.close()
688 self.__response = None
689 self.__state = _CS_IDLE
690
691 def send(self, str):
692 """Send `str' to the server."""
693 if self.sock is None:
694 if self.auto_open:
695 self.connect()
696 else:
697 raise NotConnected()
698
699 # send the data to the server. if we get a broken pipe, then close
700 # the socket. we want to reconnect when somebody tries to send again.
701 #
702 # NOTE: we DO propagate the error, though, because we cannot simply
703 # ignore the error... the caller will know if they can retry.
704 if self.debuglevel > 0:
705 print "send:", repr(str)
706 try:
707 self.sock.sendall(str)
708 except socket.error, v:
709 if v[0] == 32: # Broken pipe
710 self.close()
711 raise
712
713 def _output(self, s):
714 """Add a line of output to the current request buffer.
715
716 Assumes that the line does *not* end with \\r\\n.
717 """
718 self._buffer.append(s)
719
720 def _send_output(self):
721 """Send the currently buffered request and clear the buffer.
722
723 Appends an extra \\r\\n to the buffer.
724 """
725 self._buffer.extend(("", ""))
726 msg = "\r\n".join(self._buffer)
727 del self._buffer[:]
728 self.send(msg)
729
730 def putrequest(self, method, url, skip_host=0, skip_accept_encoding=0):
731 """Send a request to the server.
732
733 `method' specifies an HTTP request method, e.g. 'GET'.
734 `url' specifies the object being requested, e.g. '/index.html'.
735 `skip_host' if True does not add automatically a 'Host:' header
736 `skip_accept_encoding' if True does not add automatically an
737 'Accept-Encoding:' header
738 """
739
740 # if a prior response has been completed, then forget about it.
741 if self.__response and self.__response.isclosed():
742 self.__response = None
743
744
745 # in certain cases, we cannot issue another request on this connection.
746 # this occurs when:
747 # 1) we are in the process of sending a request. (_CS_REQ_STARTED)
748 # 2) a response to a previous request has signalled that it is going
749 # to close the connection upon completion.
750 # 3) the headers for the previous response have not been read, thus
751 # we cannot determine whether point (2) is true. (_CS_REQ_SENT)
752 #
753 # if there is no prior response, then we can request at will.
754 #
755 # if point (2) is true, then we will have passed the socket to the
756 # response (effectively meaning, "there is no prior response"), and
757 # will open a new one when a new request is made.
758 #
759 # Note: if a prior response exists, then we *can* start a new request.
760 # We are not allowed to begin fetching the response to this new
761 # request, however, until that prior response is complete.
762 #
763 if self.__state == _CS_IDLE:
764 self.__state = _CS_REQ_STARTED
765 else:
766 raise CannotSendRequest()
767
768 # Save the method we use, we need it later in the response phase
769 self._method = method
770 if not url:
771 url = '/'
772 str = '%s %s %s' % (method, url, self._http_vsn_str)
773
774 self._output(str)
775
776 if self._http_vsn == 11:
777 # Issue some standard headers for better HTTP/1.1 compliance
778
779 if not skip_host:
780 # this header is issued *only* for HTTP/1.1
781 # connections. more specifically, this means it is
782 # only issued when the client uses the new
783 # HTTPConnection() class. backwards-compat clients
784 # will be using HTTP/1.0 and those clients may be
785 # issuing this header themselves. we should NOT issue
786 # it twice; some web servers (such as Apache) barf
787 # when they see two Host: headers
788
789 # If we need a non-standard port,include it in the
790 # header. If the request is going through a proxy,
791 # but the host of the actual URL, not the host of the
792 # proxy.
793
794 netloc = ''
795 if url.startswith('http'):
796 nil, netloc, nil, nil, nil = urlsplit(url)
797
798 if netloc:
799 try:
800 netloc_enc = netloc.encode("ascii")
801 except UnicodeEncodeError:
802 netloc_enc = netloc.encode("idna")
803 self.putheader('Host', netloc_enc)
804 else:
805 try:
806 host_enc = self.host.encode("ascii")
807 except UnicodeEncodeError:
808 host_enc = self.host.encode("idna")
809 if self.port == HTTP_PORT:
810 self.putheader('Host', host_enc)
811 else:
812 self.putheader('Host', "%s:%s" % (host_enc, self.port))
813
814 # note: we are assuming that clients will not attempt to set these
815 # headers since *this* library must deal with the
816 # consequences. this also means that when the supporting
817 # libraries are updated to recognize other forms, then this
818 # code should be changed (removed or updated).
819
820 # we only want a Content-Encoding of "identity" since we don't
821 # support encodings such as x-gzip or x-deflate.
822 if not skip_accept_encoding:
823 self.putheader('Accept-Encoding', 'identity')
824
825 # we can accept "chunked" Transfer-Encodings, but no others
826 # NOTE: no TE header implies *only* "chunked"
827 #self.putheader('TE', 'chunked')
828
829 # if TE is supplied in the header, then it must appear in a
830 # Connection header.
831 #self.putheader('Connection', 'TE')
832
833 else:
834 # For HTTP/1.0, the server will assume "not chunked"
835 pass
836
837 def putheader(self, header, value):
838 """Send a request header line to the server.
839
840 For example: h.putheader('Accept', 'text/html')
841 """
842 if self.__state != _CS_REQ_STARTED:
843 raise CannotSendHeader()
844
845 str = '%s: %s' % (header, value)
846 self._output(str)
847
848 def endheaders(self):
849 """Indicate that the last header line has been sent to the server."""
850
851 if self.__state == _CS_REQ_STARTED:
852 self.__state = _CS_REQ_SENT
853 else:
854 raise CannotSendHeader()
855
856 self._send_output()
857
858 def request(self, method, url, body=None, headers={}):
859 """Send a complete request to the server."""
860
861 try:
862 self._send_request(method, url, body, headers)
863 except socket.error, v:
864 # trap 'Broken pipe' if we're allowed to automatically reconnect
865 if v[0] != 32 or not self.auto_open:
866 raise
867 # try one more time
868 self._send_request(method, url, body, headers)
869
870 def _send_request(self, method, url, body, headers):
871 # honour explicitly requested Host: and Accept-Encoding headers
872 header_names = dict.fromkeys([k.lower() for k in headers])
873 skips = {}
874 if 'host' in header_names:
875 skips['skip_host'] = 1
876 if 'accept-encoding' in header_names:
877 skips['skip_accept_encoding'] = 1
878
879 self.putrequest(method, url, **skips)
880
881 if body and ('content-length' not in header_names):
882 self.putheader('Content-Length', str(len(body)))
883 for hdr, value in headers.iteritems():
884 self.putheader(hdr, value)
885 self.endheaders()
886
887 if body:
888 self.send(body)
889
890 def getresponse(self):
891 "Get the response from the server."
892
893 # if a prior response has been completed, then forget about it.
894 if self.__response and self.__response.isclosed():
895 self.__response = None
896
897 #
898 # if a prior response exists, then it must be completed (otherwise, we
899 # cannot read this response's header to determine the connection-close
900 # behavior)
901 #
902 # note: if a prior response existed, but was connection-close, then the
903 # socket and response were made independent of this HTTPConnection
904 # object since a new request requires that we open a whole new
905 # connection
906 #
907 # this means the prior response had one of two states:
908 # 1) will_close: this connection was reset and the prior socket and
909 # response operate independently
910 # 2) persistent: the response was retained and we await its
911 # isclosed() status to become true.
912 #
913 if self.__state != _CS_REQ_SENT or self.__response:
914 raise ResponseNotReady()
915
916 if self.debuglevel > 0:
917 response = self.response_class(self.sock, self.debuglevel,
918 strict=self.strict,
919 method=self._method)
920 else:
921 response = self.response_class(self.sock, strict=self.strict,
922 method=self._method)
923
924 response.begin()
925 assert response.will_close != _UNKNOWN
926 self.__state = _CS_IDLE
927
928 if response.will_close:
929 # Pass the socket to the response
930 self.sock = None
931 else:
932 # remember this, so we can tell when it is complete
933 self.__response = response
934
935 return response
936
937# The next several classes are used to define FakeSocket, a socket-like
938# interface to an SSL connection.
939
940# The primary complexity comes from faking a makefile() method. The
941# standard socket makefile() implementation calls dup() on the socket
942# file descriptor. As a consequence, clients can call close() on the
943# parent socket and its makefile children in any order. The underlying
944# socket isn't closed until they are all closed.
945
946# The implementation uses reference counting to keep the socket open
947# until the last client calls close(). SharedSocket keeps track of
948# the reference counting and SharedSocketClient provides an constructor
949# and close() method that call incref() and decref() correctly.
950
951class SharedSocket:
952
953 def __init__(self, sock):
954 self.sock = sock
955 self._refcnt = 0
956
957 def incref(self):
958 self._refcnt += 1
959
960 def decref(self):
961 self._refcnt -= 1
962 assert self._refcnt >= 0
963 if self._refcnt == 0:
964 self.sock.close()
965
966 def __del__(self):
967 self.sock.close()
968
969class SharedSocketClient:
970
971 def __init__(self, shared):
972 self._closed = 0
973 self._shared = shared
974 self._shared.incref()
975 self._sock = shared.sock
976
977 def close(self):
978 if not self._closed:
979 self._shared.decref()
980 self._closed = 1
981 self._shared = None
982
983class SSLFile(SharedSocketClient):
984 """File-like object wrapping an SSL socket."""
985
986 BUFSIZE = 8192
987
988 def __init__(self, sock, ssl, bufsize=None):
989 SharedSocketClient.__init__(self, sock)
990 self._ssl = ssl
991 self._buf = ''
992 self._bufsize = bufsize or self.__class__.BUFSIZE
993
994 def _read(self):
995 buf = ''
996 # put in a loop so that we retry on transient errors
997 while True:
998 try:
999 buf = self._ssl.read(self._bufsize)
1000 except socket.sslerror, err:
1001 if (err[0] == socket.SSL_ERROR_WANT_READ
1002 or err[0] == socket.SSL_ERROR_WANT_WRITE):
1003 continue
1004 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
1005 or err[0] == socket.SSL_ERROR_EOF):
1006 break
1007 raise
1008 except socket.error, err:
1009 if err[0] == errno.EINTR:
1010 continue
1011 if err[0] == errno.EBADF:
1012 # XXX socket was closed?
1013 break
1014 raise
1015 else:
1016 break
1017 return buf
1018
1019 def read(self, size=None):
1020 L = [self._buf]
1021 avail = len(self._buf)
1022 while size is None or avail < size:
1023 s = self._read()
1024 if s == '':
1025 break
1026 L.append(s)
1027 avail += len(s)
1028 all = "".join(L)
1029 if size is None:
1030 self._buf = ''
1031 return all
1032 else:
1033 self._buf = all[size:]
1034 return all[:size]
1035
1036 def readline(self):
1037 L = [self._buf]
1038 self._buf = ''
1039 while 1:
1040 i = L[-1].find("\n")
1041 if i >= 0:
1042 break
1043 s = self._read()
1044 if s == '':
1045 break
1046 L.append(s)
1047 if i == -1:
1048 # loop exited because there is no more data
1049 return "".join(L)
1050 else:
1051 all = "".join(L)
1052 # XXX could do enough bookkeeping not to do a 2nd search
1053 i = all.find("\n") + 1
1054 line = all[:i]
1055 self._buf = all[i:]
1056 return line
1057
1058 def readlines(self, sizehint=0):
1059 total = 0
1060 list = []
1061 while True:
1062 line = self.readline()
1063 if not line:
1064 break
1065 list.append(line)
1066 total += len(line)
1067 if sizehint and total >= sizehint:
1068 break
1069 return list
1070
1071 def fileno(self):
1072 return self._sock.fileno()
1073
1074 def __iter__(self):
1075 return self
1076
1077 def next(self):
1078 line = self.readline()
1079 if not line:
1080 raise StopIteration
1081 return line
1082
1083class FakeSocket(SharedSocketClient):
1084
1085 class _closedsocket:
1086 def __getattr__(self, name):
1087 raise error(9, 'Bad file descriptor')
1088
1089 def __init__(self, sock, ssl):
1090 sock = SharedSocket(sock)
1091 SharedSocketClient.__init__(self, sock)
1092 self._ssl = ssl
1093
1094 def close(self):
1095 SharedSocketClient.close(self)
1096 self._sock = self.__class__._closedsocket()
1097
1098 def makefile(self, mode, bufsize=None):
1099 if mode != 'r' and mode != 'rb':
1100 raise UnimplementedFileMode()
1101 return SSLFile(self._shared, self._ssl, bufsize)
1102
1103 def send(self, stuff, flags = 0):
1104 return self._ssl.write(stuff)
1105
1106 sendall = send
1107
1108 def recv(self, len = 1024, flags = 0):
1109 return self._ssl.read(len)
1110
1111 def __getattr__(self, attr):
1112 return getattr(self._sock, attr)
1113
1114
1115class HTTPSConnection(HTTPConnection):
1116 "This class allows communication via SSL."
1117
1118 default_port = HTTPS_PORT
1119
1120 def __init__(self, host, port=None, key_file=None, cert_file=None,
1121 strict=None):
1122 HTTPConnection.__init__(self, host, port, strict)
1123 self.key_file = key_file
1124 self.cert_file = cert_file
1125
1126 def connect(self):
1127 "Connect to a host on a given (SSL) port."
1128
1129 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
1130 sock.connect((self.host, self.port))
1131 ssl = socket.ssl(sock, self.key_file, self.cert_file)
1132 self.sock = FakeSocket(sock, ssl)
1133
1134
1135class HTTP:
1136 "Compatibility class with httplib.py from 1.5."
1137
1138 _http_vsn = 10
1139 _http_vsn_str = 'HTTP/1.0'
1140
1141 debuglevel = 0
1142
1143 _connection_class = HTTPConnection
1144
1145 def __init__(self, host='', port=None, strict=None):
1146 "Provide a default host, since the superclass requires one."
1147
1148 # some joker passed 0 explicitly, meaning default port
1149 if port == 0:
1150 port = None
1151
1152 # Note that we may pass an empty string as the host; this will throw
1153 # an error when we attempt to connect. Presumably, the client code
1154 # will call connect before then, with a proper host.
1155 self._setup(self._connection_class(host, port, strict))
1156
1157 def _setup(self, conn):
1158 self._conn = conn
1159
1160 # set up delegation to flesh out interface
1161 self.send = conn.send
1162 self.putrequest = conn.putrequest
1163 self.endheaders = conn.endheaders
1164 self.set_debuglevel = conn.set_debuglevel
1165
1166 conn._http_vsn = self._http_vsn
1167 conn._http_vsn_str = self._http_vsn_str
1168
1169 self.file = None
1170
1171 def connect(self, host=None, port=None):
1172 "Accept arguments to set the host/port, since the superclass doesn't."
1173
1174 if host is not None:
1175 self._conn._set_hostport(host, port)
1176 self._conn.connect()
1177
1178 def getfile(self):
1179 "Provide a getfile, since the superclass' does not use this concept."
1180 return self.file
1181
1182 def putheader(self, header, *values):
1183 "The superclass allows only one value argument."
1184 self._conn.putheader(header, '\r\n\t'.join(values))
1185
1186 def getreply(self):
1187 """Compat definition since superclass does not define it.
1188
1189 Returns a tuple consisting of:
1190 - server status code (e.g. '200' if all goes well)
1191 - server "reason" corresponding to status code
1192 - any RFC822 headers in the response from the server
1193 """
1194 try:
1195 response = self._conn.getresponse()
1196 except BadStatusLine, e:
1197 ### hmm. if getresponse() ever closes the socket on a bad request,
1198 ### then we are going to have problems with self.sock
1199
1200 ### should we keep this behavior? do people use it?
1201 # keep the socket open (as a file), and return it
1202 self.file = self._conn.sock.makefile('rb', 0)
1203
1204 # close our socket -- we want to restart after any protocol error
1205 self.close()
1206
1207 self.headers = None
1208 return -1, e.line, None
1209
1210 self.headers = response.msg
1211 self.file = response.fp
1212 return response.status, response.reason, response.msg
1213
1214 def close(self):
1215 self._conn.close()
1216
1217 # note that self.file == response.fp, which gets closed by the
1218 # superclass. just clear the object ref here.
1219 ### hmm. messy. if status==-1, then self.file is owned by us.
1220 ### well... we aren't explicitly closing, but losing this ref will
1221 ### do it
1222 self.file = None
1223
1224if hasattr(socket, 'ssl'):
1225 class HTTPS(HTTP):
1226 """Compatibility with 1.5 httplib interface
1227
1228 Python 1.5.2 did not have an HTTPS class, but it defined an
1229 interface for sending http requests that is also useful for
1230 https.
1231 """
1232
1233 _connection_class = HTTPSConnection
1234
1235 def __init__(self, host='', port=None, key_file=None, cert_file=None,
1236 strict=None):
1237 # provide a default host, pass the X509 cert info
1238
1239 # urf. compensate for bad input.
1240 if port == 0:
1241 port = None
1242 self._setup(self._connection_class(host, port, key_file,
1243 cert_file, strict))
1244
1245 # we never actually use these for anything, but we keep them
1246 # here for compatibility with post-1.5.2 CVS.
1247 self.key_file = key_file
1248 self.cert_file = cert_file
1249
1250
1251class HTTPException(Exception):
1252 # Subclasses that define an __init__ must call Exception.__init__
1253 # or define self.args. Otherwise, str() will fail.
1254 pass
1255
1256class NotConnected(HTTPException):
1257 pass
1258
1259class InvalidURL(HTTPException):
1260 pass
1261
1262class UnknownProtocol(HTTPException):
1263 def __init__(self, version):
1264 self.args = version,
1265 self.version = version
1266
1267class UnknownTransferEncoding(HTTPException):
1268 pass
1269
1270class UnimplementedFileMode(HTTPException):
1271 pass
1272
1273class IncompleteRead(HTTPException):
1274 def __init__(self, partial):
1275 self.args = partial,
1276 self.partial = partial
1277
1278class ImproperConnectionState(HTTPException):
1279 pass
1280
1281class CannotSendRequest(ImproperConnectionState):
1282 pass
1283
1284class CannotSendHeader(ImproperConnectionState):
1285 pass
1286
1287class ResponseNotReady(ImproperConnectionState):
1288 pass
1289
1290class BadStatusLine(HTTPException):
1291 def __init__(self, line):
1292 self.args = line,
1293 self.line = line
1294
1295# for backwards compatibility
1296error = HTTPException
1297
1298class LineAndFileWrapper:
1299 """A limited file-like object for HTTP/0.9 responses."""
1300
1301 # The status-line parsing code calls readline(), which normally
1302 # get the HTTP status line. For a 0.9 response, however, this is
1303 # actually the first line of the body! Clients need to get a
1304 # readable file object that contains that line.
1305
1306 def __init__(self, line, file):
1307 self._line = line
1308 self._file = file
1309 self._line_consumed = 0
1310 self._line_offset = 0
1311 self._line_left = len(line)
1312
1313 def __getattr__(self, attr):
1314 return getattr(self._file, attr)
1315
1316 def _done(self):
1317 # called when the last byte is read from the line. After the
1318 # call, all read methods are delegated to the underlying file
1319 # object.
1320 self._line_consumed = 1
1321 self.read = self._file.read
1322 self.readline = self._file.readline
1323 self.readlines = self._file.readlines
1324
1325 def read(self, amt=None):
1326 if self._line_consumed:
1327 return self._file.read(amt)
1328 assert self._line_left
1329 if amt is None or amt > self._line_left:
1330 s = self._line[self._line_offset:]
1331 self._done()
1332 if amt is None:
1333 return s + self._file.read()
1334 else:
1335 return s + self._file.read(amt - len(s))
1336 else:
1337 assert amt <= self._line_left
1338 i = self._line_offset
1339 j = i + amt
1340 s = self._line[i:j]
1341 self._line_offset = j
1342 self._line_left -= amt
1343 if self._line_left == 0:
1344 self._done()
1345 return s
1346
1347 def readline(self):
1348 if self._line_consumed:
1349 return self._file.readline()
1350 assert self._line_left
1351 s = self._line[self._line_offset:]
1352 self._done()
1353 return s
1354
1355 def readlines(self, size=None):
1356 if self._line_consumed:
1357 return self._file.readlines(size)
1358 assert self._line_left
1359 L = [self._line[self._line_offset:]]
1360 self._done()
1361 if size is None:
1362 return L + self._file.readlines()
1363 else:
1364 return L + self._file.readlines(size)
1365
1366def test():
1367 """Test this module.
1368
1369 A hodge podge of tests collected here, because they have too many
1370 external dependencies for the regular test suite.
1371 """
1372
1373 import sys
1374 import getopt
1375 opts, args = getopt.getopt(sys.argv[1:], 'd')
1376 dl = 0
1377 for o, a in opts:
1378 if o == '-d': dl = dl + 1
1379 host = 'www.python.org'
1380 selector = '/'
1381 if args[0:]: host = args[0]
1382 if args[1:]: selector = args[1]
1383 h = HTTP()
1384 h.set_debuglevel(dl)
1385 h.connect(host)
1386 h.putrequest('GET', selector)
1387 h.endheaders()
1388 status, reason, headers = h.getreply()
1389 print 'status =', status
1390 print 'reason =', reason
1391 print "read", len(h.getfile().read())
1392 print
1393 if headers:
1394 for header in headers.headers: print header.strip()
1395 print
1396
1397 # minimal test that code to extract host from url works
1398 class HTTP11(HTTP):
1399 _http_vsn = 11
1400 _http_vsn_str = 'HTTP/1.1'
1401
1402 h = HTTP11('www.python.org')
1403 h.putrequest('GET', 'http://www.python.org/~jeremy/')
1404 h.endheaders()
1405 h.getreply()
1406 h.close()
1407
1408 if hasattr(socket, 'ssl'):
1409
1410 for host, selector in (('sourceforge.net', '/projects/python'),
1411 ):
1412 print "https://%s%s" % (host, selector)
1413 hs = HTTPS()
1414 hs.set_debuglevel(dl)
1415 hs.connect(host)
1416 hs.putrequest('GET', selector)
1417 hs.endheaders()
1418 status, reason, headers = hs.getreply()
1419 print 'status =', status
1420 print 'reason =', reason
1421 print "read", len(hs.getfile().read())
1422 print
1423 if headers:
1424 for header in headers.headers: print header.strip()
1425 print
1426
1427if __name__ == '__main__':
1428 test()
Note: See TracBrowser for help on using the repository browser.