/** * PANDA 3D SOFTWARE * Copyright (c) Carnegie Mellon University. All rights reserved. * * All use of this software is subject to the terms of the revised BSD * license. You should have received a copy of this license along * with this source code in a file named "LICENSE." * * @file httpChannel.h * @author drose * @date 2002-09-24 */ #ifndef HTTPCHANNEL_H #define HTTPCHANNEL_H #include "pandabase.h" // This module requires OpenSSL to compile, even if you do not intend to use // this to establish https connections; this is because it uses the OpenSSL // library to portably handle all of the socket communications. #ifdef HAVE_OPENSSL #include "httpClient.h" #include "httpEnum.h" #include "urlSpec.h" #include "documentSpec.h" #include "bioPtr.h" #include "bioStreamPtr.h" #include "pmap.h" #include "pvector.h" #include "pointerTo.h" #include "config_downloader.h" #include "filename.h" #include "typedReferenceCount.h" typedef struct bio_st BIO; class Ramfile; class HTTPClient; /** * A single channel of communication from an HTTPClient. This is similar to * the concept of a 'connection', except that HTTP is technically * connectionless; in fact, a channel may represent one unbroken connection or * it may transparently close and reopen a new connection with each request. * * A channel is conceptually a single thread of I/O. One document at a time * may be requested using a channel; a new document may (in general) not be * requested from the same HTTPChannel until the first document has been fully * retrieved. */ class EXPCL_PANDA_DOWNLOADER HTTPChannel : public TypedReferenceCount { private: HTTPChannel(HTTPClient *client); public: virtual ~HTTPChannel(); PUBLISHED: // get_status_code() will either return an HTTP-style status code >= 100 // (e.g. 404), or one of the following values. In general, these are // ordered from less-successful to more-successful. enum StatusCode { SC_incomplete = 0, SC_internal_error, SC_no_connection, SC_timeout, SC_lost_connection, SC_non_http_response, SC_invalid_http, SC_socks_invalid_version, SC_socks_no_acceptable_login_method, SC_socks_refused, SC_socks_no_connection, SC_ssl_internal_failure, SC_ssl_no_handshake, // No one returns this code, but StatusCode values higher than this are // deemed more successful than any generic HTTP response. SC_http_error_watermark, SC_ssl_invalid_server_certificate, SC_ssl_self_signed_server_certificate, SC_ssl_unexpected_server, // These errors are only generated after a download_to_*() call been // issued. SC_download_open_error, SC_download_write_error, SC_download_invalid_range, }; INLINE HTTPClient *get_client() const; INLINE bool is_valid() const; INLINE bool is_connection_ready() const; INLINE const URLSpec &get_url() const; INLINE const DocumentSpec &get_document_spec() const; INLINE HTTPEnum::HTTPVersion get_http_version() const; INLINE const std::string &get_http_version_string() const; INLINE int get_status_code() const; std::string get_status_string() const; INLINE const std::string &get_www_realm() const; INLINE const std::string &get_proxy_realm() const; INLINE const URLSpec &get_redirect() const; std::string get_header_value(const std::string &key) const; INLINE int get_num_redirect_steps() const; INLINE const URLSpec &get_redirect_step(int n) const; MAKE_SEQ(get_redirect_steps, get_num_redirect_steps, get_redirect_step); INLINE void set_persistent_connection(bool persistent_connection); INLINE bool get_persistent_connection() const; bool will_close_connection() const; INLINE void set_allow_proxy(bool allow_proxy); INLINE bool get_allow_proxy() const; INLINE void set_proxy_tunnel(bool proxy_tunnel); INLINE bool get_proxy_tunnel() const; INLINE void set_connect_timeout(double timeout_seconds); INLINE double get_connect_timeout() const; INLINE void set_blocking_connect(bool blocking_connect); INLINE bool get_blocking_connect() const; INLINE void set_http_timeout(double timeout_seconds); INLINE double get_http_timeout() const; INLINE void set_skip_body_size(size_t skip_body_size); INLINE size_t get_skip_body_size() const; INLINE void set_idle_timeout(double idle_timeout); INLINE double get_idle_timeout() const; INLINE void set_download_throttle(bool download_throttle); INLINE bool get_download_throttle() const; INLINE void set_max_bytes_per_second(double max_bytes_per_second); INLINE double get_max_bytes_per_second() const; INLINE void set_max_updates_per_second(double max_updates_per_second); INLINE double get_max_updates_per_second() const; INLINE void set_content_type(std::string content_type); INLINE std::string get_content_type() const; INLINE void set_expected_file_size(size_t file_size); std::streamsize get_file_size() const; INLINE bool is_file_size_known() const; INLINE size_t get_first_byte_requested() const; INLINE size_t get_last_byte_requested() const; INLINE size_t get_first_byte_delivered() const; INLINE size_t get_last_byte_delivered() const; void write_headers(std::ostream &out) const; INLINE void reset(); INLINE void preserve_status(); INLINE void clear_extra_headers(); INLINE void send_extra_header(const std::string &key, const std::string &value); BLOCKING INLINE bool get_document(const DocumentSpec &url); BLOCKING INLINE bool get_subdocument(const DocumentSpec &url, size_t first_byte, size_t last_byte); BLOCKING INLINE bool get_header(const DocumentSpec &url); BLOCKING INLINE bool post_form(const DocumentSpec &url, const std::string &body); BLOCKING INLINE bool put_document(const DocumentSpec &url, const std::string &body); BLOCKING INLINE bool delete_document(const DocumentSpec &url); BLOCKING INLINE bool get_trace(const DocumentSpec &url); BLOCKING INLINE bool connect_to(const DocumentSpec &url); BLOCKING INLINE bool get_options(const DocumentSpec &url); INLINE void begin_get_document(const DocumentSpec &url); INLINE void begin_get_subdocument(const DocumentSpec &url, size_t first_byte, size_t last_byte); INLINE void begin_get_header(const DocumentSpec &url); INLINE void begin_post_form(const DocumentSpec &url, const std::string &body); bool run(); INLINE void begin_connect_to(const DocumentSpec &url); ISocketStream *open_read_body(); void close_read_body(std::istream *stream) const; BLOCKING bool download_to_file(const Filename &filename, bool subdocument_resumes = true); BLOCKING bool download_to_ram(Ramfile *ramfile, bool subdocument_resumes = true); BLOCKING bool download_to_stream(std::ostream *strm, bool subdocument_resumes = true); SocketStream *get_connection(); INLINE size_t get_bytes_downloaded() const; INLINE size_t get_bytes_requested() const; INLINE bool is_download_complete() const; public: static std::string downcase(const std::string &s); void body_stream_destructs(ISocketStream *stream); private: bool reached_done_state(); bool run_try_next_proxy(); bool run_connecting(); bool run_connecting_wait(); bool run_http_proxy_ready(); bool run_http_proxy_request_sent(); bool run_http_proxy_reading_header(); bool run_socks_proxy_greet(); bool run_socks_proxy_greet_reply(); bool run_socks_proxy_connect(); bool run_socks_proxy_connect_reply(); bool run_setup_ssl(); bool run_ssl_handshake(); bool run_ready(); bool run_request_sent(); bool run_reading_header(); bool run_start_direct_file_read(); bool run_read_header(); bool run_begin_body(); bool run_reading_body(); bool run_read_body(); bool run_read_trailer(); bool run_download_to_file(); bool run_download_to_ram(); bool run_download_to_stream(); void begin_request(HTTPEnum::Method method, const DocumentSpec &url, const std::string &body, bool nonblocking, size_t first_byte, size_t last_byte); void reconsider_proxy(); void reset_for_new_request(); void finished_body(bool has_trailer); bool open_download_file(); bool server_getline(std::string &str); bool server_getline_failsafe(std::string &str); bool server_get(std::string &str, size_t num_bytes); bool server_get_failsafe(std::string &str, size_t num_bytes); bool server_send(const std::string &str, bool secret); bool parse_http_response(const std::string &line); bool parse_http_header(); bool parse_content_range(const std::string &content_range); void check_socket(); void check_preapproved_server_certificate(X509 *cert, bool &cert_preapproved, bool &cert_name_preapproved) const; bool validate_server_name(X509 *cert); static bool match_cert_name(const std::string &cert_name, const std::string &hostname); static std::string get_x509_name_component(X509_NAME *name, int nid); void make_header(); void make_proxy_request_text(); void make_request_text(); void reset_url(const URLSpec &old_url, const URLSpec &new_url); void store_header_field(const std::string &field_name, const std::string &field_value); #ifndef NDEBUG static void show_send(const std::string &message); #endif void reset_download_to(); void close_download_stream(); void reset_to_new(); void reset_body_stream(); void close_connection(); static bool more_useful_status_code(int a, int b); public: // This is declared public solely so we can make an ostream operator for it. enum State { S_new, S_try_next_proxy, S_connecting, S_connecting_wait, S_http_proxy_ready, S_http_proxy_request_sent, S_http_proxy_reading_header, S_socks_proxy_greet, S_socks_proxy_greet_reply, S_socks_proxy_connect, S_socks_proxy_connect_reply, S_setup_ssl, S_ssl_handshake, S_ready, S_request_sent, S_reading_header, S_start_direct_file_read, S_read_header, S_begin_body, S_reading_body, S_read_body, S_read_trailer, S_failure }; private: class StatusEntry { public: INLINE StatusEntry(); int _status_code; std::string _status_string; }; typedef pvector Proxies; typedef pvector StatusList; HTTPClient *_client; Proxies _proxies; size_t _proxy_next_index; StatusList _status_list; URLSpec _proxy; PT(BioPtr) _bio; PT(BioStreamPtr) _source; bool _persistent_connection; bool _allow_proxy; bool _proxy_tunnel; double _connect_timeout; double _http_timeout; size_t _skip_body_size; double _idle_timeout; bool _blocking_connect; bool _download_throttle; double _max_bytes_per_second; double _max_updates_per_second; double _seconds_per_update; int _bytes_per_update; bool _nonblocking; bool _wanted_nonblocking; std::string _send_extra_headers; DocumentSpec _document_spec; DocumentSpec _request; HTTPEnum::Method _method; std::string request_path; std::string _header; std::string _body; std::string _content_type; bool _want_ssl; bool _proxy_serves_document; bool _proxy_tunnel_now; bool _server_response_has_no_body; size_t _first_byte_requested; size_t _last_byte_requested; size_t _first_byte_delivered; size_t _last_byte_delivered; int _connect_count; enum DownloadDest { DD_none, DD_file, DD_ram, DD_stream, }; DownloadDest _download_dest; bool _subdocument_resumes; Filename _download_to_filename; Ramfile *_download_to_ramfile; std::ostream *_download_to_stream; int _read_index; HTTPEnum::HTTPVersion _http_version; std::string _http_version_string; StatusEntry _status_entry; URLSpec _redirect; std::string _proxy_realm; std::string _proxy_username; PT(HTTPAuthorization) _proxy_auth; std::string _www_realm; std::string _www_username; PT(HTTPAuthorization) _www_auth; // What type of response do we get to our HTTP request? enum ResponseType { RT_none, RT_hangup, // immediately lost connection RT_non_http, // something that wasn't an expected HTTP response RT_http_hangup, // the start of an HTTP response, then a lost connection RT_http_complete // a valid HTTP response completed }; ResponseType _response_type; // Not a phash_map, to maintain sorted order. typedef pmap Headers; Headers _headers; size_t _expected_file_size; size_t _file_size; size_t _transfer_file_size; size_t _bytes_downloaded; size_t _bytes_requested; bool _got_expected_file_size; bool _got_file_size; bool _got_transfer_file_size; // These members are used to maintain the current state while communicating // with the server. We need to store everything in the class object instead // of using local variables because in the case of nonblocking IO we have to // be able to return to the caller after any IO operation and resume later // where we left off. State _state; State _done_state; double _started_connecting_time; double _sent_request_time; bool _started_download; std::string _proxy_header; std::string _proxy_request_text; std::string _request_text; std::string _working_get; size_t _sent_so_far; std::string _current_field_name; std::string _current_field_value; ISocketStream *_body_stream; bool _owns_body_stream; BIO *_sbio; std::string _cipher_list; pvector _redirect_trail; int _last_status_code; double _last_run_time; // RAU we find that we may need a little more time for the ssl handshake // when the phase files are downloading double _extra_ssl_handshake_time; public: virtual TypeHandle get_type() const { return get_class_type(); } virtual TypeHandle force_init_type() {init_type(); return get_class_type();} static TypeHandle get_class_type() { return _type_handle; } static void init_type() { TypedReferenceCount::init_type(); register_type(_type_handle, "HTTPChannel", TypedReferenceCount::get_class_type()); } private: static TypeHandle _type_handle; friend class ChunkedStreamBuf; friend class IdentityStreamBuf; friend class HTTPClient; }; std::ostream &operator << (std::ostream &out, HTTPChannel::State state); #include "httpChannel.I" #endif // HAVE_OPENSSL #endif