1 /* $NetBSD: common.c,v 1.27 2010/06/13 21:38:09 joerg Exp $ */
3 * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
4 * Copyright (c) 2008, 2010 Joerg Sonnenberger <joerg@NetBSD.org>
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer
12 * in this position and unchanged.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. The name of the author may not be used to endorse or promote products
17 * derived from this software without specific prior written permission
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 * $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $
40 #include <sys/types.h>
41 #include <sys/socket.h>
45 #include <netinet/in.h>
46 #include <arpa/inet.h>
50 #if defined(HAVE_INTTYPES_H) || defined(NETBSD)
54 #include <nbcompat/netdb.h>
72 /*** Local data **************************************************************/
75 * Error messages for resolver errors
77 static struct fetcherr netdb_errlist[] = {
79 { EAI_NODATA, FETCH_RESOLV, "Host not found" },
81 { EAI_AGAIN, FETCH_TEMP, "Transient resolver failure" },
82 { EAI_FAIL, FETCH_RESOLV, "Non-recoverable resolver failure" },
83 { EAI_NONAME, FETCH_RESOLV, "No address record" },
84 { -1, FETCH_UNKNOWN, "Unknown resolver error" }
87 /*** Error-reporting functions ***********************************************/
90 * Map error code to string
92 static struct fetcherr *
93 fetch_finderr(struct fetcherr *p, int e)
95 while (p->num != -1 && p->num != e)
104 fetch_seterr(struct fetcherr *p, int e)
106 p = fetch_finderr(p, e);
107 fetchLastErrCode = p->cat;
108 snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
112 * Set error code according to errno
119 fetchLastErrCode = FETCH_OK;
130 fetchLastErrCode = FETCH_AUTH;
133 case EISDIR: /* XXX */
134 fetchLastErrCode = FETCH_UNAVAIL;
137 fetchLastErrCode = FETCH_MEMORY;
141 fetchLastErrCode = FETCH_TEMP;
144 fetchLastErrCode = FETCH_EXISTS;
147 fetchLastErrCode = FETCH_FULL;
155 fetchLastErrCode = FETCH_NETWORK;
159 fetchLastErrCode = FETCH_ABORT;
162 fetchLastErrCode = FETCH_TIMEOUT;
166 fetchLastErrCode = FETCH_DOWN;
169 fetchLastErrCode = FETCH_UNKNOWN;
171 snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
176 * Emit status message
179 fetch_info(const char *fmt, ...)
184 vfprintf(stderr, fmt, ap);
190 /*** Network-related utility functions ***************************************/
193 * Return the default port for a scheme
196 fetch_default_port(const char *scheme)
200 if ((se = getservbyname(scheme, "tcp")) != NULL)
201 return (ntohs(se->s_port));
202 if (strcasecmp(scheme, SCHEME_FTP) == 0)
203 return (FTP_DEFAULT_PORT);
204 if (strcasecmp(scheme, SCHEME_HTTP) == 0)
205 return (HTTP_DEFAULT_PORT);
210 * Return the default proxy port for a scheme
213 fetch_default_proxy_port(const char *scheme)
215 if (strcasecmp(scheme, SCHEME_FTP) == 0)
216 return (FTP_DEFAULT_PROXY_PORT);
217 if (strcasecmp(scheme, SCHEME_HTTP) == 0)
218 return (HTTP_DEFAULT_PROXY_PORT);
224 * Create a connection for an existing descriptor.
231 /* allocate and fill connection structure */
232 if ((conn = calloc(1, sizeof(*conn))) == NULL)
234 conn->ftp_home = NULL;
235 conn->cache_url = NULL;
236 conn->next_buf = NULL;
244 * Bind a socket to a specific local address
247 fetch_bind(int sd, int af, const char *addr)
249 struct addrinfo hints, *res, *res0;
251 memset(&hints, 0, sizeof(hints));
252 hints.ai_family = af;
253 hints.ai_socktype = SOCK_STREAM;
254 hints.ai_protocol = 0;
255 if (getaddrinfo(addr, NULL, &hints, &res0))
257 for (res = res0; res; res = res->ai_next) {
258 if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
266 * Establish a TCP connection to the specified port on the specified host.
269 fetch_connect(struct url *url, int af, int verbose)
273 const char *bindaddr;
274 struct addrinfo hints, *res, *res0;
278 fetch_info("looking up %s", url->host);
280 /* look up host name and set up socket address structure */
281 snprintf(pbuf, sizeof(pbuf), "%d", url->port);
282 memset(&hints, 0, sizeof(hints));
283 hints.ai_family = af;
284 hints.ai_socktype = SOCK_STREAM;
285 hints.ai_protocol = 0;
286 if ((error = getaddrinfo(url->host, pbuf, &hints, &res0)) != 0) {
290 bindaddr = getenv("FETCH_BIND_ADDRESS");
293 fetch_info("connecting to %s:%d", url->host, url->port);
296 for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
297 if ((sd = socket(res->ai_family, res->ai_socktype,
298 res->ai_protocol)) == -1)
300 if (bindaddr != NULL && *bindaddr != '\0' &&
301 fetch_bind(sd, res->ai_family, bindaddr) != 0) {
302 fetch_info("failed to bind to '%s'", bindaddr);
306 if (connect(sd, res->ai_addr, res->ai_addrlen) == 0)
316 if ((conn = fetch_reopen(sd)) == NULL) {
321 conn->cache_url = fetchCopyURL(url);
326 static conn_t *connection_cache;
327 static int cache_global_limit = 0;
328 static int cache_per_host_limit = 0;
331 * Initialise cache with the given limits.
334 fetchConnectionCacheInit(int global_limit, int per_host_limit)
337 if (global_limit < 0)
338 cache_global_limit = INT_MAX;
339 else if (per_host_limit > global_limit)
340 cache_global_limit = per_host_limit;
342 cache_global_limit = global_limit;
343 if (per_host_limit < 0)
344 cache_per_host_limit = INT_MAX;
346 cache_per_host_limit = per_host_limit;
350 * Flush cache and free all associated resources.
353 fetchConnectionCacheClose(void)
357 while ((conn = connection_cache) != NULL) {
358 connection_cache = conn->next_cached;
359 (*conn->cache_close)(conn);
364 * Check connection cache for an existing entry matching
365 * protocol/host/port/user/password/family.
368 fetch_cache_get(const struct url *url, int af)
370 conn_t *conn, *last_conn = NULL;
372 for (conn = connection_cache; conn; conn = conn->next_cached) {
373 if (conn->cache_url->port == url->port &&
374 strcmp(conn->cache_url->scheme, url->scheme) == 0 &&
375 strcmp(conn->cache_url->host, url->host) == 0 &&
376 strcmp(conn->cache_url->user, url->user) == 0 &&
377 strcmp(conn->cache_url->pwd, url->pwd) == 0 &&
378 (conn->cache_af == AF_UNSPEC || af == AF_UNSPEC ||
379 conn->cache_af == af)) {
380 if (last_conn != NULL)
381 last_conn->next_cached = conn->next_cached;
383 connection_cache = conn->next_cached;
392 * Put the connection back into the cache for reuse.
393 * If the connection is freed due to LRU or if the cache
394 * is explicitly closed, the given callback is called.
397 fetch_cache_put(conn_t *conn, int (*closecb)(conn_t *))
400 int global_count, host_count;
402 if (conn->cache_url == NULL || cache_global_limit == 0) {
407 global_count = host_count = 0;
409 for (iter = connection_cache; iter;
410 last = iter, iter = iter->next_cached) {
412 if (strcmp(conn->cache_url->host, iter->cache_url->host) == 0)
414 if (global_count < cache_global_limit &&
415 host_count < cache_per_host_limit)
419 last->next_cached = iter->next_cached;
421 connection_cache = iter->next_cached;
422 (*iter->cache_close)(iter);
425 conn->cache_close = closecb;
426 conn->next_cached = connection_cache;
427 connection_cache = conn;
431 * Enable SSL on a connection.
434 fetch_ssl(conn_t *conn, int verbose)
438 /* Init the SSL library and context */
439 if (!SSL_library_init()){
440 fprintf(stderr, "SSL library init failed\n");
444 SSL_load_error_strings();
446 conn->ssl_meth = SSLv23_client_method();
447 conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
448 SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
450 conn->ssl = SSL_new(conn->ssl_ctx);
451 if (conn->ssl == NULL){
452 fprintf(stderr, "SSL context creation failed\n");
455 SSL_set_fd(conn->ssl, conn->sd);
456 if (SSL_connect(conn->ssl) == -1){
457 ERR_print_errors_fp(stderr);
465 fprintf(stderr, "SSL connection established using %s\n",
466 SSL_get_cipher(conn->ssl));
467 conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
468 name = X509_get_subject_name(conn->ssl_cert);
469 str = X509_NAME_oneline(name, 0, 0);
470 printf("Certificate subject: %s\n", str);
472 name = X509_get_issuer_name(conn->ssl_cert);
473 str = X509_NAME_oneline(name, 0, 0);
474 printf("Certificate issuer: %s\n", str);
482 fprintf(stderr, "SSL support disabled\n");
489 * Read a character from a connection w/ timeout
492 fetch_read(conn_t *conn, char *buf, size_t len)
494 struct timeval now, timeout, waittv;
502 if (conn->next_len != 0) {
503 if (conn->next_len < len)
504 len = conn->next_len;
505 memmove(buf, conn->next_buf, len);
506 conn->next_len -= len;
507 conn->next_buf += len;
513 gettimeofday(&timeout, NULL);
514 timeout.tv_sec += fetchTimeout;
518 while (fetchTimeout && !FD_ISSET(conn->sd, &readfds)) {
519 FD_SET(conn->sd, &readfds);
520 gettimeofday(&now, NULL);
521 waittv.tv_sec = timeout.tv_sec - now.tv_sec;
522 waittv.tv_usec = timeout.tv_usec - now.tv_usec;
523 if (waittv.tv_usec < 0) {
524 waittv.tv_usec += 1000000;
527 if (waittv.tv_sec < 0) {
533 r = select(conn->sd + 1, &readfds, NULL, NULL, &waittv);
535 if (errno == EINTR && fetchRestartCalls)
542 if (conn->ssl != NULL)
543 rlen = SSL_read(conn->ssl, buf, len);
546 rlen = read(conn->sd, buf, len);
550 if (errno != EINTR || !fetchRestartCalls)
558 * Read a line of text from a connection w/ timeout
560 #define MIN_BUF_SIZE 1024
563 fetch_getln(conn_t *conn)
569 if (conn->buf == NULL) {
570 if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
574 conn->bufsize = MIN_BUF_SIZE;
582 * conn->bufsize != conn->buflen at this point,
583 * so the buffer can be NUL-terminated below for
584 * the case of len == 0.
586 len = fetch_read(conn, conn->buf + conn->buflen,
587 conn->bufsize - conn->buflen);
592 next = memchr(conn->buf + conn->buflen, '\n', len);
594 if (conn->buflen == conn->bufsize && next == NULL) {
596 tmpsize = conn->bufsize * 2;
597 if (tmpsize < conn->bufsize) {
601 if ((tmp = realloc(tmp, tmpsize)) == NULL) {
606 conn->bufsize = tmpsize;
608 } while (next == NULL);
612 conn->next_buf = next + 1;
613 conn->next_len = conn->buflen - (conn->next_buf - conn->buf);
614 conn->buflen = next - conn->buf;
616 conn->buf[conn->buflen] = '\0';
623 * Write a vector to a connection w/ timeout
624 * Note: can modify the iovec.
627 fetch_write(conn_t *conn, const void *buf, size_t len)
629 struct timeval now, timeout, waittv;
634 static int killed_sigpipe;
638 if (!killed_sigpipe) {
639 signal(SIGPIPE, SIG_IGN);
647 gettimeofday(&timeout, NULL);
648 timeout.tv_sec += fetchTimeout;
653 while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
654 FD_SET(conn->sd, &writefds);
655 gettimeofday(&now, NULL);
656 waittv.tv_sec = timeout.tv_sec - now.tv_sec;
657 waittv.tv_usec = timeout.tv_usec - now.tv_usec;
658 if (waittv.tv_usec < 0) {
659 waittv.tv_usec += 1000000;
662 if (waittv.tv_sec < 0) {
668 r = select(conn->sd + 1, NULL, &writefds, NULL, &waittv);
670 if (errno == EINTR && fetchRestartCalls)
677 if (conn->ssl != NULL)
678 wlen = SSL_write(conn->ssl, buf, len);
682 wlen = send(conn->sd, buf, len, 0);
684 wlen = send(conn->sd, buf, len, MSG_NOSIGNAL);
687 /* we consider a short write a failure */
693 if (errno == EINTR && fetchRestartCalls)
698 buf = (const char *)buf + wlen;
709 fetch_close(conn_t *conn)
713 ret = close(conn->sd);
715 fetchFreeURL(conn->cache_url);
716 free(conn->ftp_home);
723 /*** Directory-related utility functions *************************************/
726 fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
731 size_t base_doc_len, name_len, i;
734 if (strchr(name, '/') != NULL ||
735 strcmp(name, "..") == 0 ||
736 strcmp(name, ".") == 0)
739 if (strcmp(base->doc, "/") == 0)
742 base_doc_len = strlen(base->doc);
745 for (i = 0; name[i] != '\0'; ++i) {
746 if ((!pre_quoted && name[i] == '%') ||
747 !fetch_urlpath_safe(name[i]))
753 tmp_name = malloc( base_doc_len + name_len + 1);
754 if (tmp_name == NULL) {
760 if (ue->length + 1 >= ue->alloc_size) {
761 tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp));
768 ue->alloc_size = ue->alloc_size * 2 + 1;
772 tmp = ue->urls + ue->length;
773 strcpy(tmp->scheme, base->scheme);
774 strcpy(tmp->user, base->user);
775 strcpy(tmp->pwd, base->pwd);
776 strcpy(tmp->host, base->host);
777 tmp->port = base->port;
779 memcpy(tmp->doc, base->doc, base_doc_len);
780 tmp->doc[base_doc_len] = '/';
782 for (i = base_doc_len + 1; *name != '\0'; ++name) {
783 if ((!pre_quoted && *name == '%') ||
784 !fetch_urlpath_safe(*name)) {
786 c = (unsigned char)*name / 16;
788 tmp->doc[i++] = '0' + c;
790 tmp->doc[i++] = 'a' - 10 + c;
791 c = (unsigned char)*name % 16;
793 tmp->doc[i++] = '0' + c;
795 tmp->doc[i++] = 'a' - 10 + c;
797 tmp->doc[i++] = *name;
804 tmp->last_modified = -1;
812 fetchInitURLList(struct url_list *ue)
814 ue->length = ue->alloc_size = 0;
819 fetchAppendURLList(struct url_list *dst, const struct url_list *src)
823 len = dst->length + src->length;
824 if (len > dst->alloc_size) {
827 tmp = realloc(dst->urls, len * sizeof(*tmp));
833 dst->alloc_size = len;
837 for (i = 0, j = dst->length; i < src->length; ++i, ++j) {
838 dst->urls[j] = src->urls[i];
839 dst->urls[j].doc = strdup(src->urls[i].doc);
840 if (dst->urls[j].doc == NULL) {
842 free(dst->urls[j].doc);
853 fetchFreeURLList(struct url_list *ue)
857 for (i = 0; i < ue->length; ++i)
858 free(ue->urls[i].doc);
860 ue->length = ue->alloc_size = 0;
864 /*** Authentication-related utility functions ********************************/
867 fetch_read_word(FILE *f)
869 static char word[1024];
871 if (fscanf(f, " %1023s ", word) != 1)
877 * Get authentication data for a URL from .netrc
880 fetch_netrc_auth(struct url *url)
887 if ((p = getenv("NETRC")) != NULL) {
888 if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
889 fetch_info("$NETRC specifies a file name "
890 "longer than PATH_MAX");
894 if ((p = getenv("HOME")) != NULL) {
897 if ((pwd = getpwuid(getuid())) == NULL ||
898 (p = pwd->pw_dir) == NULL)
901 if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
905 if ((f = fopen(fn, "r")) == NULL)
907 while ((word = fetch_read_word(f)) != NULL) {
908 if (strcmp(word, "default") == 0)
910 if (strcmp(word, "machine") == 0 &&
911 (word = fetch_read_word(f)) != NULL &&
912 strcasecmp(word, url->host) == 0) {
918 while ((word = fetch_read_word(f)) != NULL) {
919 if (strcmp(word, "login") == 0) {
920 if ((word = fetch_read_word(f)) == NULL)
922 if (snprintf(url->user, sizeof(url->user),
923 "%s", word) > (int)sizeof(url->user)) {
924 fetch_info("login name in .netrc is too long");
927 } else if (strcmp(word, "password") == 0) {
928 if ((word = fetch_read_word(f)) == NULL)
930 if (snprintf(url->pwd, sizeof(url->pwd),
931 "%s", word) > (int)sizeof(url->pwd)) {
932 fetch_info("password in .netrc is too long");
935 } else if (strcmp(word, "account") == 0) {
936 if ((word = fetch_read_word(f)) == NULL)
938 /* XXX not supported! */
951 * The no_proxy environment variable specifies a set of domains for
952 * which the proxy should not be consulted; the contents is a comma-,
953 * or space-separated list of domain names. A single asterisk will
954 * override all proxy variables and no transactions will be proxied
955 * (for compatability with lynx and curl, see the discussion at
956 * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
959 fetch_no_proxy_match(const char *host)
961 const char *no_proxy, *p, *q;
964 if ((no_proxy = getenv("NO_PROXY")) == NULL &&
965 (no_proxy = getenv("no_proxy")) == NULL)
968 /* asterisk matches any hostname */
969 if (strcmp(no_proxy, "*") == 0)
972 h_len = strlen(host);
975 /* position p at the beginning of a domain suffix */
976 while (*p == ',' || isspace((unsigned char)*p))
979 /* position q at the first separator character */
981 if (*q == ',' || isspace((unsigned char)*q))
985 if (d_len > 0 && h_len > d_len &&
986 strncasecmp(host + h_len - d_len,
988 /* domain name matches */
1000 ssize_t (*io_read)(void *, void *, size_t);
1001 ssize_t (*io_write)(void *, const void *, size_t);
1002 void (*io_close)(void *);
1006 fetchIO_close(fetchIO *f)
1008 if (f->io_close != NULL)
1009 (*f->io_close)(f->io_cookie);
1015 fetchIO_unopen(void *io_cookie, ssize_t (*io_read)(void *, void *, size_t),
1016 ssize_t (*io_write)(void *, const void *, size_t),
1017 void (*io_close)(void *))
1021 f = malloc(sizeof(*f));
1025 f->io_cookie = io_cookie;
1026 f->io_read = io_read;
1027 f->io_write = io_write;
1028 f->io_close = io_close;
1034 fetchIO_read(fetchIO *f, void *buf, size_t len)
1036 if (f->io_read == NULL)
1038 return (*f->io_read)(f->io_cookie, buf, len);
1042 fetchIO_write(fetchIO *f, const void *buf, size_t len)
1044 if (f->io_read == NULL)
1046 return (*f->io_write)(f->io_cookie, buf, len);