Imported Upstream version 7.59.0upstream/7.59.0

Change-Id: I06221d49da39082f95030ab57617a1e23fbda58b

1 files changed, 270 insertions, 177 deletions

diff --git a/lib/transfer.c b/lib/transfer.c
index a577bf724..fd9af3155 100644
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -73,12 +73,39 @@
 #include "connect.h"
 #include "non-ascii.h"
 #include "http2.h"
+#include "mime.h"
+#include "strcase.h"
 
 /* The last 3 #include files should be in this order */
 #include "curl_printf.h"
 #include "curl_memory.h"
 #include "memdebug.h"
 
+#if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_SMTP) || \
+    !defined(CURL_DISABLE_IMAP)
+/*
+ * checkheaders() checks the linked list of custom headers for a
+ * particular header (prefix). Provide the prefix without colon!
+ *
+ * Returns a pointer to the first matching header or NULL if none matched.
+ */
+char *Curl_checkheaders(const struct connectdata *conn,
+                        const char *thisheader)
+{
+  struct curl_slist *head;
+  size_t thislen = strlen(thisheader);
+  struct Curl_easy *data = conn->data;
+
+  for(head = data->set.headers; head; head = head->next) {
+    if(strncasecompare(head->data, thisheader, thislen) &&
+       Curl_headersep(head->data[thislen]) )
+      return head->data;
+  }
+
+  return NULL;
+}
+#endif
+
 /*
  * This function will call the read callback to fill our buffer with data
  * to upload.
@@ -109,15 +136,18 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
 
   /* this function returns a size_t, so we typecast to int to prevent warnings
      with picky compilers */
+  Curl_set_in_callback(data, true);
   nread = (int)data->state.fread_func(data->req.upload_fromhere, 1,
                                       buffersize, data->state.in);
+  Curl_set_in_callback(data, false);
 
   if(nread == CURL_READFUNC_ABORT) {
     failf(data, "operation aborted by callback");
     *nreadp = 0;
     return CURLE_ABORTED_BY_CALLBACK;
   }
-  else if(nread == CURL_READFUNC_PAUSE) {
+  if(nread == CURL_READFUNC_PAUSE) {
+    struct SingleRequest *k = &data->req;
 
     if(conn->handler->flags & PROTOPT_NONETWORK) {
       /* protocols that work without network cannot be paused. This is
@@ -126,16 +156,15 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
       failf(data, "Read callback asked for PAUSE when not supported!");
       return CURLE_READ_ERROR;
     }
-    else {
-      struct SingleRequest *k = &data->req;
-      /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
-      k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */
-      if(data->req.upload_chunky) {
+
+    /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
+    k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */
+    if(data->req.upload_chunky) {
         /* Back out the preallocation done above */
-        data->req.upload_fromhere -= (8 + 2);
-      }
-      *nreadp = 0;
+      data->req.upload_fromhere -= (8 + 2);
     }
+    *nreadp = 0;
+
     return CURLE_OK; /* nothing was read */
   }
   else if((size_t)nread > buffersize) {
@@ -195,27 +224,30 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
            strlen(endofline_network));
 
 #ifdef CURL_DOES_CONVERSIONS
-    CURLcode result;
-    int length;
-    if(data->set.prefer_ascii) {
-      /* translate the protocol and data */
-      length = nread;
-    }
-    else {
-      /* just translate the protocol portion */
-      length = strlen(hexbuffer);
+    {
+      CURLcode result;
+      int length;
+      if(data->set.prefer_ascii)
+        /* translate the protocol and data */
+        length = nread;
+      else
+        /* just translate the protocol portion */
+        length = (int)strlen(hexbuffer);
+      result = Curl_convert_to_network(data, data->req.upload_fromhere,
+                                       length);
+      /* Curl_convert_to_network calls failf if unsuccessful */
+      if(result)
+        return result;
     }
-    result = Curl_convert_to_network(data, data->req.upload_fromhere, length);
-    /* Curl_convert_to_network calls failf if unsuccessful */
-    if(result)
-      return result;
 #endif /* CURL_DOES_CONVERSIONS */
 
-    if((nread - hexlen) == 0)
+    if((nread - hexlen) == 0) {
       /* mark this as done once this chunk is transferred */
       data->req.upload_done = TRUE;
+      infof(data, "Signaling end of chunked upload via terminating chunk.\n");
+    }
 
-    nread+=(int)strlen(endofline_native); /* for the added end of line */
+    nread += (int)strlen(endofline_native); /* for the added end of line */
   }
 #ifdef CURL_DOES_CONVERSIONS
   else if((data->set.prefer_ascii) && (!sending_http_headers)) {
@@ -241,6 +273,7 @@ CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
 CURLcode Curl_readrewind(struct connectdata *conn)
 {
   struct Curl_easy *data = conn->data;
+  curl_mimepart *mimepart = &data->set.mimepost;
 
   conn->bits.rewindaftersend = FALSE; /* we rewind now */
 
@@ -253,14 +286,28 @@ CURLcode Curl_readrewind(struct connectdata *conn)
   /* We have sent away data. If not using CURLOPT_POSTFIELDS or
      CURLOPT_HTTPPOST, call app to rewind
   */
-  if(data->set.postfields ||
-     (data->set.httpreq == HTTPREQ_POST_FORM))
+  if(conn->handler->protocol & PROTO_FAMILY_HTTP) {
+    struct HTTP *http = data->req.protop;
+
+    if(http->sendit)
+      mimepart = http->sendit;
+  }
+  if(data->set.postfields)
     ; /* do nothing */
+  else if(data->set.httpreq == HTTPREQ_POST_MIME ||
+          data->set.httpreq == HTTPREQ_POST_FORM) {
+    if(Curl_mime_rewind(mimepart)) {
+      failf(data, "Cannot rewind mime/post data");
+      return CURLE_SEND_FAIL_REWIND;
+    }
+  }
   else {
     if(data->set.seek_func) {
       int err;
 
+      Curl_set_in_callback(data, true);
       err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
+      Curl_set_in_callback(data, false);
       if(err) {
         failf(data, "seek callback returned error %d", (int)err);
         return CURLE_SEND_FAIL_REWIND;
@@ -269,8 +316,10 @@ CURLcode Curl_readrewind(struct connectdata *conn)
     else if(data->set.ioctl_func) {
       curlioerr err;
 
+      Curl_set_in_callback(data, true);
       err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
                                    data->set.ioctl_client);
+      Curl_set_in_callback(data, false);
       infof(data, "the ioctl callback returned %d\n", (int)err);
 
       if(err) {
@@ -405,8 +454,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
   /* This is where we loop until we have read everything there is to
      read or we get a CURLE_AGAIN */
   do {
-    size_t buffersize = data->set.buffer_size?
-      data->set.buffer_size : BUFSIZE;
+    size_t buffersize = data->set.buffer_size;
     size_t bytestoread = buffersize;
 
     if(
@@ -451,7 +499,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
       Curl_pgrsTime(data, TIMER_STARTTRANSFER);
       if(k->exp100 > EXP100_SEND_DATA)
         /* set time stamp to compare with when waiting for the 100 */
-        k->start100 = Curl_tvnow();
+        k->start100 = Curl_now();
     }
 
     *didwhat |= KEEP_RECV;
@@ -561,7 +609,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
             infof(data, "Ignoring the response-body\n");
           }
           if(data->state.resume_from && !k->content_range &&
-             (data->set.httpreq==HTTPREQ_GET) &&
+             (data->set.httpreq == HTTPREQ_GET) &&
              !k->ignorebody) {
 
             if(k->size == data->state.resume_from) {
@@ -642,7 +690,7 @@ static CURLcode readwrite_data(struct Curl_easy *data,
           failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res));
           return CURLE_RECV_ERROR;
         }
-        else if(CHUNKE_STOP == res) {
+        if(CHUNKE_STOP == res) {
           size_t dataleft;
           /* we're done reading chunks! */
           k->keepon &= ~KEEP_RECV; /* read no more */
@@ -681,8 +729,6 @@ static CURLcode readwrite_data(struct Curl_easy *data,
         excess = (size_t)(k->bytecount + nread - k->maxdownload);
         if(excess > 0 && !k->ignorebody) {
           if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
-            /* The 'excess' amount below can't be more than BUFSIZE which
-               always will fit in a size_t */
             infof(data,
                   "Rewinding stream by : %zu"
                   " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T
@@ -740,48 +786,19 @@ static CURLcode readwrite_data(struct Curl_easy *data,
              in http_chunks.c.
              Make sure that ALL_CONTENT_ENCODINGS contains all the
              encodings handled here. */
-#ifdef HAVE_LIBZ
-          switch(conn->data->set.http_ce_skip ?
-                 IDENTITY : k->auto_decoding) {
-          case IDENTITY:
-#endif
-            /* This is the default when the server sends no
-               Content-Encoding header. See Curl_readwrite_init; the
-               memset() call initializes k->auto_decoding to zero. */
+          if(conn->data->set.http_ce_skip || !k->writer_stack) {
             if(!k->ignorebody) {
-
 #ifndef CURL_DISABLE_POP3
-              if(conn->handler->protocol&PROTO_FAMILY_POP3)
+              if(conn->handler->protocol & PROTO_FAMILY_POP3)
                 result = Curl_pop3_write(conn, k->str, nread);
               else
 #endif /* CURL_DISABLE_POP3 */
-
                 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str,
                                            nread);
             }
-#ifdef HAVE_LIBZ
-            break;
-
-          case DEFLATE:
-            /* Assume CLIENTWRITE_BODY; headers are not encoded. */
-            if(!k->ignorebody)
-              result = Curl_unencode_deflate_write(conn, k, nread);
-            break;
-
-          case GZIP:
-            /* Assume CLIENTWRITE_BODY; headers are not encoded. */
-            if(!k->ignorebody)
-              result = Curl_unencode_gzip_write(conn, k, nread);
-            break;
-
-          default:
-            failf(data, "Unrecognized content encoding type. "
-                  "libcurl understands `identity', `deflate' and `gzip' "
-                  "content encodings.");
-            result = CURLE_BAD_CONTENT_ENCODING;
-            break;
           }
-#endif
+          else
+            result = Curl_unencode_write(conn, k->writer_stack, k->str, nread);
         }
         k->badheader = HEADER_NORMAL; /* taken care of now */
 
@@ -791,10 +808,15 @@ static CURLcode readwrite_data(struct Curl_easy *data,
 
     } /* if(!header and data to read) */
 
-    if(conn->handler->readwrite &&
-       (excess > 0 && !conn->bits.stream_was_rewound)) {
+    if(conn->handler->readwrite && excess && !conn->bits.stream_was_rewound) {
       /* Parse the excess data */
       k->str += nread;
+
+      if(&k->str[excess] > &k->buf[data->set.buffer_size]) {
+        /* the excess amount was too excessive(!), make sure
+           it doesn't read out of buffer */
+        excess = &k->buf[data->set.buffer_size] - k->str;
+      }
       nread = (ssize_t)excess;
 
       result = conn->handler->readwrite(data, conn, &nread, &readmore);
@@ -853,7 +875,6 @@ static CURLcode done_sending(struct connectdata *conn,
  */
 static CURLcode readwrite_upload(struct Curl_easy *data,
                                  struct connectdata *conn,
-                                 struct SingleRequest *k,
                                  int *didwhat)
 {
   ssize_t i, si;
@@ -861,6 +882,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
   CURLcode result;
   ssize_t nread; /* number of bytes read */
   bool sending_http_headers = FALSE;
+  struct SingleRequest *k = &data->req;
 
   if((k->bytecount == 0) && (k->writebytecount == 0))
     Curl_pgrsTime(data, TIMER_STARTTRANSFER);
@@ -871,15 +893,15 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
 
     /* only read more data if there's no upload data already
        present in the upload buffer */
-    if(0 == data->req.upload_present) {
+    if(0 == k->upload_present) {
       /* init the "upload from here" pointer */
-      data->req.upload_fromhere = k->uploadbuf;
+      k->upload_fromhere = data->state.uploadbuffer;
 
       if(!k->upload_done) {
         /* HTTP pollution, this should be written nicer to become more
            protocol agnostic. */
         int fillcount;
-        struct HTTP *http = data->req.protop;
+        struct HTTP *http = k->protop;
 
         if((k->exp100 == EXP100_SENDING_REQUEST) &&
            (http->sending == HTTPSEND_BODY)) {
@@ -888,11 +910,11 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
              go into the Expect: 100 state and await such a header */
           k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */
           k->keepon &= ~KEEP_SEND;         /* disable writing */
-          k->start100 = Curl_tvnow();       /* timeout count starts now */
+          k->start100 = Curl_now();       /* timeout count starts now */
           *didwhat &= ~KEEP_SEND;  /* we didn't write anything actually */
 
           /* set a timeout for the multi interface */
-          Curl_expire(data, data->set.expect_100_timeout);
+          Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
           break;
         }
 
@@ -905,7 +927,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
             sending_http_headers = FALSE;
         }
 
-        result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount);
+        result = Curl_fillreadbuffer(conn, UPLOAD_BUFSIZE, &fillcount);
         if(result)
           return result;
 
@@ -918,7 +940,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
         /* this is a paused transfer */
         break;
       }
-      else if(nread<=0) {
+      if(nread <= 0) {
         result = done_sending(conn, k);
         if(result)
           return result;
@@ -926,7 +948,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
       }
 
       /* store number of bytes available for upload */
-      data->req.upload_present = nread;
+      k->upload_present = nread;
 
       /* convert LF to CRLF if so asked */
       if((!sending_http_headers) && (
@@ -937,7 +959,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
          (data->set.crlf))) {
         /* Do we need to allocate a scratch buffer? */
         if(!data->state.scratch) {
-          data->state.scratch = malloc(2 * BUFSIZE);
+          data->state.scratch = malloc(2 * data->set.buffer_size);
           if(!data->state.scratch) {
             failf(data, "Failed to alloc scratch buffer!");
 
@@ -952,7 +974,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
          * must be used instead of the escape sequences \r & \n.
          */
         for(i = 0, si = 0; i < nread; i++, si++) {
-          if(data->req.upload_fromhere[i] == 0x0a) {
+          if(k->upload_fromhere[i] == 0x0a) {
             data->state.scratch[si++] = 0x0d;
             data->state.scratch[si] = 0x0a;
             if(!data->set.crlf) {
@@ -963,7 +985,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
             }
           }
           else
-            data->state.scratch[si] = data->req.upload_fromhere[i];
+            data->state.scratch[si] = k->upload_fromhere[i];
         }
 
         if(si != nread) {
@@ -972,10 +994,10 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
           nread = si;
 
           /* upload from the new (replaced) buffer instead */
-          data->req.upload_fromhere = data->state.scratch;
+          k->upload_fromhere = data->state.scratch;
 
           /* set the new amount too */
-          data->req.upload_present = nread;
+          k->upload_present = nread;
         }
       }
 
@@ -986,7 +1008,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
           return result;
       }
 #endif /* CURL_DISABLE_SMTP */
-    } /* if 0 == data->req.upload_present */
+    } /* if 0 == k->upload_present */
     else {
       /* We have a partial buffer left from a previous "round". Use
          that instead of reading more data */
@@ -994,41 +1016,42 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
 
     /* write to socket (send away data) */
     result = Curl_write(conn,
-                        conn->writesockfd,     /* socket to send to */
-                        data->req.upload_fromhere, /* buffer pointer */
-                        data->req.upload_present,  /* buffer size */
-                        &bytes_written);           /* actually sent */
+                        conn->writesockfd,  /* socket to send to */
+                        k->upload_fromhere, /* buffer pointer */
+                        k->upload_present,  /* buffer size */
+                        &bytes_written);    /* actually sent */
 
     if(result)
       return result;
 
     if(data->set.verbose)
       /* show the data before we change the pointer upload_fromhere */
-      Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere,
+      Curl_debug(data, CURLINFO_DATA_OUT, k->upload_fromhere,
                  (size_t)bytes_written, conn);
 
     k->writebytecount += bytes_written;
 
-    if(k->writebytecount == data->state.infilesize) {
+    if((!k->upload_chunky || k->forbidchunk) &&
+       (k->writebytecount == data->state.infilesize)) {
       /* we have sent all data we were supposed to */
       k->upload_done = TRUE;
       infof(data, "We are completely uploaded and fine\n");
     }
 
-    if(data->req.upload_present != bytes_written) {
+    if(k->upload_present != bytes_written) {
       /* we only wrote a part of the buffer (if anything), deal with it! */
 
       /* store the amount of bytes left in the buffer to write */
-      data->req.upload_present -= bytes_written;
+      k->upload_present -= bytes_written;
 
       /* advance the pointer where to find the buffer when the next send
          is to happen */
-      data->req.upload_fromhere += bytes_written;
+      k->upload_fromhere += bytes_written;
     }
     else {
       /* we've uploaded that buffer now */
-      data->req.upload_fromhere = k->uploadbuf;
-      data->req.upload_present = 0; /* no more bytes left */
+      k->upload_fromhere = data->state.uploadbuffer;
+      k->upload_present = 0; /* no more bytes left */
 
       if(k->upload_done) {
         result = done_sending(conn, k);
@@ -1058,7 +1081,7 @@ CURLcode Curl_readwrite(struct connectdata *conn,
 {
   struct SingleRequest *k = &data->req;
   CURLcode result;
-  int didwhat=0;
+  int didwhat = 0;
 
   curl_socket_t fd_read;
   curl_socket_t fd_write;
@@ -1108,12 +1131,12 @@ CURLcode Curl_readwrite(struct connectdata *conn,
   if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) {
     /* write */
 
-    result = readwrite_upload(data, conn, k, &didwhat);
+    result = readwrite_upload(data, conn, &didwhat);
     if(result)
       return result;
   }
 
-  k->now = Curl_tvnow();
+  k->now = Curl_now();
   if(didwhat) {
     /* Update read/write counters */
     if(k->bytecountp)
@@ -1137,11 +1160,12 @@ CURLcode Curl_readwrite(struct connectdata *conn,
 
       */
 
-      time_t ms = Curl_tvdiff(k->now, k->start100);
+      timediff_t ms = Curl_timediff(k->now, k->start100);
       if(ms >= data->set.expect_100_timeout) {
         /* we've waited long enough, continue anyway */
         k->exp100 = EXP100_SEND_DATA;
         k->keepon |= KEEP_SEND;
+        Curl_expire_done(data, EXPIRE_100_TIMEOUT);
         infof(data, "Done waiting for 100-continue\n");
       }
     }
@@ -1160,13 +1184,14 @@ CURLcode Curl_readwrite(struct connectdata *conn,
         failf(data, "Operation timed out after %ld milliseconds with %"
               CURL_FORMAT_CURL_OFF_T " out of %"
               CURL_FORMAT_CURL_OFF_T " bytes received",
-              Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount,
-              k->size);
+              Curl_timediff(k->now, data->progress.t_startsingle),
+              k->bytecount, k->size);
       }
       else {
         failf(data, "Operation timed out after %ld milliseconds with %"
               CURL_FORMAT_CURL_OFF_T " bytes received",
-              Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount);
+              Curl_timediff(k->now, data->progress.t_startsingle),
+              k->bytecount);
       }
       return CURLE_OPERATION_TIMEDOUT;
     }
@@ -1186,15 +1211,13 @@ CURLcode Curl_readwrite(struct connectdata *conn,
        */
        (k->bytecount != (k->size + data->state.crlf_conversions)) &&
 #endif /* CURL_DO_LINEEND_CONV */
-       !data->req.newurl) {
+       !k->newurl) {
       failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T
-            " bytes remaining to read",
-            k->size - k->bytecount);
+            " bytes remaining to read", k->size - k->bytecount);
       return CURLE_PARTIAL_FILE;
     }
-    else if(!(data->set.opt_no_body) &&
-            k->chunk &&
-            (conn->chunk.state != CHUNK_STOP)) {
+    if(!(data->set.opt_no_body) && k->chunk &&
+       (conn->chunk.state != CHUNK_STOP)) {
       /*
        * In chunked mode, return an error if the connection is closed prior to
        * the empty (terminating) chunk is read.
@@ -1292,6 +1315,13 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
     failf(data, "No URL set!");
     return CURLE_URL_MALFORMAT;
   }
+  /* since the URL may have been redirected in a previous use of this handle */
+  if(data->change.url_alloc) {
+    /* the already set URL is allocated, free it first! */
+    Curl_safefree(data->change.url);
+    data->change.url_alloc = FALSE;
+  }
+  data->change.url = data->set.str[STRING_SET_URL];
 
   /* Init the SSL session ID cache here. We do it here since we want to do it
      after the *_setopt() calls (that could specify the size of the cache) but
@@ -1300,7 +1330,8 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
   if(result)
     return result;
 
-  data->set.followlocation=0; /* reset the location-follow counter */
+  data->state.wildcardmatch = data->set.wildcard_enabled;
+  data->set.followlocation = 0; /* reset the location-follow counter */
   data->state.this_is_a_follow = FALSE; /* reset this */
   data->state.errorbuf = FALSE; /* no error has occurred */
   data->state.httpversion = 0; /* don't assume any particular server version */
@@ -1313,8 +1344,11 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
 
   if(data->set.httpreq == HTTPREQ_PUT)
     data->state.infilesize = data->set.filesize;
-  else
+  else {
     data->state.infilesize = data->set.postfieldsize;
+    if(data->set.postfields && (data->state.infilesize == -1))
+      data->state.infilesize = (curl_off_t)strlen(data->set.postfields);
+  }
 
   /* If there is a list of cookie files to read, do it now! */
   if(data->change.cookielist)
@@ -1339,14 +1373,14 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
 #endif
 
     Curl_initinfo(data); /* reset session-specific information "variables" */
-    Curl_pgrsResetTimesSizes(data);
+    Curl_pgrsResetTransferSizes(data);
     Curl_pgrsStartNow(data);
 
     if(data->set.timeout)
-      Curl_expire(data, data->set.timeout);
+      Curl_expire(data, data->set.timeout, EXPIRE_TIMEOUT);
 
     if(data->set.connecttimeout)
-      Curl_expire(data, data->set.connecttimeout);
+      Curl_expire(data, data->set.connecttimeout, EXPIRE_CONNECTTIMEOUT);
 
     /* In case the handle is re-used and an authentication method was picked
        in the session we need to make sure we only use the one(s) we now
@@ -1354,15 +1388,14 @@ CURLcode Curl_pretransfer(struct Curl_easy *data)
     data->state.authhost.picked &= data->state.authhost.want;
     data->state.authproxy.picked &= data->state.authproxy.want;
 
-    if(data->set.wildcardmatch) {
+    if(data->state.wildcardmatch) {
       struct WildcardData *wc = &data->wildcard;
-      if(!wc->filelist) {
+      if(wc->state < CURLWC_INIT) {
         result = Curl_wildcard_init(wc); /* init wildcard structures */
         if(result)
           return CURLE_OUT_OF_MEMORY;
       }
     }
-
   }
 
   return result;
@@ -1386,19 +1419,59 @@ CURLcode Curl_posttransfer(struct Curl_easy *data)
 
 #ifndef CURL_DISABLE_HTTP
 /*
+ * Find the separator at the end of the host name, or the '?' in cases like
+ * http://www.url.com?id=2380
+ */
+static const char *find_host_sep(const char *url)
+{
+  const char *sep;
+  const char *query;
+
+  /* Find the start of the hostname */
+  sep = strstr(url, "//");
+  if(!sep)
+    sep = url;
+  else
+    sep += 2;
+
+  query = strchr(sep, '?');
+  sep = strchr(sep, '/');
+
+  if(!sep)
+    sep = url + strlen(url);
+
+  if(!query)
+    query = url + strlen(url);
+
+  return sep < query ? sep : query;
+}
+
+/*
  * strlen_url() returns the length of the given URL if the spaces within the
  * URL were properly URL encoded.
+ * URL encoding should be skipped for host names, otherwise IDN resolution
+ * will fail.
  */
-static size_t strlen_url(const char *url)
+static size_t strlen_url(const char *url, bool relative)
 {
   const unsigned char *ptr;
-  size_t newlen=0;
-  bool left=TRUE; /* left side of the ? */
+  size_t newlen = 0;
+  bool left = TRUE; /* left side of the ? */
+  const unsigned char *host_sep = (const unsigned char *) url;
+
+  if(!relative)
+    host_sep = (const unsigned char *) find_host_sep(url);
+
+  for(ptr = (unsigned char *)url; *ptr; ptr++) {
+
+    if(ptr < host_sep) {
+      ++newlen;
+      continue;
+    }
 
-  for(ptr=(unsigned char *)url; *ptr; ptr++) {
     switch(*ptr) {
     case '?':
-      left=FALSE;
+      left = FALSE;
       /* fall through */
     default:
       if(*ptr >= 0x80)
@@ -1407,7 +1480,7 @@ static size_t strlen_url(const char *url)
       break;
     case ' ':
       if(left)
-        newlen+=3;
+        newlen += 3;
       else
         newlen++;
       break;
@@ -1418,19 +1491,32 @@ static size_t strlen_url(const char *url)
 
 /* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in
  * the source URL accordingly.
+ * URL encoding should be skipped for host names, otherwise IDN resolution
+ * will fail.
  */
-static void strcpy_url(char *output, const char *url)
+static void strcpy_url(char *output, const char *url, bool relative)
 {
   /* we must add this with whitespace-replacing */
-  bool left=TRUE;
+  bool left = TRUE;
   const unsigned char *iptr;
   char *optr = output;
+  const unsigned char *host_sep = (const unsigned char *) url;
+
+  if(!relative)
+    host_sep = (const unsigned char *) find_host_sep(url);
+
   for(iptr = (unsigned char *)url;    /* read from here */
       *iptr;         /* until zero byte */
       iptr++) {
+
+    if(iptr < host_sep) {
+      *optr++ = *iptr;
+      continue;
+    }
+
     switch(*iptr) {
     case '?':
-      left=FALSE;
+      left = FALSE;
       /* fall through */
     default:
       if(*iptr >= 0x80) {
@@ -1451,7 +1537,7 @@ static void strcpy_url(char *output, const char *url)
       break;
     }
   }
-  *optr=0; /* zero terminate output buffer */
+  *optr = 0; /* zero terminate output buffer */
 
 }
 
@@ -1483,32 +1569,33 @@ static char *concat_url(const char *base, const char *relurl)
   char *protsep;
   char *pathsep;
   size_t newlen;
+  bool host_changed = FALSE;
 
   const char *useurl = relurl;
   size_t urllen;
 
   /* we must make our own copy of the URL to play with, as it may
      point to read-only data */
-  char *url_clone=strdup(base);
+  char *url_clone = strdup(base);
 
   if(!url_clone)
     return NULL; /* skip out of this NOW */
 
   /* protsep points to the start of the host name */
-  protsep=strstr(url_clone, "//");
+  protsep = strstr(url_clone, "//");
   if(!protsep)
-    protsep=url_clone;
+    protsep = url_clone;
   else
-    protsep+=2; /* pass the slashes */
+    protsep += 2; /* pass the slashes */
 
   if('/' != relurl[0]) {
-    int level=0;
+    int level = 0;
 
     /* First we need to find out if there's a ?-letter in the URL,
        and cut it and the right-side of that off */
     pathsep = strchr(protsep, '?');
     if(pathsep)
-      *pathsep=0;
+      *pathsep = 0;
 
     /* we have a relative path to append to the last slash if there's one
        available, or if the new URL is just a query string (starts with a
@@ -1517,14 +1604,14 @@ static char *concat_url(const char *base, const char *relurl)
     if(useurl[0] != '?') {
       pathsep = strrchr(protsep, '/');
       if(pathsep)
-        *pathsep=0;
+        *pathsep = 0;
     }
 
     /* Check if there's any slash after the host name, and if so, remember
        that position instead */
     pathsep = strchr(protsep, '/');
     if(pathsep)
-      protsep = pathsep+1;
+      protsep = pathsep + 1;
     else
       protsep = NULL;
 
@@ -1532,13 +1619,13 @@ static char *concat_url(const char *base, const char *relurl)
        and act accordingly */
 
     if((useurl[0] == '.') && (useurl[1] == '/'))
-      useurl+=2; /* just skip the "./" */
+      useurl += 2; /* just skip the "./" */
 
     while((useurl[0] == '.') &&
           (useurl[1] == '.') &&
           (useurl[2] == '/')) {
       level++;
-      useurl+=3; /* pass the "../" */
+      useurl += 3; /* pass the "../" */
     }
 
     if(protsep) {
@@ -1546,9 +1633,9 @@ static char *concat_url(const char *base, const char *relurl)
         /* cut off one more level from the right of the original URL */
         pathsep = strrchr(protsep, '/');
         if(pathsep)
-          *pathsep=0;
+          *pathsep = 0;
         else {
-          *protsep=0;
+          *protsep = 0;
           break;
         }
       }
@@ -1560,9 +1647,10 @@ static char *concat_url(const char *base, const char *relurl)
     if((relurl[0] == '/') && (relurl[1] == '/')) {
       /* the new URL starts with //, just keep the protocol part from the
          original one */
-      *protsep=0;
+      *protsep = 0;
       useurl = &relurl[2]; /* we keep the slashes from the original, so we
                               skip the new ones */
+      host_changed = TRUE;
     }
     else {
       /* cut off the original URL from the first slash, or deal with URLs
@@ -1575,7 +1663,7 @@ static char *concat_url(const char *base, const char *relurl)
         char *sep = strchr(protsep, '?');
         if(sep && (sep < pathsep))
           pathsep = sep;
-        *pathsep=0;
+        *pathsep = 0;
       }
       else {
         /* There was no slash. Now, since we might be operating on a badly
@@ -1584,7 +1672,7 @@ static char *concat_url(const char *base, const char *relurl)
            ?-letter as well! */
         pathsep = strchr(protsep, '?');
         if(pathsep)
-          *pathsep=0;
+          *pathsep = 0;
       }
     }
   }
@@ -1594,7 +1682,7 @@ static char *concat_url(const char *base, const char *relurl)
      letter we replace each space with %20 while it is replaced with '+'
      on the right side of the '?' letter.
   */
-  newlen = strlen_url(useurl);
+  newlen = strlen_url(useurl, !host_changed);
 
   urllen = strlen(url_clone);
 
@@ -1616,7 +1704,7 @@ static char *concat_url(const char *base, const char *relurl)
     newest[urllen++]='/';
 
   /* then append the new piece on the right side */
-  strcpy_url(&newest[urllen], useurl);
+  strcpy_url(&newest[urllen], useurl, !host_changed);
 
   free(url_clone);
 
@@ -1629,9 +1717,7 @@ static char *concat_url(const char *base, const char *relurl)
  * as given by the remote server and set up the new URL to request.
  */
 CURLcode Curl_follow(struct Curl_easy *data,
-                     char *newurl, /* this 'newurl' is the Location: string,
-                                      and it must be malloc()ed before passed
-                                      here */
+                     char *newurl,    /* the Location: string */
                      followtype type) /* see transfer.h */
 {
 #ifdef CURL_DISABLE_HTTP
@@ -1644,37 +1730,40 @@ CURLcode Curl_follow(struct Curl_easy *data,
 
   /* Location: redirect */
   bool disallowport = FALSE;
+  bool reachedmax = FALSE;
 
   if(type == FOLLOW_REDIR) {
     if((data->set.maxredirs != -1) &&
-        (data->set.followlocation >= data->set.maxredirs)) {
-      failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
-      return CURLE_TOO_MANY_REDIRECTS;
+       (data->set.followlocation >= data->set.maxredirs)) {
+      reachedmax = TRUE;
+      type = FOLLOW_FAKE; /* switch to fake to store the would-be-redirected
+                             to URL */
     }
+    else {
+      /* mark the next request as a followed location: */
+      data->state.this_is_a_follow = TRUE;
 
-    /* mark the next request as a followed location: */
-    data->state.this_is_a_follow = TRUE;
+      data->set.followlocation++; /* count location-followers */
 
-    data->set.followlocation++; /* count location-followers */
+      if(data->set.http_auto_referer) {
+        /* We are asked to automatically set the previous URL as the referer
+           when we get the next URL. We pick the ->url field, which may or may
+           not be 100% correct */
 
-    if(data->set.http_auto_referer) {
-      /* We are asked to automatically set the previous URL as the referer
-         when we get the next URL. We pick the ->url field, which may or may
-         not be 100% correct */
+        if(data->change.referer_alloc) {
+          Curl_safefree(data->change.referer);
+          data->change.referer_alloc = FALSE;
+        }
 
-      if(data->change.referer_alloc) {
-        Curl_safefree(data->change.referer);
-        data->change.referer_alloc = FALSE;
+        data->change.referer = strdup(data->change.url);
+        if(!data->change.referer)
+          return CURLE_OUT_OF_MEMORY;
+        data->change.referer_alloc = TRUE; /* yes, free this later */
       }
-
-      data->change.referer = strdup(data->change.url);
-      if(!data->change.referer)
-        return CURLE_OUT_OF_MEMORY;
-      data->change.referer_alloc = TRUE; /* yes, free this later */
     }
   }
 
-  if(!is_absolute_url(newurl))  {
+  if(!is_absolute_url(newurl)) {
     /***
      *DANG* this is an RFC 2068 violation. The URL is supposed
      to be absolute and this doesn't seem to be that!
@@ -1682,24 +1771,22 @@ CURLcode Curl_follow(struct Curl_easy *data,
     char *absolute = concat_url(data->change.url, newurl);
     if(!absolute)
       return CURLE_OUT_OF_MEMORY;
-    free(newurl);
     newurl = absolute;
   }
   else {
     /* The new URL MAY contain space or high byte values, that means a mighty
        stupid redirect URL but we still make an effort to do "right". */
     char *newest;
-    size_t newlen = strlen_url(newurl);
+    size_t newlen = strlen_url(newurl, FALSE);
 
     /* This is an absolute URL, don't allow the custom port number */
     disallowport = TRUE;
 
-    newest = malloc(newlen+1); /* get memory for this */
+    newest = malloc(newlen + 1); /* get memory for this */
     if(!newest)
       return CURLE_OUT_OF_MEMORY;
-    strcpy_url(newest, newurl); /* create a space-free URL */
 
-    free(newurl); /* that was no good */
+    strcpy_url(newest, newurl, FALSE); /* create a space-free URL */
     newurl = newest; /* use this instead now */
 
   }
@@ -1708,6 +1795,11 @@ CURLcode Curl_follow(struct Curl_easy *data,
     /* we're only figuring out the new url if we would've followed locations
        but now we're done so we can get out! */
     data->info.wouldredirect = newurl;
+
+    if(reachedmax) {
+      failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
+      return CURLE_TOO_MANY_REDIRECTS;
+    }
     return CURLE_OK;
   }
 
@@ -1721,7 +1813,6 @@ CURLcode Curl_follow(struct Curl_easy *data,
 
   data->change.url = newurl;
   data->change.url_alloc = TRUE;
-  newurl = NULL; /* don't free! */
 
   infof(data, "Issue another request to this URL: '%s'\n", data->change.url);
 
@@ -1763,7 +1854,8 @@ CURLcode Curl_follow(struct Curl_easy *data,
      * can be overridden with CURLOPT_POSTREDIR.
      */
     if((data->set.httpreq == HTTPREQ_POST
-        || data->set.httpreq == HTTPREQ_POST_FORM)
+        || data->set.httpreq == HTTPREQ_POST_FORM
+        || data->set.httpreq == HTTPREQ_POST_MIME)
        && !(data->set.keep_post & CURL_REDIR_POST_301)) {
       infof(data, "Switch from POST to GET\n");
       data->set.httpreq = HTTPREQ_GET;
@@ -1787,7 +1879,8 @@ CURLcode Curl_follow(struct Curl_easy *data,
      * can be overridden with CURLOPT_POSTREDIR.
      */
     if((data->set.httpreq == HTTPREQ_POST
-        || data->set.httpreq == HTTPREQ_POST_FORM)
+        || data->set.httpreq == HTTPREQ_POST_FORM
+        || data->set.httpreq == HTTPREQ_POST_MIME)
        && !(data->set.keep_post & CURL_REDIR_POST_302)) {
       infof(data, "Switch from POST to GET\n");
       data->set.httpreq = HTTPREQ_GET;
@@ -1795,7 +1888,7 @@ CURLcode Curl_follow(struct Curl_easy *data,
     break;
 
   case 303: /* See Other */
-    /* Disable both types of POSTs, unless the user explicitely
+    /* Disable both types of POSTs, unless the user explicitly
        asks for POST after POST */
     if(data->set.httpreq != HTTPREQ_GET
       && !(data->set.keep_post & CURL_REDIR_POST_303)) {
@@ -1820,7 +1913,7 @@ CURLcode Curl_follow(struct Curl_easy *data,
     break;
   }
   Curl_pgrsTime(data, TIMER_REDIRECT);
-  Curl_pgrsResetTimesSizes(data);
+  Curl_pgrsResetTransferSizes(data);
 
   return CURLE_OK;
 #endif /* CURL_DISABLE_HTTP */
@@ -1944,11 +2037,11 @@ Curl_setup_transfer(
          (http->sending == HTTPSEND_BODY)) {
         /* wait with write until we either got 100-continue or a timeout */
         k->exp100 = EXP100_AWAITING_CONTINUE;
-        k->start100 = Curl_tvnow();
+        k->start100 = Curl_now();
 
         /* Set a timeout for the multi interface. Add the inaccuracy margin so
            that we don't fire slightly too early and get denied to run. */
-        Curl_expire(data, data->set.expect_100_timeout);
+        Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
       }
       else {
         if(data->state.expect100header)