• Skip to content
  • Skip to link menu
  • KDE API Reference
  • kdelibs-4.10.4 API Reference
  • KDE Home
  • Contact Us
 

KIOSlave

  • kioslave
  • http
http.cpp
Go to the documentation of this file.
1 /*
2  Copyright (C) 2000-2003 Waldo Bastian <bastian@kde.org>
3  Copyright (C) 2000-2002 George Staikos <staikos@kde.org>
4  Copyright (C) 2000-2002 Dawit Alemayehu <adawit@kde.org>
5  Copyright (C) 2001,2002 Hamish Rodda <rodda@kde.org>
6  Copyright (C) 2007 Nick Shaforostoff <shafff@ukr.net>
7  Copyright (C) 2007 Daniel Nicoletti <mirttex@users.sourceforge.net>
8  Copyright (C) 2008,2009 Andreas Hartmetz <ahartmetz@gmail.com>
9 
10  This library is free software; you can redistribute it and/or
11  modify it under the terms of the GNU Library General Public
12  License (LGPL) as published by the Free Software Foundation;
13  either version 2 of the License, or (at your option) any later
14  version.
15 
16  This library is distributed in the hope that it will be useful,
17  but WITHOUT ANY WARRANTY; without even the implied warranty of
18  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19  Library General Public License for more details.
20 
21  You should have received a copy of the GNU Library General Public License
22  along with this library; see the file COPYING.LIB. If not, write to
23  the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
24  Boston, MA 02110-1301, USA.
25 */
26 
27 // TODO delete / do not save very big files; "very big" to be defined
28 
29 #define QT_NO_CAST_FROM_ASCII
30 
31 #include "http.h"
32 
33 #include <config.h>
34 
35 #include <fcntl.h>
36 #include <utime.h>
37 #include <stdlib.h>
38 #include <stdio.h>
39 #include <sys/stat.h>
40 #include <sys/time.h>
41 #include <unistd.h> // must be explicitly included for MacOSX
42 
43 #include <QtXml/qdom.h>
44 #include <QtCore/QFile>
45 #include <QtCore/QRegExp>
46 #include <QtCore/QDate>
47 #include <QtCore/QBuffer>
48 #include <QtCore/QIODevice>
49 #include <QtDBus/QtDBus>
50 #include <QtNetwork/QAuthenticator>
51 #include <QtNetwork/QNetworkProxy>
52 #include <QtNetwork/QTcpSocket>
53 
54 #include <kurl.h>
55 #include <kdebug.h>
56 #include <klocale.h>
57 #include <kconfig.h>
58 #include <kconfiggroup.h>
59 #include <kservice.h>
60 #include <kdatetime.h>
61 #include <kcomponentdata.h>
62 #include <kmimetype.h>
63 #include <ktoolinvocation.h>
64 #include <kstandarddirs.h>
65 #include <kremoteencoding.h>
66 #include <ktcpsocket.h>
67 #include <kmessagebox.h>
68 
69 #include <kio/ioslave_defaults.h>
70 #include <kio/http_slave_defaults.h>
71 
72 #include <httpfilter.h>
73 
74 #include <solid/networking.h>
75 
76 #include <kapplication.h>
77 #include <kaboutdata.h>
78 #include <kcmdlineargs.h>
79 #include <kde_file.h>
80 #include <ktemporaryfile.h>
81 
82 #include "httpauthentication.h"
83 
84 // HeaderTokenizer declarations
85 #include "parsinghelpers.h"
86 //string parsing helpers and HeaderTokenizer implementation
87 #include "parsinghelpers.cpp"
88 
89 // KDE5 TODO (QT5) : use QString::htmlEscape or whatever https://qt.gitorious.org/qt/qtbase/merge_requests/56
90 // ends up with.
91 static QString htmlEscape(const QString &plain)
92 {
93  QString rich;
94  rich.reserve(int(plain.length() * 1.1));
95  for (int i = 0; i < plain.length(); ++i) {
96  if (plain.at(i) == QLatin1Char('<'))
97  rich += QLatin1String("&lt;");
98  else if (plain.at(i) == QLatin1Char('>'))
99  rich += QLatin1String("&gt;");
100  else if (plain.at(i) == QLatin1Char('&'))
101  rich += QLatin1String("&amp;");
102  else if (plain.at(i) == QLatin1Char('"'))
103  rich += QLatin1String("&quot;");
104  else
105  rich += plain.at(i);
106  }
107  rich.squeeze();
108  return rich;
109 }
110 
111 static bool supportedProxyScheme(const QString& scheme)
112 {
113  return (scheme.startsWith(QLatin1String("http"), Qt::CaseInsensitive)
114  || scheme == QLatin1String("socks"));
115 }
116 
117 // see filenameFromUrl(): a sha1 hash is 160 bits
118 static const int s_hashedUrlBits = 160; // this number should always be divisible by eight
119 static const int s_hashedUrlNibbles = s_hashedUrlBits / 4;
120 static const int s_hashedUrlBytes = s_hashedUrlBits / 8;
121 static const int s_MaxInMemPostBufSize = 256 * 1024; // Write anyting over 256 KB to file...
122 
123 using namespace KIO;
124 
125 extern "C" int KDE_EXPORT kdemain( int argc, char **argv )
126 {
127  QCoreApplication app( argc, argv ); // needed for QSocketNotifier
128  KComponentData componentData( "kio_http", "kdelibs4" );
129  (void) KGlobal::locale();
130 
131  if (argc != 4)
132  {
133  fprintf(stderr, "Usage: kio_http protocol domain-socket1 domain-socket2\n");
134  exit(-1);
135  }
136 
137  HTTPProtocol slave(argv[1], argv[2], argv[3]);
138  slave.dispatchLoop();
139  return 0;
140 }
141 
142 /*********************************** Generic utility functions ********************/
143 
144 static QString toQString(const QByteArray& value)
145 {
146  return QString::fromLatin1(value.constData(), value.size());
147 }
148 
149 static bool isCrossDomainRequest( const QString& fqdn, const QString& originURL )
150 {
151  //TODO read the RFC
152  if (originURL == QLatin1String("true")) // Backwards compatibility
153  return true;
154 
155  KUrl url ( originURL );
156 
157  // Document Origin domain
158  QString a = url.host();
159  // Current request domain
160  QString b = fqdn;
161 
162  if (a == b)
163  return false;
164 
165  QStringList la = a.split(QLatin1Char('.'), QString::SkipEmptyParts);
166  QStringList lb = b.split(QLatin1Char('.'), QString::SkipEmptyParts);
167 
168  if (qMin(la.count(), lb.count()) < 2) {
169  return true; // better safe than sorry...
170  }
171 
172  while(la.count() > 2)
173  la.pop_front();
174  while(lb.count() > 2)
175  lb.pop_front();
176 
177  return la != lb;
178 }
179 
180 /*
181  Eliminates any custom header that could potentially alter the request
182 */
183 static QString sanitizeCustomHTTPHeader(const QString& _header)
184 {
185  QString sanitizedHeaders;
186  const QStringList headers = _header.split(QRegExp(QLatin1String("[\r\n]")));
187 
188  for(QStringList::ConstIterator it = headers.begin(); it != headers.end(); ++it)
189  {
190  // Do not allow Request line to be specified and ignore
191  // the other HTTP headers.
192  if (!(*it).contains(QLatin1Char(':')) ||
193  (*it).startsWith(QLatin1String("host"), Qt::CaseInsensitive) ||
194  (*it).startsWith(QLatin1String("proxy-authorization"), Qt::CaseInsensitive) ||
195  (*it).startsWith(QLatin1String("via"), Qt::CaseInsensitive))
196  continue;
197 
198  sanitizedHeaders += (*it);
199  sanitizedHeaders += QLatin1String("\r\n");
200  }
201  sanitizedHeaders.chop(2);
202 
203  return sanitizedHeaders;
204 }
205 
206 static bool isPotentialSpoofingAttack(const HTTPProtocol::HTTPRequest& request, const KConfigGroup* config)
207 {
208  // kDebug(7113) << request.url << "response code: " << request.responseCode << "previous response code:" << request.prevResponseCode;
209  if (config->readEntry("no-spoof-check", false)) {
210  return false;
211  }
212 
213  if (request.url.user().isEmpty()) {
214  return false;
215  }
216 
217  // We already have cached authentication.
218  if (config->readEntry(QLatin1String("cached-www-auth"), false)) {
219  return false;
220  }
221 
222  const QString userName = config->readEntry(QLatin1String("LastSpoofedUserName"), QString());
223  return ((userName.isEmpty() || userName != request.url.user()) && request.responseCode != 401 && request.prevResponseCode != 401);
224 }
225 
226 // for a given response code, conclude if the response is going to/likely to have a response body
227 static bool canHaveResponseBody(int responseCode, KIO::HTTP_METHOD method)
228 {
229 /* RFC 2616 says...
230  1xx: false
231  200: method HEAD: false, otherwise:true
232  201: true
233  202: true
234  203: see 200
235  204: false
236  205: false
237  206: true
238  300: see 200
239  301: see 200
240  302: see 200
241  303: see 200
242  304: false
243  305: probably like 300, RFC seems to expect disconnection afterwards...
244  306: (reserved), for simplicity do it just like 200
245  307: see 200
246  4xx: see 200
247  5xx :see 200
248 */
249  if (responseCode >= 100 && responseCode < 200) {
250  return false;
251  }
252  switch (responseCode) {
253  case 201:
254  case 202:
255  case 206:
256  // RFC 2616 does not mention HEAD in the description of the above. if the assert turns out
257  // to be a problem the response code should probably be treated just like 200 and friends.
258  Q_ASSERT(method != HTTP_HEAD);
259  return true;
260  case 204:
261  case 205:
262  case 304:
263  return false;
264  default:
265  break;
266  }
267  // safe (and for most remaining response codes exactly correct) default
268  return method != HTTP_HEAD;
269 }
270 
271 static bool isEncryptedHttpVariety(const QByteArray &p)
272 {
273  return p == "https" || p == "webdavs";
274 }
275 
276 static bool isValidProxy(const KUrl &u)
277 {
278  return u.isValid() && u.hasHost();
279 }
280 
281 static bool isHttpProxy(const KUrl &u)
282 {
283  return isValidProxy(u) && u.protocol() == QLatin1String("http");
284 }
285 
286 static QIODevice* createPostBufferDeviceFor (KIO::filesize_t size)
287 {
288  QIODevice* device;
289  if (size > static_cast<KIO::filesize_t>(s_MaxInMemPostBufSize))
290  device = new KTemporaryFile;
291  else
292  device = new QBuffer;
293 
294  if (!device->open(QIODevice::ReadWrite))
295  return 0;
296 
297  return device;
298 }
299 
300 QByteArray HTTPProtocol::HTTPRequest::methodString() const
301 {
302  if (!methodStringOverride.isEmpty())
303  return (methodStringOverride + QLatin1Char(' ')).toLatin1();
304 
305  switch(method) {
306  case HTTP_GET:
307  return "GET ";
308  case HTTP_PUT:
309  return "PUT ";
310  case HTTP_POST:
311  return "POST ";
312  case HTTP_HEAD:
313  return "HEAD ";
314  case HTTP_DELETE:
315  return "DELETE ";
316  case HTTP_OPTIONS:
317  return "OPTIONS ";
318  case DAV_PROPFIND:
319  return "PROPFIND ";
320  case DAV_PROPPATCH:
321  return "PROPPATCH ";
322  case DAV_MKCOL:
323  return "MKCOL ";
324  case DAV_COPY:
325  return "COPY ";
326  case DAV_MOVE:
327  return "MOVE ";
328  case DAV_LOCK:
329  return "LOCK ";
330  case DAV_UNLOCK:
331  return "UNLOCK ";
332  case DAV_SEARCH:
333  return "SEARCH ";
334  case DAV_SUBSCRIBE:
335  return "SUBSCRIBE ";
336  case DAV_UNSUBSCRIBE:
337  return "UNSUBSCRIBE ";
338  case DAV_POLL:
339  return "POLL ";
340  case DAV_NOTIFY:
341  return "NOTIFY ";
342  case DAV_REPORT:
343  return "REPORT ";
344  default:
345  Q_ASSERT(false);
346  return QByteArray();
347  }
348 }
349 
350 static QString formatHttpDate(qint64 date)
351 {
352  KDateTime dt;
353  dt.setTime_t(date);
354  QString ret = dt.toString(KDateTime::RFCDateDay);
355  ret.chop(6); // remove " +0000"
356  // RFCDate[Day] omits the second if zero, but HTTP requires it; see bug 240585.
357  if (!dt.time().second()) {
358  ret.append(QLatin1String(":00"));
359  }
360  ret.append(QLatin1String(" GMT"));
361  return ret;
362 }
363 
364 static bool isAuthenticationRequired(int responseCode)
365 {
366  return (responseCode == 401) || (responseCode == 407);
367 }
368 
369 #define NO_SIZE ((KIO::filesize_t) -1)
370 
371 #ifdef HAVE_STRTOLL
372 #define STRTOLL strtoll
373 #else
374 #define STRTOLL strtol
375 #endif
376 
377 
378 /************************************** HTTPProtocol **********************************************/
379 
380 
381 HTTPProtocol::HTTPProtocol( const QByteArray &protocol, const QByteArray &pool,
382  const QByteArray &app )
383  : TCPSlaveBase(protocol, pool, app, isEncryptedHttpVariety(protocol))
384  , m_iSize(NO_SIZE)
385  , m_iPostDataSize(NO_SIZE)
386  , m_isBusy(false)
387  , m_POSTbuf(0)
388  , m_maxCacheAge(DEFAULT_MAX_CACHE_AGE)
389  , m_maxCacheSize(DEFAULT_MAX_CACHE_SIZE)
390  , m_protocol(protocol)
391  , m_wwwAuth(0)
392  , m_proxyAuth(0)
393  , m_socketProxyAuth(0)
394  , m_iError(0)
395  , m_isLoadingErrorPage(false)
396  , m_remoteRespTimeout(DEFAULT_RESPONSE_TIMEOUT)
397 {
398  reparseConfiguration();
399  setBlocking(true);
400  connect(socket(), SIGNAL(proxyAuthenticationRequired(QNetworkProxy,QAuthenticator*)),
401  this, SLOT(proxyAuthenticationForSocket(QNetworkProxy,QAuthenticator*)));
402 }
403 
404 HTTPProtocol::~HTTPProtocol()
405 {
406  httpClose(false);
407 }
408 
409 void HTTPProtocol::reparseConfiguration()
410 {
411  kDebug(7113);
412 
413  delete m_proxyAuth;
414  delete m_wwwAuth;
415  m_proxyAuth = 0;
416  m_wwwAuth = 0;
417  m_request.proxyUrl.clear(); //TODO revisit
418  m_request.proxyUrls.clear();
419 
420  TCPSlaveBase::reparseConfiguration();
421 }
422 
423 void HTTPProtocol::resetConnectionSettings()
424 {
425  m_isEOF = false;
426  m_iError = 0;
427  m_isLoadingErrorPage = false;
428 }
429 
430 quint16 HTTPProtocol::defaultPort() const
431 {
432  return isEncryptedHttpVariety(m_protocol) ? DEFAULT_HTTPS_PORT : DEFAULT_HTTP_PORT;
433 }
434 
435 void HTTPProtocol::resetResponseParsing()
436 {
437  m_isRedirection = false;
438  m_isChunked = false;
439  m_iSize = NO_SIZE;
440  clearUnreadBuffer();
441 
442  m_responseHeaders.clear();
443  m_contentEncodings.clear();
444  m_transferEncodings.clear();
445  m_contentMD5.clear();
446  m_mimeType.clear();
447 
448  setMetaData(QLatin1String("request-id"), m_request.id);
449 }
450 
451 void HTTPProtocol::resetSessionSettings()
452 {
453  // Follow HTTP/1.1 spec and enable keep-alive by default
454  // unless the remote side tells us otherwise or we determine
455  // the persistent link has been terminated by the remote end.
456  m_request.isKeepAlive = true;
457  m_request.keepAliveTimeout = 0;
458 
459  m_request.redirectUrl = KUrl();
460  m_request.useCookieJar = config()->readEntry("Cookies", false);
461  m_request.cacheTag.useCache = config()->readEntry("UseCache", true);
462  m_request.preferErrorPage = config()->readEntry("errorPage", true);
463  const bool noAuth = config()->readEntry("no-auth", false);
464  m_request.doNotWWWAuthenticate = config()->readEntry("no-www-auth", noAuth);
465  m_request.doNotProxyAuthenticate = config()->readEntry("no-proxy-auth", noAuth);
466  m_strCacheDir = config()->readPathEntry("CacheDir", QString());
467  m_maxCacheAge = config()->readEntry("MaxCacheAge", DEFAULT_MAX_CACHE_AGE);
468  m_request.windowId = config()->readEntry("window-id");
469 
470  m_request.methodStringOverride = metaData(QLatin1String("CustomHTTPMethod"));
471 
472  kDebug(7113) << "Window Id =" << m_request.windowId;
473  kDebug(7113) << "ssl_was_in_use =" << metaData(QLatin1String("ssl_was_in_use"));
474 
475  m_request.referrer.clear();
476  // RFC 2616: do not send the referrer if the referrer page was served using SSL and
477  // the current page does not use SSL.
478  if ( config()->readEntry("SendReferrer", true) &&
479  (isEncryptedHttpVariety(m_protocol) || metaData(QLatin1String("ssl_was_in_use")) != QLatin1String("TRUE") ) )
480  {
481  KUrl refUrl(metaData(QLatin1String("referrer")));
482  if (refUrl.isValid()) {
483  // Sanitize
484  QString protocol = refUrl.protocol();
485  if (protocol.startsWith(QLatin1String("webdav"))) {
486  protocol.replace(0, 6, QLatin1String("http"));
487  refUrl.setProtocol(protocol);
488  }
489 
490  if (protocol.startsWith(QLatin1String("http"))) {
491  m_request.referrer = toQString(refUrl.toEncoded(QUrl::RemoveUserInfo | QUrl::RemoveFragment));
492  }
493  }
494  }
495 
496  if (config()->readEntry("SendLanguageSettings", true)) {
497  m_request.charsets = config()->readEntry("Charsets", DEFAULT_PARTIAL_CHARSET_HEADER);
498  if (!m_request.charsets.contains(QLatin1String("*;"), Qt::CaseInsensitive)) {
499  m_request.charsets += QLatin1String(",*;q=0.5");
500  }
501  m_request.languages = config()->readEntry("Languages", DEFAULT_LANGUAGE_HEADER);
502  } else {
503  m_request.charsets.clear();
504  m_request.languages.clear();
505  }
506 
507  // Adjust the offset value based on the "resume" meta-data.
508  QString resumeOffset = metaData(QLatin1String("resume"));
509  if (!resumeOffset.isEmpty()) {
510  m_request.offset = resumeOffset.toULongLong();
511  } else {
512  m_request.offset = 0;
513  }
514  // Same procedure for endoffset.
515  QString resumeEndOffset = metaData(QLatin1String("resume_until"));
516  if (!resumeEndOffset.isEmpty()) {
517  m_request.endoffset = resumeEndOffset.toULongLong();
518  } else {
519  m_request.endoffset = 0;
520  }
521 
522  m_request.disablePassDialog = config()->readEntry("DisablePassDlg", false);
523  m_request.allowTransferCompression = config()->readEntry("AllowCompressedPage", true);
524  m_request.id = metaData(QLatin1String("request-id"));
525 
526  // Store user agent for this host.
527  if (config()->readEntry("SendUserAgent", true)) {
528  m_request.userAgent = metaData(QLatin1String("UserAgent"));
529  } else {
530  m_request.userAgent.clear();
531  }
532 
533  m_request.cacheTag.etag.clear();
534  // -1 is also the value returned by KDateTime::toTime_t() from an invalid instance.
535  m_request.cacheTag.servedDate = -1;
536  m_request.cacheTag.lastModifiedDate = -1;
537  m_request.cacheTag.expireDate = -1;
538 
539  m_request.responseCode = 0;
540  m_request.prevResponseCode = 0;
541 
542  delete m_wwwAuth;
543  m_wwwAuth = 0;
544  delete m_socketProxyAuth;
545  m_socketProxyAuth = 0;
546 
547  // Obtain timeout values
548  m_remoteRespTimeout = responseTimeout();
549 
550  // Bounce back the actual referrer sent
551  setMetaData(QLatin1String("referrer"), m_request.referrer);
552 
553  // Reset the post data size
554  m_iPostDataSize = NO_SIZE;
555 }
556 
557 void HTTPProtocol::setHost( const QString& host, quint16 port,
558  const QString& user, const QString& pass )
559 {
560  // Reset the webdav-capable flags for this host
561  if ( m_request.url.host() != host )
562  m_davHostOk = m_davHostUnsupported = false;
563 
564  m_request.url.setHost(host);
565 
566  // is it an IPv6 address?
567  if (host.indexOf(QLatin1Char(':')) == -1) {
568  m_request.encoded_hostname = toQString(QUrl::toAce(host));
569  } else {
570  int pos = host.indexOf(QLatin1Char('%'));
571  if (pos == -1)
572  m_request.encoded_hostname = QLatin1Char('[') + host + QLatin1Char(']');
573  else
574  // don't send the scope-id in IPv6 addresses to the server
575  m_request.encoded_hostname = QLatin1Char('[') + host.left(pos) + QLatin1Char(']');
576  }
577  m_request.url.setPort((port > 0 && port != defaultPort()) ? port : -1);
578  m_request.url.setUser(user);
579  m_request.url.setPass(pass);
580 
581  // On new connection always clear previous proxy information...
582  m_request.proxyUrl.clear();
583  m_request.proxyUrls.clear();
584 
585  kDebug(7113) << "Hostname is now:" << m_request.url.host()
586  << "(" << m_request.encoded_hostname << ")";
587 }
588 
589 bool HTTPProtocol::maybeSetRequestUrl(const KUrl &u)
590 {
591  kDebug(7113) << u;
592 
593  m_request.url = u;
594  m_request.url.setPort(u.port(defaultPort()) != defaultPort() ? u.port() : -1);
595 
596  if (u.host().isEmpty()) {
597  error( KIO::ERR_UNKNOWN_HOST, i18n("No host specified."));
598  return false;
599  }
600 
601  if (u.path().isEmpty()) {
602  KUrl newUrl(u);
603  newUrl.setPath(QLatin1String("/"));
604  redirection(newUrl);
605  finished();
606  return false;
607  }
608 
609  return true;
610 }
611 
612 void HTTPProtocol::proceedUntilResponseContent( bool dataInternal /* = false */ )
613 {
614  kDebug (7113);
615 
616  const bool status = (proceedUntilResponseHeader() && readBody(dataInternal));
617 
618  // If not an error condition or internal request, close
619  // the connection based on the keep alive settings...
620  if (!m_iError && !dataInternal) {
621  httpClose(m_request.isKeepAlive);
622  }
623 
624  // if data is required internally or we got error, don't finish,
625  // it is processed before we finish()
626  if (dataInternal || !status) {
627  return;
628  }
629 
630  if (!sendHttpError()) {
631  finished();
632  }
633 }
634 
635 bool HTTPProtocol::proceedUntilResponseHeader()
636 {
637  kDebug (7113);
638 
639  // Retry the request until it succeeds or an unrecoverable error occurs.
640  // Recoverable errors are, for example:
641  // - Proxy or server authentication required: Ask for credentials and try again,
642  // this time with an authorization header in the request.
643  // - Server-initiated timeout on keep-alive connection: Reconnect and try again
644 
645  while (true) {
646  if (!sendQuery()) {
647  return false;
648  }
649  if (readResponseHeader()) {
650  // Success, finish the request.
651  break;
652  }
653 
654  // If not loading error page and the response code requires us to resend the query,
655  // then throw away any error message that might have been sent by the server.
656  if (!m_isLoadingErrorPage && isAuthenticationRequired(m_request.responseCode)) {
657  // This gets rid of any error page sent with 401 or 407 authentication required response...
658  readBody(true);
659  }
660 
661  // no success, close the cache file so the cache state is reset - that way most other code
662  // doesn't have to deal with the cache being in various states.
663  cacheFileClose();
664  if (m_iError || m_isLoadingErrorPage) {
665  // Unrecoverable error, abort everything.
666  // Also, if we've just loaded an error page there is nothing more to do.
667  // In that case we abort to avoid loops; some webservers manage to send 401 and
668  // no authentication request. Or an auth request we don't understand.
669  return false;
670  }
671 
672  if (!m_request.isKeepAlive) {
673  httpCloseConnection();
674  m_request.isKeepAlive = true;
675  m_request.keepAliveTimeout = 0;
676  }
677  }
678 
679  // Do not save authorization if the current response code is
680  // 4xx (client error) or 5xx (server error).
681  kDebug(7113) << "Previous Response:" << m_request.prevResponseCode;
682  kDebug(7113) << "Current Response:" << m_request.responseCode;
683 
684  setMetaData(QLatin1String("responsecode"), QString::number(m_request.responseCode));
685  setMetaData(QLatin1String("content-type"), m_mimeType);
686 
687  // At this point sendBody() should have delivered any POST data.
688  clearPostDataBuffer();
689 
690  return true;
691 }
692 
693 void HTTPProtocol::stat(const KUrl& url)
694 {
695  kDebug(7113) << url;
696 
697  if (!maybeSetRequestUrl(url))
698  return;
699  resetSessionSettings();
700 
701  if ( m_protocol != "webdav" && m_protocol != "webdavs" )
702  {
703  QString statSide = metaData(QLatin1String("statSide"));
704  if (statSide != QLatin1String("source"))
705  {
706  // When uploading we assume the file doesn't exit
707  error( ERR_DOES_NOT_EXIST, url.prettyUrl() );
708  return;
709  }
710 
711  // When downloading we assume it exists
712  UDSEntry entry;
713  entry.insert( KIO::UDSEntry::UDS_NAME, url.fileName() );
714  entry.insert( KIO::UDSEntry::UDS_FILE_TYPE, S_IFREG ); // a file
715  entry.insert( KIO::UDSEntry::UDS_ACCESS, S_IRUSR | S_IRGRP | S_IROTH ); // readable by everybody
716 
717  statEntry( entry );
718  finished();
719  return;
720  }
721 
722  davStatList( url );
723 }
724 
725 void HTTPProtocol::listDir( const KUrl& url )
726 {
727  kDebug(7113) << url;
728 
729  if (!maybeSetRequestUrl(url))
730  return;
731  resetSessionSettings();
732 
733  davStatList( url, false );
734 }
735 
736 void HTTPProtocol::davSetRequest( const QByteArray& requestXML )
737 {
738  // insert the document into the POST buffer, kill trailing zero byte
739  cachePostData(requestXML);
740 }
741 
742 void HTTPProtocol::davStatList( const KUrl& url, bool stat )
743 {
744  UDSEntry entry;
745 
746  // check to make sure this host supports WebDAV
747  if ( !davHostOk() )
748  return;
749 
750  // Maybe it's a disguised SEARCH...
751  QString query = metaData(QLatin1String("davSearchQuery"));
752  if ( !query.isEmpty() )
753  {
754  QByteArray request = "<?xml version=\"1.0\"?>\r\n";
755  request.append( "<D:searchrequest xmlns:D=\"DAV:\">\r\n" );
756  request.append( query.toUtf8() );
757  request.append( "</D:searchrequest>\r\n" );
758 
759  davSetRequest( request );
760  } else {
761  // We are only after certain features...
762  QByteArray request;
763  request = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
764  "<D:propfind xmlns:D=\"DAV:\">";
765 
766  // insert additional XML request from the davRequestResponse metadata
767  if ( hasMetaData(QLatin1String("davRequestResponse")) )
768  request += metaData(QLatin1String("davRequestResponse")).toUtf8();
769  else {
770  // No special request, ask for default properties
771  request += "<D:prop>"
772  "<D:creationdate/>"
773  "<D:getcontentlength/>"
774  "<D:displayname/>"
775  "<D:source/>"
776  "<D:getcontentlanguage/>"
777  "<D:getcontenttype/>"
778  "<D:getlastmodified/>"
779  "<D:getetag/>"
780  "<D:supportedlock/>"
781  "<D:lockdiscovery/>"
782  "<D:resourcetype/>"
783  "</D:prop>";
784  }
785  request += "</D:propfind>";
786 
787  davSetRequest( request );
788  }
789 
790  // WebDAV Stat or List...
791  m_request.method = query.isEmpty() ? DAV_PROPFIND : DAV_SEARCH;
792  m_request.url.setQuery(QString());
793  m_request.cacheTag.policy = CC_Reload;
794  m_request.davData.depth = stat ? 0 : 1;
795  if (!stat)
796  m_request.url.adjustPath(KUrl::AddTrailingSlash);
797 
798  proceedUntilResponseContent( true );
799  infoMessage(QLatin1String(""));
800 
801  // Has a redirection already been called? If so, we're done.
802  if (m_isRedirection || m_iError) {
803  if (m_isRedirection) {
804  davFinished();
805  }
806  return;
807  }
808 
809  QDomDocument multiResponse;
810  multiResponse.setContent( m_webDavDataBuf, true );
811 
812  bool hasResponse = false;
813 
814  // kDebug(7113) << endl << multiResponse.toString(2);
815 
816  for ( QDomNode n = multiResponse.documentElement().firstChild();
817  !n.isNull(); n = n.nextSibling()) {
818  QDomElement thisResponse = n.toElement();
819  if (thisResponse.isNull())
820  continue;
821 
822  hasResponse = true;
823 
824  QDomElement href = thisResponse.namedItem(QLatin1String("href")).toElement();
825  if ( !href.isNull() ) {
826  entry.clear();
827 
828  QString urlStr = QUrl::fromPercentEncoding(href.text().toUtf8());
829 #if 0 // qt4/kde4 say: it's all utf8...
830  int encoding = remoteEncoding()->encodingMib();
831  if ((encoding == 106) && (!KStringHandler::isUtf8(KUrl::decode_string(urlStr, 4).toLatin1())))
832  encoding = 4; // Use latin1 if the file is not actually utf-8
833 
834  KUrl thisURL ( urlStr, encoding );
835 #else
836  KUrl thisURL( urlStr );
837 #endif
838 
839  if ( thisURL.isValid() ) {
840  QString name = thisURL.fileName();
841 
842  // base dir of a listDir(): name should be "."
843  if ( !stat && thisURL.path(KUrl::AddTrailingSlash).length() == url.path(KUrl::AddTrailingSlash).length() )
844  name = QLatin1Char('.');
845 
846  entry.insert( KIO::UDSEntry::UDS_NAME, name.isEmpty() ? href.text() : name );
847  }
848 
849  QDomNodeList propstats = thisResponse.elementsByTagName(QLatin1String("propstat"));
850 
851  davParsePropstats( propstats, entry );
852 
853  // Since a lot of webdav servers seem not to send the content-type information
854  // for the requested directory listings, we attempt to guess the mime-type from
855  // the resource name so long as the resource is not a directory.
856  if (entry.stringValue(KIO::UDSEntry::UDS_MIME_TYPE).isEmpty() &&
857  entry.numberValue(KIO::UDSEntry::UDS_FILE_TYPE) != S_IFDIR) {
858  int accuracy = 0;
859  KMimeType::Ptr mime = KMimeType::findByUrl(thisURL.fileName(), 0, false, true, &accuracy);
860  if (mime && !mime->isDefault() && accuracy == 100) {
861  kDebug(7113) << "Setting" << mime->name() << "as guessed mime type for" << thisURL.fileName();
862  entry.insert( KIO::UDSEntry::UDS_GUESSED_MIME_TYPE, mime->name());
863  }
864  }
865 
866  if ( stat ) {
867  // return an item
868  statEntry( entry );
869  davFinished();
870  return;
871  }
872 
873  listEntry( entry, false );
874  } else {
875  kDebug(7113) << "Error: no URL contained in response to PROPFIND on" << url;
876  }
877  }
878 
879  if ( stat || !hasResponse ) {
880  error( ERR_DOES_NOT_EXIST, url.prettyUrl() );
881  return;
882  }
883 
884  listEntry( entry, true );
885  davFinished();
886 }
887 
888 void HTTPProtocol::davGeneric( const KUrl& url, KIO::HTTP_METHOD method, qint64 size )
889 {
890  kDebug(7113) << url;
891 
892  if (!maybeSetRequestUrl(url))
893  return;
894  resetSessionSettings();
895 
896  // check to make sure this host supports WebDAV
897  if ( !davHostOk() )
898  return;
899 
900  // WebDAV method
901  m_request.method = method;
902  m_request.url.setQuery(QString());
903  m_request.cacheTag.policy = CC_Reload;
904 
905  m_iPostDataSize = (size > -1 ? static_cast<KIO::filesize_t>(size) : NO_SIZE);
906  proceedUntilResponseContent();
907 }
908 
909 int HTTPProtocol::codeFromResponse( const QString& response )
910 {
911  const int firstSpace = response.indexOf( QLatin1Char(' ') );
912  const int secondSpace = response.indexOf( QLatin1Char(' '), firstSpace + 1 );
913  return response.mid( firstSpace + 1, secondSpace - firstSpace - 1 ).toInt();
914 }
915 
916 void HTTPProtocol::davParsePropstats( const QDomNodeList& propstats, UDSEntry& entry )
917 {
918  QString mimeType;
919  bool foundExecutable = false;
920  bool isDirectory = false;
921  uint lockCount = 0;
922  uint supportedLockCount = 0;
923 
924  for ( int i = 0; i < propstats.count(); i++)
925  {
926  QDomElement propstat = propstats.item(i).toElement();
927 
928  QDomElement status = propstat.namedItem(QLatin1String("status")).toElement();
929  if ( status.isNull() )
930  {
931  // error, no status code in this propstat
932  kDebug(7113) << "Error, no status code in this propstat";
933  return;
934  }
935 
936  int code = codeFromResponse( status.text() );
937 
938  if ( code != 200 )
939  {
940  kDebug(7113) << "Got status code" << code << "(this may mean that some properties are unavailable)";
941  continue;
942  }
943 
944  QDomElement prop = propstat.namedItem( QLatin1String("prop") ).toElement();
945  if ( prop.isNull() )
946  {
947  kDebug(7113) << "Error: no prop segment in this propstat.";
948  return;
949  }
950 
951  if ( hasMetaData( QLatin1String("davRequestResponse") ) )
952  {
953  QDomDocument doc;
954  doc.appendChild(prop);
955  entry.insert( KIO::UDSEntry::UDS_XML_PROPERTIES, doc.toString() );
956  }
957 
958  for ( QDomNode n = prop.firstChild(); !n.isNull(); n = n.nextSibling() )
959  {
960  QDomElement property = n.toElement();
961  if (property.isNull())
962  continue;
963 
964  if ( property.namespaceURI() != QLatin1String("DAV:") )
965  {
966  // break out - we're only interested in properties from the DAV namespace
967  continue;
968  }
969 
970  if ( property.tagName() == QLatin1String("creationdate") )
971  {
972  // Resource creation date. Should be is ISO 8601 format.
973  entry.insert( KIO::UDSEntry::UDS_CREATION_TIME, parseDateTime( property.text(), property.attribute(QLatin1String("dt")) ) );
974  }
975  else if ( property.tagName() == QLatin1String("getcontentlength") )
976  {
977  // Content length (file size)
978  entry.insert( KIO::UDSEntry::UDS_SIZE, property.text().toULong() );
979  }
980  else if ( property.tagName() == QLatin1String("displayname") )
981  {
982  // Name suitable for presentation to the user
983  setMetaData( QLatin1String("davDisplayName"), property.text() );
984  }
985  else if ( property.tagName() == QLatin1String("source") )
986  {
987  // Source template location
988  QDomElement source = property.namedItem( QLatin1String("link") ).toElement()
989  .namedItem( QLatin1String("dst") ).toElement();
990  if ( !source.isNull() )
991  setMetaData( QLatin1String("davSource"), source.text() );
992  }
993  else if ( property.tagName() == QLatin1String("getcontentlanguage") )
994  {
995  // equiv. to Content-Language header on a GET
996  setMetaData( QLatin1String("davContentLanguage"), property.text() );
997  }
998  else if ( property.tagName() == QLatin1String("getcontenttype") )
999  {
1000  // Content type (mime type)
1001  // This may require adjustments for other server-side webdav implementations
1002  // (tested with Apache + mod_dav 1.0.3)
1003  if ( property.text() == QLatin1String("httpd/unix-directory") )
1004  {
1005  isDirectory = true;
1006  }
1007  else
1008  {
1009  mimeType = property.text();
1010  }
1011  }
1012  else if ( property.tagName() == QLatin1String("executable") )
1013  {
1014  // File executable status
1015  if ( property.text() == QLatin1String("T") )
1016  foundExecutable = true;
1017 
1018  }
1019  else if ( property.tagName() == QLatin1String("getlastmodified") )
1020  {
1021  // Last modification date
1022  entry.insert( KIO::UDSEntry::UDS_MODIFICATION_TIME, parseDateTime( property.text(), property.attribute(QLatin1String("dt")) ) );
1023  }
1024  else if ( property.tagName() == QLatin1String("getetag") )
1025  {
1026  // Entity tag
1027  setMetaData( QLatin1String("davEntityTag"), property.text() );
1028  }
1029  else if ( property.tagName() == QLatin1String("supportedlock") )
1030  {
1031  // Supported locking specifications
1032  for ( QDomNode n2 = property.firstChild(); !n2.isNull(); n2 = n2.nextSibling() )
1033  {
1034  QDomElement lockEntry = n2.toElement();
1035  if ( lockEntry.tagName() == QLatin1String("lockentry") )
1036  {
1037  QDomElement lockScope = lockEntry.namedItem( QLatin1String("lockscope") ).toElement();
1038  QDomElement lockType = lockEntry.namedItem( QLatin1String("locktype") ).toElement();
1039  if ( !lockScope.isNull() && !lockType.isNull() )
1040  {
1041  // Lock type was properly specified
1042  supportedLockCount++;
1043  const QString lockCountStr = QString::number(supportedLockCount);
1044  const QString scope = lockScope.firstChild().toElement().tagName();
1045  const QString type = lockType.firstChild().toElement().tagName();
1046 
1047  setMetaData( QLatin1String("davSupportedLockScope") + lockCountStr, scope );
1048  setMetaData( QLatin1String("davSupportedLockType") + lockCountStr, type );
1049  }
1050  }
1051  }
1052  }
1053  else if ( property.tagName() == QLatin1String("lockdiscovery") )
1054  {
1055  // Lists the available locks
1056  davParseActiveLocks( property.elementsByTagName( QLatin1String("activelock") ), lockCount );
1057  }
1058  else if ( property.tagName() == QLatin1String("resourcetype") )
1059  {
1060  // Resource type. "Specifies the nature of the resource."
1061  if ( !property.namedItem( QLatin1String("collection") ).toElement().isNull() )
1062  {
1063  // This is a collection (directory)
1064  isDirectory = true;
1065  }
1066  }
1067  else
1068  {
1069  kDebug(7113) << "Found unknown webdav property:" << property.tagName();
1070  }
1071  }
1072  }
1073 
1074  setMetaData( QLatin1String("davLockCount"), QString::number(lockCount) );
1075  setMetaData( QLatin1String("davSupportedLockCount"), QString::number(supportedLockCount) );
1076 
1077  entry.insert( KIO::UDSEntry::UDS_FILE_TYPE, isDirectory ? S_IFDIR : S_IFREG );
1078 
1079  if ( foundExecutable || isDirectory )
1080  {
1081  // File was executable, or is a directory.
1082  entry.insert( KIO::UDSEntry::UDS_ACCESS, 0700 );
1083  }
1084  else
1085  {
1086  entry.insert( KIO::UDSEntry::UDS_ACCESS, 0600 );
1087  }
1088 
1089  if ( !isDirectory && !mimeType.isEmpty() )
1090  {
1091  entry.insert( KIO::UDSEntry::UDS_MIME_TYPE, mimeType );
1092  }
1093 }
1094 
1095 void HTTPProtocol::davParseActiveLocks( const QDomNodeList& activeLocks,
1096  uint& lockCount )
1097 {
1098  for ( int i = 0; i < activeLocks.count(); i++ )
1099  {
1100  const QDomElement activeLock = activeLocks.item(i).toElement();
1101 
1102  lockCount++;
1103  // required
1104  const QDomElement lockScope = activeLock.namedItem( QLatin1String("lockscope") ).toElement();
1105  const QDomElement lockType = activeLock.namedItem( QLatin1String("locktype") ).toElement();
1106  const QDomElement lockDepth = activeLock.namedItem( QLatin1String("depth") ).toElement();
1107  // optional
1108  const QDomElement lockOwner = activeLock.namedItem( QLatin1String("owner") ).toElement();
1109  const QDomElement lockTimeout = activeLock.namedItem( QLatin1String("timeout") ).toElement();
1110  const QDomElement lockToken = activeLock.namedItem( QLatin1String("locktoken") ).toElement();
1111 
1112  if ( !lockScope.isNull() && !lockType.isNull() && !lockDepth.isNull() )
1113  {
1114  // lock was properly specified
1115  lockCount++;
1116  const QString lockCountStr = QString::number(lockCount);
1117  const QString scope = lockScope.firstChild().toElement().tagName();
1118  const QString type = lockType.firstChild().toElement().tagName();
1119  const QString depth = lockDepth.text();
1120 
1121  setMetaData( QLatin1String("davLockScope") + lockCountStr, scope );
1122  setMetaData( QLatin1String("davLockType") + lockCountStr, type );
1123  setMetaData( QLatin1String("davLockDepth") + lockCountStr, depth );
1124 
1125  if ( !lockOwner.isNull() )
1126  setMetaData( QLatin1String("davLockOwner") + lockCountStr, lockOwner.text() );
1127 
1128  if ( !lockTimeout.isNull() )
1129  setMetaData( QLatin1String("davLockTimeout") + lockCountStr, lockTimeout.text() );
1130 
1131  if ( !lockToken.isNull() )
1132  {
1133  QDomElement tokenVal = lockScope.namedItem( QLatin1String("href") ).toElement();
1134  if ( !tokenVal.isNull() )
1135  setMetaData( QLatin1String("davLockToken") + lockCountStr, tokenVal.text() );
1136  }
1137  }
1138  }
1139 }
1140 
1141 long HTTPProtocol::parseDateTime( const QString& input, const QString& type )
1142 {
1143  if ( type == QLatin1String("dateTime.tz") )
1144  {
1145  return KDateTime::fromString( input, KDateTime::ISODate ).toTime_t();
1146  }
1147  else if ( type == QLatin1String("dateTime.rfc1123") )
1148  {
1149  return KDateTime::fromString( input, KDateTime::RFCDate ).toTime_t();
1150  }
1151 
1152  // format not advertised... try to parse anyway
1153  time_t time = KDateTime::fromString( input, KDateTime::RFCDate ).toTime_t();
1154  if ( time != 0 )
1155  return time;
1156 
1157  return KDateTime::fromString( input, KDateTime::ISODate ).toTime_t();
1158 }
1159 
1160 QString HTTPProtocol::davProcessLocks()
1161 {
1162  if ( hasMetaData( QLatin1String("davLockCount") ) )
1163  {
1164  QString response = QLatin1String("If:");
1165  int numLocks = metaData( QLatin1String("davLockCount") ).toInt();
1166  bool bracketsOpen = false;
1167  for ( int i = 0; i < numLocks; i++ )
1168  {
1169  const QString countStr = QString::number(i);
1170  if ( hasMetaData( QLatin1String("davLockToken") + countStr ) )
1171  {
1172  if ( hasMetaData( QLatin1String("davLockURL") + countStr ) )
1173  {
1174  if ( bracketsOpen )
1175  {
1176  response += QLatin1Char(')');
1177  bracketsOpen = false;
1178  }
1179  response += QLatin1String(" <") + metaData( QLatin1String("davLockURL") + countStr ) + QLatin1Char('>');
1180  }
1181 
1182  if ( !bracketsOpen )
1183  {
1184  response += QLatin1String(" (");
1185  bracketsOpen = true;
1186  }
1187  else
1188  {
1189  response += QLatin1Char(' ');
1190  }
1191 
1192  if ( hasMetaData( QLatin1String("davLockNot") + countStr ) )
1193  response += QLatin1String("Not ");
1194 
1195  response += QLatin1Char('<') + metaData( QLatin1String("davLockToken") + countStr ) + QLatin1Char('>');
1196  }
1197  }
1198 
1199  if ( bracketsOpen )
1200  response += QLatin1Char(')');
1201 
1202  response += QLatin1String("\r\n");
1203  return response;
1204  }
1205 
1206  return QString();
1207 }
1208 
1209 bool HTTPProtocol::davHostOk()
1210 {
1211  // FIXME needs to be reworked. Switched off for now.
1212  return true;
1213 
1214  // cached?
1215  if ( m_davHostOk )
1216  {
1217  kDebug(7113) << "true";
1218  return true;
1219  }
1220  else if ( m_davHostUnsupported )
1221  {
1222  kDebug(7113) << " false";
1223  davError( -2 );
1224  return false;
1225  }
1226 
1227  m_request.method = HTTP_OPTIONS;
1228 
1229  // query the server's capabilities generally, not for a specific URL
1230  m_request.url.setPath(QLatin1String("*"));
1231  m_request.url.setQuery(QString());
1232  m_request.cacheTag.policy = CC_Reload;
1233 
1234  // clear davVersions variable, which holds the response to the DAV: header
1235  m_davCapabilities.clear();
1236 
1237  proceedUntilResponseHeader();
1238 
1239  if (m_davCapabilities.count())
1240  {
1241  for (int i = 0; i < m_davCapabilities.count(); i++)
1242  {
1243  bool ok;
1244  uint verNo = m_davCapabilities[i].toUInt(&ok);
1245  if (ok && verNo > 0 && verNo < 3)
1246  {
1247  m_davHostOk = true;
1248  kDebug(7113) << "Server supports DAV version" << verNo;
1249  }
1250  }
1251 
1252  if ( m_davHostOk )
1253  return true;
1254  }
1255 
1256  m_davHostUnsupported = true;
1257  davError( -2 );
1258  return false;
1259 }
1260 
1261 // This function is for closing proceedUntilResponseHeader(); requests
1262 // Required because there may or may not be further info expected
1263 void HTTPProtocol::davFinished()
1264 {
1265  // TODO: Check with the DAV extension developers
1266  httpClose(m_request.isKeepAlive);
1267  finished();
1268 }
1269 
1270 void HTTPProtocol::mkdir( const KUrl& url, int )
1271 {
1272  kDebug(7113) << url;
1273 
1274  if (!maybeSetRequestUrl(url))
1275  return;
1276  resetSessionSettings();
1277 
1278  m_request.method = DAV_MKCOL;
1279  m_request.url.setQuery(QString());
1280  m_request.cacheTag.policy = CC_Reload;
1281 
1282  proceedUntilResponseHeader();
1283 
1284  if ( m_request.responseCode == 201 )
1285  davFinished();
1286  else
1287  davError();
1288 }
1289 
1290 void HTTPProtocol::get( const KUrl& url )
1291 {
1292  kDebug(7113) << url;
1293 
1294  if (!maybeSetRequestUrl(url))
1295  return;
1296  resetSessionSettings();
1297 
1298  m_request.method = HTTP_GET;
1299 
1300  QString tmp(metaData(QLatin1String("cache")));
1301  if (!tmp.isEmpty())
1302  m_request.cacheTag.policy = parseCacheControl(tmp);
1303  else
1304  m_request.cacheTag.policy = DEFAULT_CACHE_CONTROL;
1305 
1306  proceedUntilResponseContent();
1307 }
1308 
1309 void HTTPProtocol::put( const KUrl &url, int, KIO::JobFlags flags )
1310 {
1311  kDebug(7113) << url;
1312 
1313  if (!maybeSetRequestUrl(url))
1314  return;
1315 
1316  resetSessionSettings();
1317 
1318  // Webdav hosts are capable of observing overwrite == false
1319  if (m_protocol.startsWith("webdav")) { // krazy:exclude=strings
1320  if (!(flags & KIO::Overwrite)) {
1321  // check to make sure this host supports WebDAV
1322  if (!davHostOk())
1323  return;
1324 
1325  const QByteArray request ("<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
1326  "<D:propfind xmlns:D=\"DAV:\"><D:prop>"
1327  "<D:creationdate/>"
1328  "<D:getcontentlength/>"
1329  "<D:displayname/>"
1330  "<D:resourcetype/>"
1331  "</D:prop></D:propfind>");
1332 
1333  davSetRequest( request );
1334 
1335  // WebDAV Stat or List...
1336  m_request.method = DAV_PROPFIND;
1337  m_request.url.setQuery(QString());
1338  m_request.cacheTag.policy = CC_Reload;
1339  m_request.davData.depth = 0;
1340 
1341  proceedUntilResponseContent(true);
1342 
1343  if (!m_request.isKeepAlive) {
1344  httpCloseConnection(); // close connection if server requested it.
1345  m_request.isKeepAlive = true; // reset the keep alive flag.
1346  }
1347 
1348  if (m_request.responseCode == 207) {
1349  error(ERR_FILE_ALREADY_EXIST, QString());
1350  return;
1351  }
1352 
1353  // force re-authentication...
1354  delete m_wwwAuth;
1355  m_wwwAuth = 0;
1356  }
1357  }
1358 
1359  m_request.method = HTTP_PUT;
1360  m_request.cacheTag.policy = CC_Reload;
1361 
1362  proceedUntilResponseContent();
1363 }
1364 
1365 void HTTPProtocol::copy( const KUrl& src, const KUrl& dest, int, KIO::JobFlags flags )
1366 {
1367  kDebug(7113) << src << "->" << dest;
1368 
1369  if (!maybeSetRequestUrl(dest) || !maybeSetRequestUrl(src))
1370  return;
1371  resetSessionSettings();
1372 
1373  // destination has to be "http(s)://..."
1374  KUrl newDest = dest;
1375  if (newDest.protocol() == QLatin1String("webdavs"))
1376  newDest.setProtocol(QLatin1String("https"));
1377  else if (newDest.protocol() == QLatin1String("webdav"))
1378  newDest.setProtocol(QLatin1String("http"));
1379 
1380  m_request.method = DAV_COPY;
1381  m_request.davData.desturl = newDest.url();
1382  m_request.davData.overwrite = (flags & KIO::Overwrite);
1383  m_request.url.setQuery(QString());
1384  m_request.cacheTag.policy = CC_Reload;
1385 
1386  proceedUntilResponseHeader();
1387 
1388  // The server returns a HTTP/1.1 201 Created or 204 No Content on successful completion
1389  if ( m_request.responseCode == 201 || m_request.responseCode == 204 )
1390  davFinished();
1391  else
1392  davError();
1393 }
1394 
1395 void HTTPProtocol::rename( const KUrl& src, const KUrl& dest, KIO::JobFlags flags )
1396 {
1397  kDebug(7113) << src << "->" << dest;
1398 
1399  if (!maybeSetRequestUrl(dest) || !maybeSetRequestUrl(src))
1400  return;
1401  resetSessionSettings();
1402 
1403  // destination has to be "http://..."
1404  KUrl newDest = dest;
1405  if (newDest.protocol() == QLatin1String("webdavs"))
1406  newDest.setProtocol(QLatin1String("https"));
1407  else if (newDest.protocol() == QLatin1String("webdav"))
1408  newDest.setProtocol(QLatin1String("http"));
1409 
1410  m_request.method = DAV_MOVE;
1411  m_request.davData.desturl = newDest.url();
1412  m_request.davData.overwrite = (flags & KIO::Overwrite);
1413  m_request.url.setQuery(QString());
1414  m_request.cacheTag.policy = CC_Reload;
1415 
1416  proceedUntilResponseHeader();
1417 
1418  // Work around strict Apache-2 WebDAV implementation which refuses to cooperate
1419  // with webdav://host/directory, instead requiring webdav://host/directory/
1420  // (strangely enough it accepts Destination: without a trailing slash)
1421  // See BR# 209508 and BR#187970
1422  if ( m_request.responseCode == 301) {
1423  m_request.url = m_request.redirectUrl;
1424  m_request.method = DAV_MOVE;
1425  m_request.davData.desturl = newDest.url();
1426  m_request.davData.overwrite = (flags & KIO::Overwrite);
1427  m_request.url.setQuery(QString());
1428  m_request.cacheTag.policy = CC_Reload;
1429  // force re-authentication...
1430  delete m_wwwAuth;
1431  m_wwwAuth = 0;
1432  proceedUntilResponseHeader();
1433  }
1434 
1435  if ( m_request.responseCode == 201 )
1436  davFinished();
1437  else
1438  davError();
1439 }
1440 
1441 void HTTPProtocol::del( const KUrl& url, bool )
1442 {
1443  kDebug(7113) << url;
1444 
1445  if (!maybeSetRequestUrl(url))
1446  return;
1447 
1448  resetSessionSettings();
1449 
1450  m_request.method = HTTP_DELETE;
1451  m_request.cacheTag.policy = CC_Reload;
1452 
1453  if (m_protocol.startsWith("webdav")) { //krazy:exclude=strings due to QByteArray
1454  m_request.url.setQuery(QString());
1455  if (!proceedUntilResponseHeader()) {
1456  return;
1457  }
1458 
1459  // The server returns a HTTP/1.1 200 Ok or HTTP/1.1 204 No Content
1460  // on successful completion.
1461  if ( m_request.responseCode == 200 || m_request.responseCode == 204 || m_isRedirection)
1462  davFinished();
1463  else
1464  davError();
1465 
1466  return;
1467  }
1468 
1469  proceedUntilResponseContent();
1470 }
1471 
1472 void HTTPProtocol::post( const KUrl& url, qint64 size )
1473 {
1474  kDebug(7113) << url;
1475 
1476  if (!maybeSetRequestUrl(url))
1477  return;
1478  resetSessionSettings();
1479 
1480  m_request.method = HTTP_POST;
1481  m_request.cacheTag.policy= CC_Reload;
1482 
1483  m_iPostDataSize = (size > -1 ? static_cast<KIO::filesize_t>(size) : NO_SIZE);
1484  proceedUntilResponseContent();
1485 }
1486 
1487 void HTTPProtocol::davLock( const KUrl& url, const QString& scope,
1488  const QString& type, const QString& owner )
1489 {
1490  kDebug(7113) << url;
1491 
1492  if (!maybeSetRequestUrl(url))
1493  return;
1494  resetSessionSettings();
1495 
1496  m_request.method = DAV_LOCK;
1497  m_request.url.setQuery(QString());
1498  m_request.cacheTag.policy= CC_Reload;
1499 
1500  /* Create appropriate lock XML request. */
1501  QDomDocument lockReq;
1502 
1503  QDomElement lockInfo = lockReq.createElementNS( QLatin1String("DAV:"), QLatin1String("lockinfo") );
1504  lockReq.appendChild( lockInfo );
1505 
1506  QDomElement lockScope = lockReq.createElement( QLatin1String("lockscope") );
1507  lockInfo.appendChild( lockScope );
1508 
1509  lockScope.appendChild( lockReq.createElement( scope ) );
1510 
1511  QDomElement lockType = lockReq.createElement( QLatin1String("locktype") );
1512  lockInfo.appendChild( lockType );
1513 
1514  lockType.appendChild( lockReq.createElement( type ) );
1515 
1516  if ( !owner.isNull() ) {
1517  QDomElement ownerElement = lockReq.createElement( QLatin1String("owner") );
1518  lockReq.appendChild( ownerElement );
1519 
1520  QDomElement ownerHref = lockReq.createElement( QLatin1String("href") );
1521  ownerElement.appendChild( ownerHref );
1522 
1523  ownerHref.appendChild( lockReq.createTextNode( owner ) );
1524  }
1525 
1526  // insert the document into the POST buffer
1527  cachePostData(lockReq.toByteArray());
1528 
1529  proceedUntilResponseContent( true );
1530 
1531  if ( m_request.responseCode == 200 ) {
1532  // success
1533  QDomDocument multiResponse;
1534  multiResponse.setContent( m_webDavDataBuf, true );
1535 
1536  QDomElement prop = multiResponse.documentElement().namedItem( QLatin1String("prop") ).toElement();
1537 
1538  QDomElement lockdiscovery = prop.namedItem( QLatin1String("lockdiscovery") ).toElement();
1539 
1540  uint lockCount = 0;
1541  davParseActiveLocks( lockdiscovery.elementsByTagName( QLatin1String("activelock") ), lockCount );
1542 
1543  setMetaData( QLatin1String("davLockCount"), QString::number( lockCount ) );
1544 
1545  finished();
1546 
1547  } else
1548  davError();
1549 }
1550 
1551 void HTTPProtocol::davUnlock( const KUrl& url )
1552 {
1553  kDebug(7113) << url;
1554 
1555  if (!maybeSetRequestUrl(url))
1556  return;
1557  resetSessionSettings();
1558 
1559  m_request.method = DAV_UNLOCK;
1560  m_request.url.setQuery(QString());
1561  m_request.cacheTag.policy= CC_Reload;
1562 
1563  proceedUntilResponseContent( true );
1564 
1565  if ( m_request.responseCode == 200 )
1566  finished();
1567  else
1568  davError();
1569 }
1570 
1571 QString HTTPProtocol::davError( int code /* = -1 */, const QString &_url )
1572 {
1573  bool callError = false;
1574  if ( code == -1 ) {
1575  code = m_request.responseCode;
1576  callError = true;
1577  }
1578  if ( code == -2 ) {
1579  callError = true;
1580  }
1581 
1582  QString url = _url;
1583  if ( !url.isNull() )
1584  url = m_request.url.prettyUrl();
1585 
1586  QString action, errorString;
1587  int errorCode = ERR_SLAVE_DEFINED;
1588 
1589  // for 412 Precondition Failed
1590  QString ow = i18n( "Otherwise, the request would have succeeded." );
1591 
1592  switch ( m_request.method ) {
1593  case DAV_PROPFIND:
1594  action = i18nc( "request type", "retrieve property values" );
1595  break;
1596  case DAV_PROPPATCH:
1597  action = i18nc( "request type", "set property values" );
1598  break;
1599  case DAV_MKCOL:
1600  action = i18nc( "request type", "create the requested folder" );
1601  break;
1602  case DAV_COPY:
1603  action = i18nc( "request type", "copy the specified file or folder" );
1604  break;
1605  case DAV_MOVE:
1606  action = i18nc( "request type", "move the specified file or folder" );
1607  break;
1608  case DAV_SEARCH:
1609  action = i18nc( "request type", "search in the specified folder" );
1610  break;
1611  case DAV_LOCK:
1612  action = i18nc( "request type", "lock the specified file or folder" );
1613  break;
1614  case DAV_UNLOCK:
1615  action = i18nc( "request type", "unlock the specified file or folder" );
1616  break;
1617  case HTTP_DELETE:
1618  action = i18nc( "request type", "delete the specified file or folder" );
1619  break;
1620  case HTTP_OPTIONS:
1621  action = i18nc( "request type", "query the server's capabilities" );
1622  break;
1623  case HTTP_GET:
1624  action = i18nc( "request type", "retrieve the contents of the specified file or folder" );
1625  break;
1626  case DAV_REPORT:
1627  action = i18nc( "request type", "run a report in the specified folder" );
1628  break;
1629  case HTTP_PUT:
1630  case HTTP_POST:
1631  case HTTP_HEAD:
1632  default:
1633  // this should not happen, this function is for webdav errors only
1634  Q_ASSERT(0);
1635  }
1636 
1637  // default error message if the following code fails
1638  errorString = i18nc("%1: code, %2: request type", "An unexpected error (%1) occurred "
1639  "while attempting to %2.", code, action);
1640 
1641  switch ( code )
1642  {
1643  case -2:
1644  // internal error: OPTIONS request did not specify DAV compliance
1645  // ERR_UNSUPPORTED_PROTOCOL
1646  errorString = i18n("The server does not support the WebDAV protocol.");
1647  break;
1648  case 207:
1649  // 207 Multi-status
1650  {
1651  // our error info is in the returned XML document.
1652  // retrieve the XML document
1653 
1654  // there was an error retrieving the XML document.
1655  // ironic, eh?
1656  if ( !readBody( true ) && m_iError )
1657  return QString();
1658 
1659  QStringList errors;
1660  QDomDocument multiResponse;
1661 
1662  multiResponse.setContent( m_webDavDataBuf, true );
1663 
1664  QDomElement multistatus = multiResponse.documentElement().namedItem( QLatin1String("multistatus") ).toElement();
1665 
1666  QDomNodeList responses = multistatus.elementsByTagName( QLatin1String("response") );
1667 
1668  for (int i = 0; i < responses.count(); i++)
1669  {
1670  int errCode;
1671  QString errUrl;
1672 
1673  QDomElement response = responses.item(i).toElement();
1674  QDomElement code = response.namedItem( QLatin1String("status") ).toElement();
1675 
1676  if ( !code.isNull() )
1677  {
1678  errCode = codeFromResponse( code.text() );
1679  QDomElement href = response.namedItem( QLatin1String("href") ).toElement();
1680  if ( !href.isNull() )
1681  errUrl = href.text();
1682  errors << davError( errCode, errUrl );
1683  }
1684  }
1685 
1686  //kError = ERR_SLAVE_DEFINED;
1687  errorString = i18nc( "%1: request type, %2: url",
1688  "An error occurred while attempting to %1, %2. A "
1689  "summary of the reasons is below.", action, url );
1690 
1691  errorString += QLatin1String("<ul>");
1692 
1693  Q_FOREACH(const QString& error, errors)
1694  errorString += QLatin1String("<li>") + error + QLatin1String("</li>");
1695 
1696  errorString += QLatin1String("</ul>");
1697  }
1698  case 403:
1699  case 500: // hack: Apache mod_dav returns this instead of 403 (!)
1700  // 403 Forbidden
1701  // ERR_ACCESS_DENIED
1702  errorString = i18nc( "%1: request type", "Access was denied while attempting to %1.", action );
1703  break;
1704  case 405:
1705  // 405 Method Not Allowed
1706  if ( m_request.method == DAV_MKCOL ) {
1707  // ERR_DIR_ALREADY_EXIST
1708  errorString = url;
1709  errorCode = ERR_DIR_ALREADY_EXIST;
1710  }
1711  break;
1712  case 409:
1713  // 409 Conflict
1714  // ERR_ACCESS_DENIED
1715  errorString = i18n("A resource cannot be created at the destination "
1716  "until one or more intermediate collections (folders) "
1717  "have been created.");
1718  break;
1719  case 412:
1720  // 412 Precondition failed
1721  if ( m_request.method == DAV_COPY || m_request.method == DAV_MOVE ) {
1722  // ERR_ACCESS_DENIED
1723  errorString = i18n("The server was unable to maintain the liveness of "
1724  "the properties listed in the propertybehavior XML "
1725  "element or you attempted to overwrite a file while "
1726  "requesting that files are not overwritten. %1",
1727  ow );
1728 
1729  } else if ( m_request.method == DAV_LOCK ) {
1730  // ERR_ACCESS_DENIED
1731  errorString = i18n("The requested lock could not be granted. %1", ow );
1732  }
1733  break;
1734  case 415:
1735  // 415 Unsupported Media Type
1736  // ERR_ACCESS_DENIED
1737  errorString = i18n("The server does not support the request type of the body.");
1738  break;
1739  case 423:
1740  // 423 Locked
1741  // ERR_ACCESS_DENIED
1742  errorString = i18nc( "%1: request type", "Unable to %1 because the resource is locked.", action );
1743  break;
1744  case 425:
1745  // 424 Failed Dependency
1746  errorString = i18n("This action was prevented by another error.");
1747  break;
1748  case 502:
1749  // 502 Bad Gateway
1750  if ( m_request.method == DAV_COPY || m_request.method == DAV_MOVE ) {
1751  // ERR_WRITE_ACCESS_DENIED
1752  errorString = i18nc( "%1: request type", "Unable to %1 because the destination server refuses "
1753  "to accept the file or folder.", action );
1754  }
1755  break;
1756  case 507:
1757  // 507 Insufficient Storage
1758  // ERR_DISK_FULL
1759  errorString = i18n("The destination resource does not have sufficient space "
1760  "to record the state of the resource after the execution "
1761  "of this method.");
1762  break;
1763  default:
1764  break;
1765  }
1766 
1767  // if ( kError != ERR_SLAVE_DEFINED )
1768  //errorString += " (" + url + ')';
1769 
1770  if ( callError )
1771  error( errorCode, errorString );
1772 
1773  return errorString;
1774 }
1775 
1776 // HTTP generic error
1777 static int httpGenericError(const HTTPProtocol::HTTPRequest& request, QString* errorString)
1778 {
1779  Q_ASSERT(errorString);
1780 
1781  int errorCode = 0;
1782  errorString->clear();
1783 
1784  if (request.responseCode == 204) {
1785  errorCode = ERR_NO_CONTENT;
1786  }
1787 
1788  return errorCode;
1789 }
1790 
1791 // HTTP DELETE specific errors
1792 static int httpDelError(const HTTPProtocol::HTTPRequest& request, QString* errorString)
1793 {
1794  Q_ASSERT(errorString);
1795 
1796  int errorCode = 0;
1797  const int responseCode = request.responseCode;
1798  errorString->clear();
1799 
1800  switch (responseCode) {
1801  case 204:
1802  errorCode = ERR_NO_CONTENT;
1803  break;
1804  default:
1805  break;
1806  }
1807 
1808  if (!errorCode
1809  && (responseCode < 200 || responseCode > 400)
1810  && responseCode != 404) {
1811  errorCode = ERR_SLAVE_DEFINED;
1812  *errorString = i18n( "The resource cannot be deleted." );
1813  }
1814 
1815  return errorCode;
1816 }
1817 
1818 // HTTP PUT specific errors
1819 static int httpPutError(const HTTPProtocol::HTTPRequest& request, QString* errorString)
1820 {
1821  Q_ASSERT(errorString);
1822 
1823  int errorCode = 0;
1824  const int responseCode = request.responseCode;
1825  const QString action (i18nc("request type", "upload %1", request.url.prettyUrl()));
1826 
1827  switch (responseCode) {
1828  case 403:
1829  case 405:
1830  case 500: // hack: Apache mod_dav returns this instead of 403 (!)
1831  // 403 Forbidden
1832  // 405 Method Not Allowed
1833  // ERR_ACCESS_DENIED
1834  *errorString = i18nc( "%1: request type", "Access was denied while attempting to %1.", action );
1835  errorCode = ERR_SLAVE_DEFINED;
1836  break;
1837  case 409:
1838  // 409 Conflict
1839  // ERR_ACCESS_DENIED
1840  *errorString = i18n("A resource cannot be created at the destination "
1841  "until one or more intermediate collections (folders) "
1842  "have been created.");
1843  errorCode = ERR_SLAVE_DEFINED;
1844  break;
1845  case 423:
1846  // 423 Locked
1847  // ERR_ACCESS_DENIED
1848  *errorString = i18nc( "%1: request type", "Unable to %1 because the resource is locked.", action );
1849  errorCode = ERR_SLAVE_DEFINED;
1850  break;
1851  case 502:
1852  // 502 Bad Gateway
1853  // ERR_WRITE_ACCESS_DENIED;
1854  *errorString = i18nc( "%1: request type", "Unable to %1 because the destination server refuses "
1855  "to accept the file or folder.", action );
1856  errorCode = ERR_SLAVE_DEFINED;
1857  break;
1858  case 507:
1859  // 507 Insufficient Storage
1860  // ERR_DISK_FULL
1861  *errorString = i18n("The destination resource does not have sufficient space "
1862  "to record the state of the resource after the execution "
1863  "of this method.");
1864  errorCode = ERR_SLAVE_DEFINED;
1865  break;
1866  default:
1867  break;
1868  }
1869 
1870  if (!errorCode
1871  && (responseCode < 200 || responseCode > 400)
1872  && responseCode != 404) {
1873  errorCode = ERR_SLAVE_DEFINED;
1874  *errorString = i18nc("%1: response code, %2: request type",
1875  "An unexpected error (%1) occurred while attempting to %2.",
1876  responseCode, action);
1877  }
1878 
1879  return errorCode;
1880 }
1881 
1882 bool HTTPProtocol::sendHttpError()
1883 {
1884  QString errorString;
1885  int errorCode = 0;
1886 
1887  switch (m_request.method) {
1888  case HTTP_GET:
1889  case HTTP_POST:
1890  errorCode = httpGenericError(m_request, &errorString);
1891  break;
1892  case HTTP_PUT:
1893  errorCode = httpPutError(m_request, &errorString);
1894  break;
1895  case HTTP_DELETE:
1896  errorCode = httpDelError(m_request, &errorString);
1897  break;
1898  default:
1899  break;
1900  }
1901 
1902  // Force any message previously shown by the client to be cleared.
1903  infoMessage(QLatin1String(""));
1904 
1905  if (errorCode) {
1906  error( errorCode, errorString );
1907  return true;
1908  }
1909 
1910  return false;
1911 }
1912 
1913 bool HTTPProtocol::sendErrorPageNotification()
1914 {
1915  if (!m_request.preferErrorPage)
1916  return false;
1917 
1918  if (m_isLoadingErrorPage)
1919  kWarning(7113) << "called twice during one request, something is probably wrong.";
1920 
1921  m_isLoadingErrorPage = true;
1922  SlaveBase::errorPage();
1923  return true;
1924 }
1925 
1926 bool HTTPProtocol::isOffline()
1927 {
1928  // ### TEMPORARY WORKAROUND (While investigating why solid may
1929  // produce false positives)
1930  return false;
1931 
1932  Solid::Networking::Status status = Solid::Networking::status();
1933 
1934  kDebug(7113) << "networkstatus:" << status;
1935 
1936  // on error or unknown, we assume online
1937  return status == Solid::Networking::Unconnected;
1938 }
1939 
1940 void HTTPProtocol::multiGet(const QByteArray &data)
1941 {
1942  QDataStream stream(data);
1943  quint32 n;
1944  stream >> n;
1945 
1946  kDebug(7113) << n;
1947 
1948  HTTPRequest saveRequest;
1949  if (m_isBusy)
1950  saveRequest = m_request;
1951 
1952  resetSessionSettings();
1953 
1954  for (unsigned i = 0; i < n; ++i) {
1955  KUrl url;
1956  stream >> url >> mIncomingMetaData;
1957 
1958  if (!maybeSetRequestUrl(url))
1959  continue;
1960 
1961  //### should maybe call resetSessionSettings() if the server/domain is
1962  // different from the last request!
1963 
1964  kDebug(7113) << url;
1965 
1966  m_request.method = HTTP_GET;
1967  m_request.isKeepAlive = true; //readResponseHeader clears it if necessary
1968 
1969  QString tmp = metaData(QLatin1String("cache"));
1970  if (!tmp.isEmpty())
1971  m_request.cacheTag.policy= parseCacheControl(tmp);
1972  else
1973  m_request.cacheTag.policy= DEFAULT_CACHE_CONTROL;
1974 
1975  m_requestQueue.append(m_request);
1976  }
1977 
1978  if (m_isBusy)
1979  m_request = saveRequest;
1980 #if 0
1981  if (!m_isBusy) {
1982  m_isBusy = true;
1983  QMutableListIterator<HTTPRequest> it(m_requestQueue);
1984  while (it.hasNext()) {
1985  m_request = it.next();
1986  it.remove();
1987  proceedUntilResponseContent();
1988  }
1989  m_isBusy = false;
1990  }
1991 #endif
1992  if (!m_isBusy) {
1993  m_isBusy = true;
1994  QMutableListIterator<HTTPRequest> it(m_requestQueue);
1995  // send the requests
1996  while (it.hasNext()) {
1997  m_request = it.next();
1998  sendQuery();
1999  // save the request state so we can pick it up again in the collection phase
2000  it.setValue(m_request);
2001  kDebug(7113) << "check one: isKeepAlive =" << m_request.isKeepAlive;
2002  if (m_request.cacheTag.ioMode != ReadFromCache) {
2003  m_server.initFrom(m_request);
2004  }
2005  }
2006  // collect the responses
2007  //### for the moment we use a hack: instead of saving and restoring request-id
2008  // we just count up like ParallelGetJobs does.
2009  int requestId = 0;
2010  Q_FOREACH (const HTTPRequest &r, m_requestQueue) {
2011  m_request = r;
2012  kDebug(7113) << "check two: isKeepAlive =" << m_request.isKeepAlive;
2013  setMetaData(QLatin1String("request-id"), QString::number(requestId++));
2014  sendAndKeepMetaData();
2015  if (!(readResponseHeader() && readBody())) {
2016  return;
2017  }
2018  // the "next job" signal for ParallelGetJob is data of size zero which
2019  // readBody() sends without our intervention.
2020  kDebug(7113) << "check three: isKeepAlive =" << m_request.isKeepAlive;
2021  httpClose(m_request.isKeepAlive); //actually keep-alive is mandatory for pipelining
2022  }
2023 
2024  finished();
2025  m_requestQueue.clear();
2026  m_isBusy = false;
2027  }
2028 }
2029 
2030 ssize_t HTTPProtocol::write (const void *_buf, size_t nbytes)
2031 {
2032  size_t sent = 0;
2033  const char* buf = static_cast<const char*>(_buf);
2034  while (sent < nbytes)
2035  {
2036  int n = TCPSlaveBase::write(buf + sent, nbytes - sent);
2037 
2038  if (n < 0) {
2039  // some error occurred
2040  return -1;
2041  }
2042 
2043  sent += n;
2044  }
2045 
2046  return sent;
2047 }
2048 
2049 void HTTPProtocol::clearUnreadBuffer()
2050 {
2051  m_unreadBuf.clear();
2052 }
2053 
2054 // Note: the implementation of unread/readBuffered assumes that unread will only
2055 // be used when there is extra data we don't want to handle, and not to wait for more data.
2056 void HTTPProtocol::unread(char *buf, size_t size)
2057 {
2058  // implement LIFO (stack) semantics
2059  const int newSize = m_unreadBuf.size() + size;
2060  m_unreadBuf.resize(newSize);
2061  for (size_t i = 0; i < size; i++) {
2062  m_unreadBuf.data()[newSize - i - 1] = buf[i];
2063  }
2064  if (size) {
2065  //hey, we still have data, closed connection or not!
2066  m_isEOF = false;
2067  }
2068 }
2069 
2070 size_t HTTPProtocol::readBuffered(char *buf, size_t size, bool unlimited)
2071 {
2072  size_t bytesRead = 0;
2073  if (!m_unreadBuf.isEmpty()) {
2074  const int bufSize = m_unreadBuf.size();
2075  bytesRead = qMin((int)size, bufSize);
2076 
2077  for (size_t i = 0; i < bytesRead; i++) {
2078  buf[i] = m_unreadBuf.constData()[bufSize - i - 1];
2079  }
2080  m_unreadBuf.truncate(bufSize - bytesRead);
2081 
2082  // If we have an unread buffer and the size of the content returned by the
2083  // server is unknown, e.g. chuncked transfer, return the bytes read here since
2084  // we may already have enough data to complete the response and don't want to
2085  // wait for more. See BR# 180631.
2086  if (unlimited)
2087  return bytesRead;
2088  }
2089  if (bytesRead < size) {
2090  int rawRead = TCPSlaveBase::read(buf + bytesRead, size - bytesRead);
2091  if (rawRead < 1) {
2092  m_isEOF = true;
2093  return bytesRead;
2094  }
2095  bytesRead += rawRead;
2096  }
2097  return bytesRead;
2098 }
2099 
2100 //### this method will detect an n*(\r\n) sequence if it crosses invocations.
2101 // it will look (n*2 - 1) bytes before start at most and never before buf, naturally.
2102 // supported number of newlines are one and two, in line with HTTP syntax.
2103 // return true if numNewlines newlines were found.
2104 bool HTTPProtocol::readDelimitedText(char *buf, int *idx, int end, int numNewlines)
2105 {
2106  Q_ASSERT(numNewlines >=1 && numNewlines <= 2);
2107  char mybuf[64]; //somewhere close to the usual line length to avoid unread()ing too much
2108  int pos = *idx;
2109  while (pos < end && !m_isEOF) {
2110  int step = qMin((int)sizeof(mybuf), end - pos);
2111  if (m_isChunked) {
2112  //we might be reading the end of the very last chunk after which there is no data.
2113  //don't try to read any more bytes than there are because it causes stalls
2114  //(yes, it shouldn't stall but it does)
2115  step = 1;
2116  }
2117  size_t bufferFill = readBuffered(mybuf, step);
2118 
2119  for (size_t i = 0; i < bufferFill ; ++i, ++pos) {
2120  // we copy the data from mybuf to buf immediately and look for the newlines in buf.
2121  // that way we don't miss newlines split over several invocations of this method.
2122  buf[pos] = mybuf[i];
2123 
2124  // did we just copy one or two times the (usually) \r\n delimiter?
2125  // until we find even more broken webservers in the wild let's assume that they either
2126  // send \r\n (RFC compliant) or \n (broken) as delimiter...
2127  if (buf[pos] == '\n') {
2128  bool found = numNewlines == 1;
2129  if (!found) { // looking for two newlines
2130  // Detect \n\n and \n\r\n. The other cases (\r\n\n, \r\n\r\n) are covered by the first two.
2131  found = ((pos >= 1 && buf[pos - 1] == '\n') ||
2132  (pos >= 2 && buf[pos - 2] == '\n' && buf[pos - 1] == '\r'));
2133  }
2134  if (found) {
2135  i++; // unread bytes *after* CRLF
2136  unread(&mybuf[i], bufferFill - i);
2137  *idx = pos + 1;
2138  return true;
2139  }
2140  }
2141  }
2142  }
2143  *idx = pos;
2144  return false;
2145 }
2146 
2147 static bool isCompatibleNextUrl(const KUrl &previous, const KUrl &now)
2148 {
2149  if (previous.host() != now.host() || previous.port() != now.port()) {
2150  return false;
2151  }
2152  if (previous.user().isEmpty() && previous.pass().isEmpty()) {
2153  return true;
2154  }
2155  return previous.user() == now.user() && previous.pass() == now.pass();
2156 }
2157 
2158 bool HTTPProtocol::httpShouldCloseConnection()
2159 {
2160  kDebug(7113);
2161 
2162  if (!isConnected()) {
2163  return false;
2164  }
2165 
2166  if (!m_request.proxyUrls.isEmpty() && !isAutoSsl()) {
2167  Q_FOREACH(const QString& url, m_request.proxyUrls) {
2168  if (url != QLatin1String("DIRECT")) {
2169  if (isCompatibleNextUrl(m_server.proxyUrl, KUrl(url))) {
2170  return false;
2171  }
2172  }
2173  }
2174  return true;
2175  }
2176 
2177  return !isCompatibleNextUrl(m_server.url, m_request.url);
2178 }
2179 
2180 bool HTTPProtocol::httpOpenConnection()
2181 {
2182  kDebug(7113);
2183  m_server.clear();
2184 
2185  // Only save proxy auth information after proxy authentication has
2186  // actually taken place, which will set up exactly this connection.
2187  disconnect(socket(), SIGNAL(connected()),
2188  this, SLOT(saveProxyAuthenticationForSocket()));
2189 
2190  clearUnreadBuffer();
2191 
2192  int connectError = 0;
2193  QString errorString;
2194 
2195  // Get proxy information...
2196  if (m_request.proxyUrls.isEmpty()) {
2197  m_request.proxyUrls = config()->readEntry("ProxyUrls", QStringList());
2198  kDebug(7113) << "Proxy URLs:" << m_request.proxyUrls;
2199  }
2200 
2201  if (m_request.proxyUrls.isEmpty()) {
2202  connectError = connectToHost(m_request.url.host(), m_request.url.port(defaultPort()), &errorString);
2203  } else {
2204  KUrl::List badProxyUrls;
2205  Q_FOREACH(const QString& proxyUrl, m_request.proxyUrls) {
2206  const KUrl url (proxyUrl);
2207  const QString scheme (url.protocol());
2208 
2209  if (!supportedProxyScheme(scheme)) {
2210  connectError = ERR_COULD_NOT_CONNECT;
2211  errorString = url.url();
2212  continue;
2213  }
2214 
2215  const bool isDirectConnect = (proxyUrl == QLatin1String("DIRECT"));
2216  QNetworkProxy::ProxyType proxyType = QNetworkProxy::NoProxy;
2217  if (url.protocol() == QLatin1String("socks")) {
2218  proxyType = QNetworkProxy::Socks5Proxy;
2219  } else if (!isDirectConnect && isAutoSsl()) {
2220  proxyType = QNetworkProxy::HttpProxy;
2221  }
2222 
2223  kDebug(7113) << "Connecting to proxy: address=" << proxyUrl << "type=" << proxyType;
2224 
2225  if (proxyType == QNetworkProxy::NoProxy) {
2226  // Only way proxy url and request url are the same is when the
2227  // proxy URL list contains a "DIRECT" entry. See resetSessionSettings().
2228  if (isDirectConnect) {
2229  connectError = connectToHost(m_request.url.host(), m_request.url.port(defaultPort()), &errorString);
2230  kDebug(7113) << "Connected DIRECT: host=" << m_request.url.host() << "post=" << m_request.url.port(defaultPort());
2231  } else {
2232  connectError = connectToHost(url.host(), url.port(), &errorString);
2233  if (connectError == 0) {
2234  m_request.proxyUrl = url;
2235  kDebug(7113) << "Connected to proxy: host=" << url.host() << "port=" << url.port();
2236  } else {
2237  if (connectError == ERR_UNKNOWN_HOST)
2238  connectError = ERR_UNKNOWN_PROXY_HOST;
2239  kDebug(7113) << "Failed to connect to proxy:" << proxyUrl;
2240  badProxyUrls << url;
2241  }
2242  }
2243  if (connectError == 0) {
2244  break;
2245  }
2246  } else {
2247  QNetworkProxy proxy (proxyType, url.host(), url.port(), url.user(), url.pass());
2248  QNetworkProxy::setApplicationProxy(proxy);
2249  connectError = connectToHost(m_request.url.host(), m_request.url.port(defaultPort()), &errorString);
2250  if (connectError == 0) {
2251  kDebug(7113) << "Tunneling thru proxy: host=" << url.host() << "port=" << url.port();
2252  break;
2253  } else {
2254  if (connectError == ERR_UNKNOWN_HOST)
2255  connectError = ERR_UNKNOWN_PROXY_HOST;
2256  kDebug(7113) << "Failed to connect to proxy:" << proxyUrl;
2257  badProxyUrls << url;
2258  QNetworkProxy::setApplicationProxy(QNetworkProxy::NoProxy);
2259  }
2260  }
2261  }
2262 
2263  if (!badProxyUrls.isEmpty()) {
2264  //TODO: Notify the client of BAD proxy addresses (needed for PAC setups).
2265  }
2266  }
2267 
2268  if (connectError != 0) {
2269  error (connectError, errorString);
2270  return false;
2271  }
2272 
2273  // Disable Nagle's algorithm, i.e turn on TCP_NODELAY.
2274  KTcpSocket *sock = qobject_cast<KTcpSocket*>(socket());
2275  if (sock) {
2276  // kDebug(7113) << "TCP_NODELAY:" << sock->socketOption(QAbstractSocket::LowDelayOption);
2277  sock->setSocketOption(QAbstractSocket::LowDelayOption, 1);
2278  }
2279 
2280  m_server.initFrom(m_request);
2281  connected();
2282  return true;
2283 }
2284 
2285 bool HTTPProtocol::satisfyRequestFromCache(bool *cacheHasPage)
2286 {
2287  kDebug(7113);
2288 
2289  if (m_request.cacheTag.useCache) {
2290  const bool offline = isOffline();
2291 
2292  if (offline && m_request.cacheTag.policy != KIO::CC_Reload) {
2293  m_request.cacheTag.policy= KIO::CC_CacheOnly;
2294  }
2295 
2296  const bool isCacheOnly = m_request.cacheTag.policy == KIO::CC_CacheOnly;
2297  const CacheTag::CachePlan plan = m_request.cacheTag.plan(m_maxCacheAge);
2298 
2299  bool openForReading = false;
2300  if (plan == CacheTag::UseCached || plan == CacheTag::ValidateCached) {
2301  openForReading = cacheFileOpenRead();
2302 
2303  if (!openForReading && (isCacheOnly || offline)) {
2304  // cache-only or offline -> we give a definite answer and it is "no"
2305  *cacheHasPage = false;
2306  if (isCacheOnly) {
2307  error(ERR_DOES_NOT_EXIST, m_request.url.url());
2308  } else if (offline) {
2309  error(ERR_COULD_NOT_CONNECT, m_request.url.url());
2310  }
2311  return true;
2312  }
2313  }
2314 
2315  if (openForReading) {
2316  m_request.cacheTag.ioMode = ReadFromCache;
2317  *cacheHasPage = true;
2318  // return false if validation is required, so a network request will be sent
2319  return m_request.cacheTag.plan(m_maxCacheAge) == CacheTag::UseCached;
2320  }
2321  }
2322  *cacheHasPage = false;
2323  return false;
2324 }
2325 
2326 QString HTTPProtocol::formatRequestUri() const
2327 {
2328  // Only specify protocol, host and port when they are not already clear, i.e. when
2329  // we handle HTTP proxying ourself and the proxy server needs to know them.
2330  // Sending protocol/host/port in other cases confuses some servers, and it's not their fault.
2331  if (isHttpProxy(m_request.proxyUrl) && !isAutoSsl()) {
2332  KUrl u;
2333 
2334  QString protocol = m_request.url.protocol();
2335  if (protocol.startsWith(QLatin1String("webdav"))) {
2336  protocol.replace(0, qstrlen("webdav"), QLatin1String("http"));
2337  }
2338  u.setProtocol(protocol);
2339 
2340  u.setHost(m_request.url.host());
2341  // if the URL contained the default port it should have been stripped earlier
2342  Q_ASSERT(m_request.url.port() != defaultPort());
2343  u.setPort(m_request.url.port());
2344  u.setEncodedPathAndQuery(m_request.url.encodedPathAndQuery(
2345  KUrl::LeaveTrailingSlash, KUrl::AvoidEmptyPath));
2346  return u.url();
2347  } else {
2348  return m_request.url.encodedPathAndQuery(KUrl::LeaveTrailingSlash, KUrl::AvoidEmptyPath);
2349  }
2350 }
2351 
2367 bool HTTPProtocol::sendQuery()
2368 {
2369  kDebug(7113);
2370 
2371  // Cannot have an https request without autoSsl! This can
2372  // only happen if the current installation does not support SSL...
2373  if (isEncryptedHttpVariety(m_protocol) && !isAutoSsl()) {
2374  error(ERR_UNSUPPORTED_PROTOCOL, toQString(m_protocol));
2375  return false;
2376  }
2377 
2378  // Check the reusability of the current connection.
2379  if (httpShouldCloseConnection()) {
2380  httpCloseConnection();
2381  }
2382 
2383  // Create a new connection to the remote machine if we do
2384  // not already have one...
2385  // NB: the !m_socketProxyAuth condition is a workaround for a proxied Qt socket sometimes
2386  // looking disconnected after receiving the initial 407 response.
2387  // I guess the Qt socket fails to hide the effect of proxy-connection: close after receiving
2388  // the 407 header.
2389  if ((!isConnected() && !m_socketProxyAuth))
2390  {
2391  if (!httpOpenConnection())
2392  {
2393  kDebug(7113) << "Couldn't connect, oopsie!";
2394  return false;
2395  }
2396  }
2397 
2398  m_request.cacheTag.ioMode = NoCache;
2399  m_request.cacheTag.servedDate = -1;
2400  m_request.cacheTag.lastModifiedDate = -1;
2401  m_request.cacheTag.expireDate = -1;
2402 
2403  QString header;
2404 
2405  bool hasBodyData = false;
2406  bool hasDavData = false;
2407 
2408  {
2409  header = toQString(m_request.methodString());
2410  QString davHeader;
2411 
2412  // Fill in some values depending on the HTTP method to guide further processing
2413  switch (m_request.method)
2414  {
2415  case HTTP_GET: {
2416  bool cacheHasPage = false;
2417  if (satisfyRequestFromCache(&cacheHasPage)) {
2418  kDebug(7113) << "cacheHasPage =" << cacheHasPage;
2419  return cacheHasPage;
2420  }
2421  if (!cacheHasPage) {
2422  // start a new cache file later if appropriate
2423  m_request.cacheTag.ioMode = WriteToCache;
2424  }
2425  break;
2426  }
2427  case HTTP_HEAD:
2428  break;
2429  case HTTP_PUT:
2430  case HTTP_POST:
2431  hasBodyData = true;
2432  break;
2433  case HTTP_DELETE:
2434  case HTTP_OPTIONS:
2435  break;
2436  case DAV_PROPFIND:
2437  hasDavData = true;
2438  davHeader = QLatin1String("Depth: ");
2439  if ( hasMetaData( QLatin1String("davDepth") ) )
2440  {
2441  kDebug(7113) << "Reading DAV depth from metadata:" << metaData( QLatin1String("davDepth") );
2442  davHeader += metaData( QLatin1String("davDepth") );
2443  }
2444  else
2445  {
2446  if ( m_request.davData.depth == 2 )
2447  davHeader += QLatin1String("infinity");
2448  else
2449  davHeader += QString::number( m_request.davData.depth );
2450  }
2451  davHeader += QLatin1String("\r\n");
2452  break;
2453  case DAV_PROPPATCH:
2454  hasDavData = true;
2455  break;
2456  case DAV_MKCOL:
2457  break;
2458  case DAV_COPY:
2459  case DAV_MOVE:
2460  davHeader = QLatin1String("Destination: ") + m_request.davData.desturl;
2461  // infinity depth means copy recursively
2462  // (optional for copy -> but is the desired action)
2463  davHeader += QLatin1String("\r\nDepth: infinity\r\nOverwrite: ");
2464  davHeader += QLatin1Char(m_request.davData.overwrite ? 'T' : 'F');
2465  davHeader += QLatin1String("\r\n");
2466  break;
2467  case DAV_LOCK:
2468  davHeader = QLatin1String("Timeout: ");
2469  {
2470  uint timeout = 0;
2471  if ( hasMetaData( QLatin1String("davTimeout") ) )
2472  timeout = metaData( QLatin1String("davTimeout") ).toUInt();
2473  if ( timeout == 0 )
2474  davHeader += QLatin1String("Infinite");
2475  else
2476  davHeader += QLatin1String("Seconds-") + QString::number(timeout);
2477  }
2478  davHeader += QLatin1String("\r\n");
2479  hasDavData = true;
2480  break;
2481  case DAV_UNLOCK:
2482  davHeader = QLatin1String("Lock-token: ") + metaData(QLatin1String("davLockToken")) + QLatin1String("\r\n");
2483  break;
2484  case DAV_SEARCH:
2485  case DAV_REPORT:
2486  hasDavData = true;
2487  /* fall through */
2488  case DAV_SUBSCRIBE:
2489  case DAV_UNSUBSCRIBE:
2490  case DAV_POLL:
2491  break;
2492  default:
2493  error (ERR_UNSUPPORTED_ACTION, QString());
2494  return false;
2495  }
2496  // DAV_POLL; DAV_NOTIFY
2497 
2498  header += formatRequestUri() + QLatin1String(" HTTP/1.1\r\n"); /* start header */
2499 
2500  /* support for virtual hosts and required by HTTP 1.1 */
2501  header += QLatin1String("Host: ") + m_request.encoded_hostname;
2502  if (m_request.url.port(defaultPort()) != defaultPort()) {
2503  header += QLatin1Char(':') + QString::number(m_request.url.port());
2504  }
2505  header += QLatin1String("\r\n");
2506 
2507  // Support old HTTP/1.0 style keep-alive header for compatibility
2508  // purposes as well as performance improvements while giving end
2509  // users the ability to disable this feature for proxy servers that
2510  // don't support it, e.g. junkbuster proxy server.
2511  if (isHttpProxy(m_request.proxyUrl) && !isAutoSsl()) {
2512  header += QLatin1String("Proxy-Connection: ");
2513  } else {
2514  header += QLatin1String("Connection: ");
2515  }
2516  if (m_request.isKeepAlive) {
2517  header += QLatin1String("keep-alive\r\n");
2518  } else {
2519  header += QLatin1String("close\r\n");
2520  }
2521 
2522  if (!m_request.userAgent.isEmpty())
2523  {
2524  header += QLatin1String("User-Agent: ");
2525  header += m_request.userAgent;
2526  header += QLatin1String("\r\n");
2527  }
2528 
2529  if (!m_request.referrer.isEmpty())
2530  {
2531  header += QLatin1String("Referer: "); //Don't try to correct spelling!
2532  header += m_request.referrer;
2533  header += QLatin1String("\r\n");
2534  }
2535 
2536  if ( m_request.endoffset > m_request.offset )
2537  {
2538  header += QLatin1String("Range: bytes=");
2539  header += KIO::number(m_request.offset);
2540  header += QLatin1Char('-');
2541  header += KIO::number(m_request.endoffset);
2542  header += QLatin1String("\r\n");
2543  kDebug(7103) << "kio_http : Range =" << KIO::number(m_request.offset)
2544  << "-" << KIO::number(m_request.endoffset);
2545  }
2546  else if ( m_request.offset > 0 && m_request.endoffset == 0 )
2547  {
2548  header += QLatin1String("Range: bytes=");
2549  header += KIO::number(m_request.offset);
2550  header += QLatin1String("-\r\n");
2551  kDebug(7103) << "kio_http: Range =" << KIO::number(m_request.offset);
2552  }
2553 
2554  if ( !m_request.cacheTag.useCache || m_request.cacheTag.policy==CC_Reload )
2555  {
2556  /* No caching for reload */
2557  header += QLatin1String("Pragma: no-cache\r\n"); /* for HTTP/1.0 caches */
2558  header += QLatin1String("Cache-control: no-cache\r\n"); /* for HTTP >=1.1 caches */
2559  }
2560  else if (m_request.cacheTag.plan(m_maxCacheAge) == CacheTag::ValidateCached)
2561  {
2562  kDebug(7113) << "needs validation, performing conditional get.";
2563  /* conditional get */
2564  if (!m_request.cacheTag.etag.isEmpty())
2565  header += QLatin1String("If-None-Match: ") + m_request.cacheTag.etag + QLatin1String("\r\n");
2566 
2567  if (m_request.cacheTag.lastModifiedDate != -1) {
2568  const QString httpDate = formatHttpDate(m_request.cacheTag.lastModifiedDate);
2569  header += QLatin1String("If-Modified-Since: ") + httpDate + QLatin1String("\r\n");
2570  setMetaData(QLatin1String("modified"), httpDate);
2571  }
2572  }
2573 
2574  header += QLatin1String("Accept: ");
2575  const QString acceptHeader = metaData(QLatin1String("accept"));
2576  if (!acceptHeader.isEmpty())
2577  header += acceptHeader;
2578  else
2579  header += QLatin1String(DEFAULT_ACCEPT_HEADER);
2580  header += QLatin1String("\r\n");
2581 
2582  if (m_request.allowTransferCompression)
2583  header += QLatin1String("Accept-Encoding: gzip, deflate, x-gzip, x-deflate\r\n");
2584 
2585  if (!m_request.charsets.isEmpty())
2586  header += QLatin1String("Accept-Charset: ") + m_request.charsets + QLatin1String("\r\n");
2587 
2588  if (!m_request.languages.isEmpty())
2589  header += QLatin1String("Accept-Language: ") + m_request.languages + QLatin1String("\r\n");
2590 
2591  QString cookieStr;
2592  const QString cookieMode = metaData(QLatin1String("cookies")).toLower();
2593 
2594  if (cookieMode == QLatin1String("none"))
2595  {
2596  m_request.cookieMode = HTTPRequest::CookiesNone;
2597  }
2598  else if (cookieMode == QLatin1String("manual"))
2599  {
2600  m_request.cookieMode = HTTPRequest::CookiesManual;
2601  cookieStr = metaData(QLatin1String("setcookies"));
2602  }
2603  else
2604  {
2605  m_request.cookieMode = HTTPRequest::CookiesAuto;
2606  if (m_request.useCookieJar)
2607  cookieStr = findCookies(m_request.url.url());
2608  }
2609 
2610  if (!cookieStr.isEmpty())
2611  header += cookieStr + QLatin1String("\r\n");
2612 
2613  const QString customHeader = metaData( QLatin1String("customHTTPHeader") );
2614  if (!customHeader.isEmpty())
2615  {
2616  header += sanitizeCustomHTTPHeader(customHeader);
2617  header += QLatin1String("\r\n");
2618  }
2619 
2620  const QString contentType = metaData(QLatin1String("content-type"));
2621  if (!contentType.isEmpty())
2622  {
2623  if (!contentType.startsWith(QLatin1String("content-type"), Qt::CaseInsensitive))
2624  header += QLatin1String("Content-Type: ");
2625  header += contentType;
2626  header += QLatin1String("\r\n");
2627  }
2628 
2629  // DoNotTrack feature...
2630  if (config()->readEntry("DoNotTrack", false))
2631  header += QLatin1String("DNT: 1\r\n");
2632 
2633  // Remember that at least one failed (with 401 or 407) request/response
2634  // roundtrip is necessary for the server to tell us that it requires
2635  // authentication. However, we proactively add authentication headers if when
2636  // we have cached credentials to avoid the extra roundtrip where possible.
2637  header += authenticationHeader();
2638 
2639  if ( m_protocol == "webdav" || m_protocol == "webdavs" )
2640  {
2641  header += davProcessLocks();
2642 
2643  // add extra webdav headers, if supplied
2644  davHeader += metaData(QLatin1String("davHeader"));
2645 
2646  // Set content type of webdav data
2647  if (hasDavData)
2648  davHeader += QLatin1String("Content-Type: text/xml; charset=utf-8\r\n");
2649 
2650  // add extra header elements for WebDAV
2651  header += davHeader;
2652  }
2653  }
2654 
2655  kDebug(7103) << "============ Sending Header:";
2656  Q_FOREACH (const QString &s, header.split(QLatin1String("\r\n"), QString::SkipEmptyParts)) {
2657  kDebug(7103) << s;
2658  }
2659 
2660  // End the header iff there is no payload data. If we do have payload data
2661  // sendBody() will add another field to the header, Content-Length.
2662  if (!hasBodyData && !hasDavData)
2663  header += QLatin1String("\r\n");
2664 
2665 
2666  // Now that we have our formatted header, let's send it!
2667 
2668  // Clear out per-connection settings...
2669  resetConnectionSettings();
2670 
2671  // Send the data to the remote machine...
2672  ssize_t written = write(header.toLatin1(), header.length());
2673  bool sendOk = (written == (ssize_t) header.length());
2674  if (!sendOk)
2675  {
2676  kDebug(7113) << "Connection broken! (" << m_request.url.host() << ")"
2677  << " -- intended to write" << header.length()
2678  << "bytes but wrote" << (int)written << ".";
2679 
2680  // The server might have closed the connection due to a timeout, or maybe
2681  // some transport problem arose while the connection was idle.
2682  if (m_request.isKeepAlive)
2683  {
2684  httpCloseConnection();
2685  return true; // Try again
2686  }
2687 
2688  kDebug(7113) << "sendOk == false. Connection broken !"
2689  << " -- intended to write" << header.length()
2690  << "bytes but wrote" << (int)written << ".";
2691  error( ERR_CONNECTION_BROKEN, m_request.url.host() );
2692  return false;
2693  }
2694  else
2695  kDebug(7113) << "sent it!";
2696 
2697  bool res = true;
2698  if (hasBodyData || hasDavData)
2699  res = sendBody();
2700 
2701  infoMessage(i18n("%1 contacted. Waiting for reply...", m_request.url.host()));
2702 
2703  return res;
2704 }
2705 
2706 void HTTPProtocol::forwardHttpResponseHeader(bool forwardImmediately)
2707 {
2708  // Send the response header if it was requested...
2709  if (!config()->readEntry("PropagateHttpHeader", false))
2710  return;
2711 
2712  setMetaData(QLatin1String("HTTP-Headers"), m_responseHeaders.join(QString(QLatin1Char('\n'))));
2713 
2714  if (forwardImmediately)
2715  sendMetaData();
2716 }
2717 
2718 bool HTTPProtocol::parseHeaderFromCache()
2719 {
2720  kDebug(7113);
2721  if (!cacheFileReadTextHeader2()) {
2722  return false;
2723  }
2724 
2725  Q_FOREACH (const QString &str, m_responseHeaders) {
2726  const QString header = str.trimmed();
2727  if (header.startsWith(QLatin1String("content-type:"), Qt::CaseInsensitive)) {
2728  int pos = header.indexOf(QLatin1String("charset="), Qt::CaseInsensitive);
2729  if (pos != -1) {
2730  const QString charset = header.mid(pos + 8).toLower();
2731  m_request.cacheTag.charset = charset;
2732  setMetaData(QLatin1String("charset"), charset);
2733  }
2734  } else if (header.startsWith(QLatin1String("content-language:"), Qt::CaseInsensitive)) {
2735  const QString language = header.mid(17).trimmed().toLower();
2736  setMetaData(QLatin1String("content-language"), language);
2737  } else if (header.startsWith(QLatin1String("content-disposition:"), Qt::CaseInsensitive)) {
2738  parseContentDisposition(header.mid(20).toLower());
2739  }
2740  }
2741 
2742  if (m_request.cacheTag.lastModifiedDate != -1) {
2743  setMetaData(QLatin1String("modified"), formatHttpDate(m_request.cacheTag.lastModifiedDate));
2744  }
2745 
2746  // this header comes from the cache, so the response must have been cacheable :)
2747  setCacheabilityMetadata(true);
2748  kDebug(7113) << "Emitting mimeType" << m_mimeType;
2749  forwardHttpResponseHeader(false);
2750  mimeType(m_mimeType);
2751  // IMPORTANT: Do not remove the call below or the http response headers will
2752  // not be available to the application if this slave is put on hold.
2753  forwardHttpResponseHeader();
2754  return true;
2755 }
2756 
2757 void HTTPProtocol::fixupResponseMimetype()
2758 {
2759  if (m_mimeType.isEmpty())
2760  return;
2761 
2762  kDebug(7113) << "before fixup" << m_mimeType;
2763  // Convert some common mimetypes to standard mimetypes
2764  if (m_mimeType == QLatin1String("application/x-targz"))
2765  m_mimeType = QLatin1String("application/x-compressed-tar");
2766  else if (m_mimeType == QLatin1String("image/x-png"))
2767  m_mimeType = QLatin1String("image/png");
2768  else if (m_mimeType == QLatin1String("audio/x-mp3") || m_mimeType == QLatin1String("audio/x-mpeg") || m_mimeType == QLatin1String("audio/mp3"))
2769  m_mimeType = QLatin1String("audio/mpeg");
2770  else if (m_mimeType == QLatin1String("audio/microsoft-wave"))
2771  m_mimeType = QLatin1String("audio/x-wav");
2772  else if (m_mimeType == QLatin1String("image/x-ms-bmp"))
2773  m_mimeType = QLatin1String("image/bmp");
2774 
2775  // Crypto ones....
2776  else if (m_mimeType == QLatin1String("application/pkix-cert") ||
2777  m_mimeType == QLatin1String("application/binary-certificate")) {
2778  m_mimeType = QLatin1String("application/x-x509-ca-cert");
2779  }
2780 
2781  // Prefer application/x-compressed-tar or x-gzpostscript over application/x-gzip.
2782  else if (m_mimeType == QLatin1String("application/x-gzip")) {
2783  if ((m_request.url.path().endsWith(QLatin1String(".tar.gz"))) ||
2784  (m_request.url.path().endsWith(QLatin1String(".tar"))))
2785  m_mimeType = QLatin1String("application/x-compressed-tar");
2786  if ((m_request.url.path().endsWith(QLatin1String(".ps.gz"))))
2787  m_mimeType = QLatin1String("application/x-gzpostscript");
2788  }
2789 
2790  // Prefer application/x-xz-compressed-tar over application/x-xz for LMZA compressed
2791  // tar files. Arch Linux AUR servers notoriously send the wrong mimetype for this.
2792  else if(m_mimeType == QLatin1String("application/x-xz")) {
2793  if (m_request.url.path().endsWith(QLatin1String(".tar.xz")) ||
2794  m_request.url.path().endsWith(QLatin1String(".txz"))) {
2795  m_mimeType = QLatin1String("application/x-xz-compressed-tar");
2796  }
2797  }
2798 
2799  // Some webservers say "text/plain" when they mean "application/x-bzip"
2800  else if ((m_mimeType == QLatin1String("text/plain")) || (m_mimeType == QLatin1String("application/octet-stream"))) {
2801  const QString ext = QFileInfo(m_request.url.path()).suffix().toUpper();
2802  if (ext == QLatin1String("BZ2"))
2803  m_mimeType = QLatin1String("application/x-bzip");
2804  else if (ext == QLatin1String("PEM"))
2805  m_mimeType = QLatin1String("application/x-x509-ca-cert");
2806  else if (ext == QLatin1String("SWF"))
2807  m_mimeType = QLatin1String("application/x-shockwave-flash");
2808  else if (ext == QLatin1String("PLS"))
2809  m_mimeType = QLatin1String("audio/x-scpls");
2810  else if (ext == QLatin1String("WMV"))
2811  m_mimeType = QLatin1String("video/x-ms-wmv");
2812  else if (ext == QLatin1String("WEBM"))
2813  m_mimeType = QLatin1String("video/webm");
2814  else if (ext == QLatin1String("DEB"))
2815  m_mimeType = QLatin1String("application/x-deb");
2816  }
2817  kDebug(7113) << "after fixup" << m_mimeType;
2818 }
2819 
2820 
2821 void HTTPProtocol::fixupResponseContentEncoding()
2822 {
2823  // WABA: Correct for tgz files with a gzip-encoding.
2824  // They really shouldn't put gzip in the Content-Encoding field!
2825  // Web-servers really shouldn't do this: They let Content-Size refer
2826  // to the size of the tgz file, not to the size of the tar file,
2827  // while the Content-Type refers to "tar" instead of "tgz".
2828  if (!m_contentEncodings.isEmpty() && m_contentEncodings.last() == QLatin1String("gzip")) {
2829  if (m_mimeType == QLatin1String("application/x-tar")) {
2830  m_contentEncodings.removeLast();
2831  m_mimeType = QLatin1String("application/x-compressed-tar");
2832  } else if (m_mimeType == QLatin1String("application/postscript")) {
2833  // LEONB: Adding another exception for psgz files.
2834  // Could we use the mimelnk files instead of hardcoding all this?
2835  m_contentEncodings.removeLast();
2836  m_mimeType = QLatin1String("application/x-gzpostscript");
2837  } else if ((m_request.allowTransferCompression &&
2838  m_mimeType == QLatin1String("text/html"))
2839  ||
2840  (m_request.allowTransferCompression &&
2841  m_mimeType != QLatin1String("application/x-compressed-tar") &&
2842  m_mimeType != QLatin1String("application/x-tgz") && // deprecated name
2843  m_mimeType != QLatin1String("application/x-targz") && // deprecated name
2844  m_mimeType != QLatin1String("application/x-gzip"))) {
2845  // Unzip!
2846  } else {
2847  m_contentEncodings.removeLast();
2848  m_mimeType = QLatin1String("application/x-gzip");
2849  }
2850  }
2851 
2852  // We can't handle "bzip2" encoding (yet). So if we get something with
2853  // bzip2 encoding, we change the mimetype to "application/x-bzip".
2854  // Note for future changes: some web-servers send both "bzip2" as
2855  // encoding and "application/x-bzip[2]" as mimetype. That is wrong.
2856  // currently that doesn't bother us, because we remove the encoding
2857  // and set the mimetype to x-bzip anyway.
2858  if (!m_contentEncodings.isEmpty() && m_contentEncodings.last() == QLatin1String("bzip2")) {
2859  m_contentEncodings.removeLast();
2860  m_mimeType = QLatin1String("application/x-bzip");
2861  }
2862 }
2863 
2864 //Return true if the term was found, false otherwise. Advance *pos.
2865 //If (*pos + strlen(term) >= end) just advance *pos to end and return false.
2866 //This means that users should always search for the shortest terms first.
2867 static bool consume(const char input[], int *pos, int end, const char *term)
2868 {
2869  // note: gcc/g++ is quite good at optimizing away redundant strlen()s
2870  int idx = *pos;
2871  if (idx + (int)strlen(term) >= end) {
2872  *pos = end;
2873  return false;
2874  }
2875  if (strncasecmp(&input[idx], term, strlen(term)) == 0) {
2876  *pos = idx + strlen(term);
2877  return true;
2878  }
2879  return false;
2880 }
2881 
2888 bool HTTPProtocol::readResponseHeader()
2889 {
2890  resetResponseParsing();
2891  if (m_request.cacheTag.ioMode == ReadFromCache &&
2892  m_request.cacheTag.plan(m_maxCacheAge) == CacheTag::UseCached) {
2893  // parseHeaderFromCache replaces this method in case of cached content
2894  return parseHeaderFromCache();
2895  }
2896 
2897 try_again:
2898  kDebug(7113);
2899 
2900  bool upgradeRequired = false; // Server demands that we upgrade to something
2901  // This is also true if we ask to upgrade and
2902  // the server accepts, since we are now
2903  // committed to doing so
2904  bool noHeadersFound = false;
2905 
2906  m_request.cacheTag.charset.clear();
2907  m_responseHeaders.clear();
2908 
2909  static const int maxHeaderSize = 128 * 1024;
2910 
2911  char buffer[maxHeaderSize];
2912  bool cont = false;
2913  bool bCanResume = false;
2914 
2915  if (!isConnected()) {
2916  kDebug(7113) << "No connection.";
2917  return false; // Reestablish connection and try again
2918  }
2919 
2920 #if 0
2921  // NOTE: This is unnecessary since TCPSlaveBase::read does the same exact
2922  // thing. Plus, if we are unable to read from the socket we need to resend
2923  // the request as done below, not error out! Do not assume remote server
2924  // will honor persistent connections!!
2925  if (!waitForResponse(m_remoteRespTimeout)) {
2926  kDebug(7113) << "Got socket error:" << socket()->errorString();
2927  // No response error
2928  error(ERR_SERVER_TIMEOUT , m_request.url.host());
2929  return false;
2930  }
2931 #endif
2932 
2933  int bufPos = 0;
2934  bool foundDelimiter = readDelimitedText(buffer, &bufPos, maxHeaderSize, 1);
2935  if (!foundDelimiter && bufPos < maxHeaderSize) {
2936  kDebug(7113) << "EOF while waiting for header start.";
2937  if (m_request.isKeepAlive) {
2938  // Try to reestablish connection.
2939  httpCloseConnection();
2940  return false; // Reestablish connection and try again.
2941  }
2942 
2943  if (m_request.method == HTTP_HEAD) {
2944  // HACK
2945  // Some web-servers fail to respond properly to a HEAD request.
2946  // We compensate for their failure to properly implement the HTTP standard
2947  // by assuming that they will be sending html.
2948  kDebug(7113) << "HEAD -> returned mimetype:" << DEFAULT_MIME_TYPE;
2949  mimeType(QLatin1String(DEFAULT_MIME_TYPE));
2950  return true;
2951  }
2952 
2953  kDebug(7113) << "Connection broken !";
2954  error( ERR_CONNECTION_BROKEN, m_request.url.host() );
2955  return false;
2956  }
2957  if (!foundDelimiter) {
2958  //### buffer too small for first line of header(!)
2959  Q_ASSERT(0);
2960  }
2961 
2962  kDebug(7103) << "============ Received Status Response:";
2963  kDebug(7103) << QByteArray(buffer, bufPos).trimmed();
2964 
2965  HTTP_REV httpRev = HTTP_None;
2966  int idx = 0;
2967 
2968  if (idx != bufPos && buffer[idx] == '<') {
2969  kDebug(7103) << "No valid HTTP header found! Document starts with XML/HTML tag";
2970  // document starts with a tag, assume HTML instead of text/plain
2971  m_mimeType = QLatin1String("text/html");
2972  m_request.responseCode = 200; // Fake it
2973  httpRev = HTTP_Unknown;
2974  m_request.isKeepAlive = false;
2975  noHeadersFound = true;
2976  // put string back
2977  unread(buffer, bufPos);
2978  goto endParsing;
2979  }
2980 
2981  // "HTTP/1.1" or similar
2982  if (consume(buffer, &idx, bufPos, "ICY ")) {
2983  httpRev = SHOUTCAST;
2984  m_request.isKeepAlive = false;
2985  } else if (consume(buffer, &idx, bufPos, "HTTP/")) {
2986  if (consume(buffer, &idx, bufPos, "1.0")) {
2987  httpRev = HTTP_10;
2988  m_request.isKeepAlive = false;
2989  } else if (consume(buffer, &idx, bufPos, "1.1")) {
2990  httpRev = HTTP_11;
2991  }
2992  }
2993 
2994  if (httpRev == HTTP_None && bufPos != 0) {
2995  // Remote server does not seem to speak HTTP at all
2996  // Put the crap back into the buffer and hope for the best
2997  kDebug(7113) << "DO NOT WANT." << bufPos;
2998  unread(buffer, bufPos);
2999  if (m_request.responseCode) {
3000  m_request.prevResponseCode = m_request.responseCode;
3001  }
3002  m_request.responseCode = 200; // Fake it
3003  httpRev = HTTP_Unknown;
3004  m_request.isKeepAlive = false;
3005  noHeadersFound = true;
3006  goto endParsing;
3007  }
3008 
3009  // response code //### maybe wrong if we need several iterations for this response...
3010  //### also, do multiple iterations (cf. try_again) to parse one header work w/ pipelining?
3011  if (m_request.responseCode) {
3012  m_request.prevResponseCode = m_request.responseCode;
3013  }
3014  skipSpace(buffer, &idx, bufPos);
3015  //TODO saner handling of invalid response code strings
3016  if (idx != bufPos) {
3017  m_request.responseCode = atoi(&buffer[idx]);
3018  } else {
3019  m_request.responseCode = 200;
3020  }
3021  // move idx to start of (yet to be fetched) next line, skipping the "OK"
3022  idx = bufPos;
3023  // (don't bother parsing the "OK", what do we do if it isn't there anyway?)
3024 
3025  // immediately act on most response codes...
3026 
3027  // Protect users against bogus username intended to fool them into visiting
3028  // sites they had no intention of visiting.
3029  if (isPotentialSpoofingAttack(m_request, config())) {
3030  // kDebug(7113) << "**** POTENTIAL ADDRESS SPOOFING:" << m_request.url;
3031  const int result =
3032  messageBox(WarningYesNo,
3033  i18nc("@info Security check on url being accessed",
3034  "<p>You are about to log in to the site \"%1\" "
3035  "with the username \"%2\", but the website "
3036  "does not require authentication. "
3037  "This may be an attempt to trick you.</p>"
3038  "<p>Is \"%1\" the site you want to visit?</p>",
3039  m_request.url.host(), m_request.url.user()),
3040  i18nc("@title:window", "Confirm Website Access"));
3041  if (result == KMessageBox::No) {
3042  error(ERR_USER_CANCELED, m_request.url.url());
3043  return false;
3044  }
3045  setMetaData(QLatin1String("{internal~currenthost}LastSpoofedUserName"), m_request.url.user());
3046  }
3047 
3048  if (m_request.responseCode != 200 && m_request.responseCode != 304) {
3049  m_request.cacheTag.ioMode = NoCache;
3050  }
3051 
3052  if (m_request.responseCode >= 500 && m_request.responseCode <= 599) {
3053  // Server side errors
3054 
3055  if (m_request.method == HTTP_HEAD) {
3056  ; // Ignore error
3057  } else {
3058  if (!sendErrorPageNotification()) {
3059  error(ERR_INTERNAL_SERVER, m_request.url.prettyUrl());
3060  return false;
3061  }
3062  }
3063  } else if (m_request.responseCode == 416) {
3064  // Range not supported
3065  m_request.offset = 0;
3066  return false; // Try again.
3067  } else if (m_request.responseCode == 426) {
3068  // Upgrade Required
3069  upgradeRequired = true;
3070  } else if (!isAuthenticationRequired(m_request.responseCode) && m_request.responseCode >= 400 && m_request.responseCode <= 499) {
3071  // Any other client errors
3072  // Tell that we will only get an error page here.
3073  if (!sendErrorPageNotification()) {
3074  if (m_request.responseCode == 403)
3075  error(ERR_ACCESS_DENIED, m_request.url.prettyUrl());
3076  else
3077  error(ERR_DOES_NOT_EXIST, m_request.url.prettyUrl());
3078  return false;
3079  }
3080  } else if (m_request.responseCode >= 301 && m_request.responseCode<= 303) {
3081  // 301 Moved permanently
3082  if (m_request.responseCode == 301) {
3083  setMetaData(QLatin1String("permanent-redirect"), QLatin1String("true"));
3084  }
3085  // 302 Found (temporary location)
3086  // 303 See Other
3087  // NOTE: This is wrong according to RFC 2616 (section 10.3.[2-4,8]).
3088  // However, because almost all client implementations treat a 301/302
3089  // response as a 303 response in violation of the spec, many servers
3090  // have simply adapted to this way of doing things! Thus, we are
3091  // forced to do the same thing. Otherwise, we loose compatibility and
3092  // might not be able to correctly retrieve sites that redirect.
3093  if (m_request.method != HTTP_HEAD) {
3094  m_request.method = HTTP_GET; // Force a GET
3095  }
3096  } else if (m_request.responseCode == 204) {
3097  // No content
3098 
3099  // error(ERR_NO_CONTENT, i18n("Data have been successfully sent."));
3100  // Short circuit and do nothing!
3101 
3102  // The original handling here was wrong, this is not an error: eg. in the
3103  // example of a 204 No Content response to a PUT completing.
3104  // m_iError = true;
3105  // return false;
3106  } else if (m_request.responseCode == 206) {
3107  if (m_request.offset) {
3108  bCanResume = true;
3109  }
3110  } else if (m_request.responseCode == 102) {
3111  // Processing (for WebDAV)
3112  /***
3113  * This status code is given when the server expects the
3114  * command to take significant time to complete. So, inform
3115  * the user.
3116  */
3117  infoMessage( i18n( "Server processing request, please wait..." ) );
3118  cont = true;
3119  } else if (m_request.responseCode == 100) {
3120  // We got 'Continue' - ignore it
3121  cont = true;
3122  }
3123 
3124 endParsing:
3125  bool authRequiresAnotherRoundtrip = false;
3126 
3127  // Skip the whole header parsing if we got no HTTP headers at all
3128  if (!noHeadersFound) {
3129  // Auth handling
3130  const bool wasAuthError = isAuthenticationRequired(m_request.prevResponseCode);
3131  const bool isAuthError = isAuthenticationRequired(m_request.responseCode);
3132  const bool sameAuthError = (m_request.responseCode == m_request.prevResponseCode);
3133  kDebug(7113) << "wasAuthError=" << wasAuthError << "isAuthError=" << isAuthError
3134  << "sameAuthError=" << sameAuthError;
3135  // Not the same authorization error as before and no generic error?
3136  // -> save the successful credentials.
3137  if (wasAuthError && (m_request.responseCode < 400 || (isAuthError && !sameAuthError))) {
3138  saveAuthenticationData();
3139  }
3140 
3141  // done with the first line; now tokenize the other lines
3142 
3143  // TODO review use of STRTOLL vs. QByteArray::toInt()
3144 
3145  foundDelimiter = readDelimitedText(buffer, &bufPos, maxHeaderSize, 2);
3146  kDebug(7113) << " -- full response:" << endl << QByteArray(buffer, bufPos).trimmed();
3147  // Use this to see newlines:
3148  //kDebug(7113) << " -- full response:" << endl << QByteArray(buffer, bufPos).replace("\r", "\\r").replace("\n", "\\n\n");
3149  Q_ASSERT(foundDelimiter);
3150 
3151  //NOTE because tokenizer will overwrite newlines in case of line continuations in the header
3152  // unread(buffer, bufSize) will not generally work anymore. we don't need it either.
3153  // either we have a http response line -> try to parse the header, fail if it doesn't work
3154  // or we have garbage -> fail.
3155  HeaderTokenizer tokenizer(buffer);
3156  tokenizer.tokenize(idx, sizeof(buffer));
3157 
3158  // Note that not receiving "accept-ranges" means that all bets are off
3159  // wrt the server supporting ranges.
3160  TokenIterator tIt = tokenizer.iterator("accept-ranges");
3161  if (tIt.hasNext() && tIt.next().toLower().startsWith("none")) { // krazy:exclude=strings
3162  bCanResume = false;
3163  }
3164 
3165  tIt = tokenizer.iterator("keep-alive");
3166  while (tIt.hasNext()) {
3167  QByteArray ka = tIt.next().trimmed().toLower();
3168  if (ka.startsWith("timeout=")) { // krazy:exclude=strings
3169  int ka_timeout = ka.mid(qstrlen("timeout=")).trimmed().toInt();
3170  if (ka_timeout > 0)
3171  m_request.keepAliveTimeout = ka_timeout;
3172  if (httpRev == HTTP_10) {
3173  m_request.isKeepAlive = true;
3174  }
3175 
3176  break; // we want to fetch ka timeout only
3177  }
3178  }
3179 
3180  // get the size of our data
3181  tIt = tokenizer.iterator("content-length");
3182  if (tIt.hasNext()) {
3183  m_iSize = STRTOLL(tIt.next().constData(), 0, 10);
3184  }
3185 
3186  tIt = tokenizer.iterator("content-location");
3187  if (tIt.hasNext()) {
3188  setMetaData(QLatin1String("content-location"), toQString(tIt.next().trimmed()));
3189  }
3190 
3191  // which type of data do we have?
3192  QString mediaValue;
3193  QString mediaAttribute;
3194  tIt = tokenizer.iterator("content-type");
3195  if (tIt.hasNext()) {
3196  QList<QByteArray> l = tIt.next().split(';');
3197  if (!l.isEmpty()) {
3198  // Assign the mime-type.
3199  m_mimeType = toQString(l.first().trimmed().toLower());
3200  if (m_mimeType.startsWith(QLatin1Char('"'))) {
3201  m_mimeType.remove(0, 1);
3202  }
3203  if (m_mimeType.endsWith(QLatin1Char('"'))) {
3204  m_mimeType.chop(1);
3205  }
3206  kDebug(7113) << "Content-type:" << m_mimeType;
3207  l.removeFirst();
3208  }
3209 
3210  // If we still have text, then it means we have a mime-type with a
3211  // parameter (eg: charset=iso-8851) ; so let's get that...
3212  Q_FOREACH (const QByteArray &statement, l) {
3213  const int index = statement.indexOf('=');
3214  if (index <= 0) {
3215  mediaAttribute = toQString(statement.mid(0, index));
3216  } else {
3217  mediaAttribute = toQString(statement.mid(0, index));
3218  mediaValue = toQString(statement.mid(index+1));
3219  }
3220  mediaAttribute = mediaAttribute.trimmed();
3221  mediaValue = mediaValue.trimmed();
3222 
3223  bool quoted = false;
3224  if (mediaValue.startsWith(QLatin1Char('"'))) {
3225  quoted = true;
3226  mediaValue.remove(0, 1);
3227  }
3228 
3229  if (mediaValue.endsWith(QLatin1Char('"'))) {
3230  mediaValue.chop(1);
3231  }
3232 
3233  kDebug (7113) << "Encoding-type:" << mediaAttribute << "=" << mediaValue;
3234 
3235  if (mediaAttribute == QLatin1String("charset")) {
3236  mediaValue = mediaValue.toLower();
3237  m_request.cacheTag.charset = mediaValue;
3238  setMetaData(QLatin1String("charset"), mediaValue);
3239  } else {
3240  setMetaData(QLatin1String("media-") + mediaAttribute, mediaValue);
3241  if (quoted) {
3242  setMetaData(QLatin1String("media-") + mediaAttribute + QLatin1String("-kio-quoted"),
3243  QLatin1String("true"));
3244  }
3245  }
3246  }
3247  }
3248 
3249  // content?
3250  tIt = tokenizer.iterator("content-encoding");
3251  while (tIt.hasNext()) {
3252  // This is so wrong !! No wonder kio_http is stripping the
3253  // gzip encoding from downloaded files. This solves multiple
3254  // bug reports and caitoo's problem with downloads when such a
3255  // header is encountered...
3256 
3257  // A quote from RFC 2616:
3258  // " When present, its (Content-Encoding) value indicates what additional
3259  // content have been applied to the entity body, and thus what decoding
3260  // mechanism must be applied to obtain the media-type referenced by the
3261  // Content-Type header field. Content-Encoding is primarily used to allow
3262  // a document to be compressed without loosing the identity of its underlying
3263  // media type. Simply put if it is specified, this is the actual mime-type
3264  // we should use when we pull the resource !!!
3265  addEncoding(toQString(tIt.next()), m_contentEncodings);
3266  }
3267  // Refer to RFC 2616 sec 15.5/19.5.1 and RFC 2183
3268  tIt = tokenizer.iterator("content-disposition");
3269  if (tIt.hasNext()) {
3270  parseContentDisposition(toQString(tIt.next()));
3271  }
3272  tIt = tokenizer.iterator("content-language");
3273  if (tIt.hasNext()) {
3274  QString language = toQString(tIt.next().trimmed());
3275  if (!language.isEmpty()) {
3276  setMetaData(QLatin1String("content-language"), language);
3277  }
3278  }
3279 
3280  tIt = tokenizer.iterator("proxy-connection");
3281  if (tIt.hasNext() && isHttpProxy(m_request.proxyUrl) && !isAutoSsl()) {
3282  QByteArray pc = tIt.next().toLower();
3283  if (pc.startsWith("close")) { // krazy:exclude=strings
3284  m_request.isKeepAlive = false;
3285  } else if (pc.startsWith("keep-alive")) { // krazy:exclude=strings
3286  m_request.isKeepAlive = true;
3287  }
3288  }
3289 
3290  tIt = tokenizer.iterator("link");
3291  if (tIt.hasNext()) {
3292  // We only support Link: <url>; rel="type" so far
3293  QStringList link = toQString(tIt.next()).split(QLatin1Char(';'), QString::SkipEmptyParts);
3294  if (link.count() == 2) {
3295  QString rel = link[1].trimmed();
3296  if (rel.startsWith(QLatin1String("rel=\""))) {
3297  rel = rel.mid(5, rel.length() - 6);
3298  if (rel.toLower() == QLatin1String("pageservices")) {
3299  //### the remove() part looks fishy!
3300  QString url = link[0].remove(QRegExp(QLatin1String("[<>]"))).trimmed();
3301  setMetaData(QLatin1String("PageServices"), url);
3302  }
3303  }
3304  }
3305  }
3306 
3307  tIt = tokenizer.iterator("p3p");
3308  if (tIt.hasNext()) {
3309  // P3P privacy policy information
3310  QStringList policyrefs, compact;
3311  while (tIt.hasNext()) {
3312  QStringList policy = toQString(tIt.next().simplified())
3313  .split(QLatin1Char('='), QString::SkipEmptyParts);
3314  if (policy.count() == 2) {
3315  if (policy[0].toLower() == QLatin1String("policyref")) {
3316  policyrefs << policy[1].remove(QRegExp(QLatin1String("[\")\']"))).trimmed();
3317  } else if (policy[0].toLower() == QLatin1String("cp")) {
3318  // We convert to cp\ncp\ncp\n[...]\ncp to be consistent with
3319  // other metadata sent in strings. This could be a bit more
3320  // efficient but I'm going for correctness right now.
3321  const QString s = policy[1].remove(QRegExp(QLatin1String("[\")\']")));
3322  const QStringList cps = s.split(QLatin1Char(' '), QString::SkipEmptyParts);
3323  compact << cps;
3324  }
3325  }
3326  }
3327  if (!policyrefs.isEmpty()) {
3328  setMetaData(QLatin1String("PrivacyPolicy"), policyrefs.join(QLatin1String("\n")));
3329  }
3330  if (!compact.isEmpty()) {
3331  setMetaData(QLatin1String("PrivacyCompactPolicy"), compact.join(QLatin1String("\n")));
3332  }
3333  }
3334 
3335  // continue only if we know that we're at least HTTP/1.0
3336  if (httpRev == HTTP_11 || httpRev == HTTP_10) {
3337  // let them tell us if we should stay alive or not
3338  tIt = tokenizer.iterator("connection");
3339  while (tIt.hasNext()) {
3340  QByteArray connection = tIt.next().toLower();
3341  if (!(isHttpProxy(m_request.proxyUrl) && !isAutoSsl())) {
3342  if (connection.startsWith("close")) { // krazy:exclude=strings
3343  m_request.isKeepAlive = false;
3344  } else if (connection.startsWith("keep-alive")) { // krazy:exclude=strings
3345  m_request.isKeepAlive = true;
3346  }
3347  }
3348  if (connection.startsWith("upgrade")) { // krazy:exclude=strings
3349  if (m_request.responseCode == 101) {
3350  // Ok, an upgrade was accepted, now we must do it
3351  upgradeRequired = true;
3352  } else if (upgradeRequired) { // 426
3353  // Nothing to do since we did it above already
3354  }
3355  }
3356  }
3357  // what kind of encoding do we have? transfer?
3358  tIt = tokenizer.iterator("transfer-encoding");
3359  while (tIt.hasNext()) {
3360  // If multiple encodings have been applied to an entity, the
3361  // transfer-codings MUST be listed in the order in which they
3362  // were applied.
3363  addEncoding(toQString(tIt.next().trimmed()), m_transferEncodings);
3364  }
3365 
3366  // md5 signature
3367  tIt = tokenizer.iterator("content-md5");
3368  if (tIt.hasNext()) {
3369  m_contentMD5 = toQString(tIt.next().trimmed());
3370  }
3371 
3372  // *** Responses to the HTTP OPTIONS method follow
3373  // WebDAV capabilities
3374  tIt = tokenizer.iterator("dav");
3375  while (tIt.hasNext()) {
3376  m_davCapabilities << toQString(tIt.next());
3377  }
3378  // *** Responses to the HTTP OPTIONS method finished
3379  }
3380 
3381 
3382  // Now process the HTTP/1.1 upgrade
3383  QStringList upgradeOffers;
3384  tIt = tokenizer.iterator("upgrade");
3385  if (tIt.hasNext()) {
3386  // Now we have to check to see what is offered for the upgrade
3387  QString offered = toQString(tIt.next());
3388  upgradeOffers = offered.split(QRegExp(QLatin1String("[ \n,\r\t]")), QString::SkipEmptyParts);
3389  }
3390  Q_FOREACH (const QString &opt, upgradeOffers) {
3391  if (opt == QLatin1String("TLS/1.0")) {
3392  if (!startSsl() && upgradeRequired) {
3393  error(ERR_UPGRADE_REQUIRED, opt);
3394  return false;
3395  }
3396  } else if (opt == QLatin1String("HTTP/1.1")) {
3397  httpRev = HTTP_11;
3398  } else if (upgradeRequired) {
3399  // we are told to do an upgrade we don't understand
3400  error(ERR_UPGRADE_REQUIRED, opt);
3401  return false;
3402  }
3403  }
3404 
3405  // Harvest cookies (mmm, cookie fields!)
3406  QByteArray cookieStr; // In case we get a cookie.
3407  tIt = tokenizer.iterator("set-cookie");
3408  while (tIt.hasNext()) {
3409  cookieStr += "Set-Cookie: ";
3410  cookieStr += tIt.next();
3411  cookieStr += '\n';
3412  }
3413  if (!cookieStr.isEmpty()) {
3414  if ((m_request.cookieMode == HTTPRequest::CookiesAuto) && m_request.useCookieJar) {
3415  // Give cookies to the cookiejar.
3416  const QString domain = config()->readEntry("cross-domain");
3417  if (!domain.isEmpty() && isCrossDomainRequest(m_request.url.host(), domain)) {
3418  cookieStr = "Cross-Domain\n" + cookieStr;
3419  }
3420  addCookies( m_request.url.url(), cookieStr );
3421  } else if (m_request.cookieMode == HTTPRequest::CookiesManual) {
3422  // Pass cookie to application
3423  setMetaData(QLatin1String("setcookies"), QString::fromUtf8(cookieStr)); // ## is encoding ok?
3424  }
3425  }
3426 
3427  // We need to reread the header if we got a '100 Continue' or '102 Processing'
3428  // This may be a non keepalive connection so we handle this kind of loop internally
3429  if ( cont )
3430  {
3431  kDebug(7113) << "cont; returning to mark try_again";
3432  goto try_again;
3433  }
3434 
3435  if (!m_isChunked && (m_iSize == NO_SIZE) && m_request.isKeepAlive &&
3436  canHaveResponseBody(m_request.responseCode, m_request.method)) {
3437  kDebug(7113) << "Ignoring keep-alive: otherwise unable to determine response body length.";
3438  m_request.isKeepAlive = false;
3439  }
3440 
3441  // TODO cache the proxy auth data (not doing this means a small performance regression for now)
3442 
3443  // we may need to send (Proxy or WWW) authorization data
3444  if ((!m_request.doNotWWWAuthenticate && m_request.responseCode == 401) ||
3445  (!m_request.doNotProxyAuthenticate && m_request.responseCode == 407)) {
3446  authRequiresAnotherRoundtrip = handleAuthenticationHeader(&tokenizer);
3447  if (m_iError) {
3448  // If error is set, then handleAuthenticationHeader failed.
3449  return false;
3450  }
3451  } else {
3452  authRequiresAnotherRoundtrip = false;
3453  }
3454 
3455  QString locationStr;
3456  // In fact we should do redirection only if we have a redirection response code (300 range)
3457  tIt = tokenizer.iterator("location");
3458  if (tIt.hasNext() && m_request.responseCode > 299 && m_request.responseCode < 400) {
3459  locationStr = QString::fromUtf8(tIt.next().trimmed());
3460  }
3461  // We need to do a redirect
3462  if (!locationStr.isEmpty())
3463  {
3464  KUrl u(m_request.url, locationStr);
3465  if(!u.isValid())
3466  {
3467  error(ERR_MALFORMED_URL, u.prettyUrl());
3468  return false;
3469  }
3470 
3471  // preserve #ref: (bug 124654)
3472  // if we were at http://host/resource1#ref, we sent a GET for "/resource1"
3473  // if we got redirected to http://host/resource2, then we have to re-add
3474  // the fragment:
3475  if (m_request.url.hasRef() && !u.hasRef() &&
3476  (m_request.url.host() == u.host()) &&
3477  (m_request.url.protocol() == u.protocol()))
3478  u.setRef(m_request.url.ref());
3479 
3480  m_isRedirection = true;
3481 
3482  if (!m_request.id.isEmpty())
3483  {
3484  sendMetaData();
3485  }
3486 
3487  // If we're redirected to a http:// url, remember that we're doing webdav...
3488  if (m_protocol == "webdav" || m_protocol == "webdavs"){
3489  if(u.protocol() == QLatin1String("http")){
3490  u.setProtocol(QLatin1String("webdav"));
3491  }else if(u.protocol() == QLatin1String("https")){
3492  u.setProtocol(QLatin1String("webdavs"));
3493  }
3494 
3495  m_request.redirectUrl = u;
3496  }
3497 
3498  kDebug(7113) << "Re-directing from" << m_request.url
3499  << "to" << u;
3500 
3501  redirection(u);
3502 
3503  // It would be hard to cache the redirection response correctly. The possible benefit
3504  // is small (if at all, assuming fast disk and slow network), so don't do it.
3505  cacheFileClose();
3506  setCacheabilityMetadata(false);
3507  }
3508 
3509  // Inform the job that we can indeed resume...
3510  if (bCanResume && m_request.offset) {
3511  //TODO turn off caching???
3512  canResume();
3513  } else {
3514  m_request.offset = 0;
3515  }
3516 
3517  // Correct a few common wrong content encodings
3518  fixupResponseContentEncoding();
3519 
3520  // Correct some common incorrect pseudo-mimetypes
3521  fixupResponseMimetype();
3522 
3523  // parse everything related to expire and other dates, and cache directives; also switch
3524  // between cache reading and writing depending on cache validation result.
3525  cacheParseResponseHeader(tokenizer);
3526  }
3527 
3528  if (m_request.cacheTag.ioMode == ReadFromCache) {
3529  if (m_request.cacheTag.policy == CC_Verify &&
3530  m_request.cacheTag.plan(m_maxCacheAge) != CacheTag::UseCached) {
3531  kDebug(7113) << "Reading resource from cache even though the cache plan is not "
3532  "UseCached; the server is probably sending wrong expiry information.";
3533  }
3534  // parseHeaderFromCache replaces this method in case of cached content
3535  return parseHeaderFromCache();
3536  }
3537 
3538  if (config()->readEntry("PropagateHttpHeader", false) ||
3539  m_request.cacheTag.ioMode == WriteToCache) {
3540  // store header lines if they will be used; note that the tokenizer removing
3541  // line continuation special cases is probably more good than bad.
3542  int nextLinePos = 0;
3543  int prevLinePos = 0;
3544  bool haveMore = true;
3545  while (haveMore) {
3546  haveMore = nextLine(buffer, &nextLinePos, bufPos);
3547  int prevLineEnd = nextLinePos;
3548  while (buffer[prevLineEnd - 1] == '\r' || buffer[prevLineEnd - 1] == '\n') {
3549  prevLineEnd--;
3550  }
3551 
3552  m_responseHeaders.append(QString::fromLatin1(&buffer[prevLinePos],
3553  prevLineEnd - prevLinePos));
3554  prevLinePos = nextLinePos;
3555  }
3556 
3557  // IMPORTANT: Do not remove this line because forwardHttpResponseHeader
3558  // is called below. This line is here to ensure the response headers are
3559  // available to the client before it receives mimetype information.
3560  // The support for putting ioslaves on hold in the KIO-QNAM integration
3561  // will break if this line is removed.
3562  setMetaData(QLatin1String("HTTP-Headers"), m_responseHeaders.join(QString(QLatin1Char('\n'))));
3563  }
3564 
3565  // Let the app know about the mime-type iff this is not a redirection and
3566  // the mime-type string is not empty.
3567  if (!m_isRedirection && m_request.responseCode != 204 &&
3568  (!m_mimeType.isEmpty() || m_request.method == HTTP_HEAD) &&
3569  (m_isLoadingErrorPage || !authRequiresAnotherRoundtrip)) {
3570  kDebug(7113) << "Emitting mimetype " << m_mimeType;
3571  mimeType( m_mimeType );
3572  }
3573 
3574  // IMPORTANT: Do not move the function call below before doing any
3575  // redirection. Otherwise it might mess up some sites, see BR# 150904.
3576  forwardHttpResponseHeader();
3577 
3578  if (m_request.method == HTTP_HEAD)
3579  return true;
3580 
3581  return !authRequiresAnotherRoundtrip; // return true if no more credentials need to be sent
3582 }
3583 
3584 void HTTPProtocol::parseContentDisposition(const QString &disposition)
3585 {
3586  const QMap<QString, QString> parameters = contentDispositionParser(disposition);
3587 
3588  QMap<QString, QString>::const_iterator i = parameters.constBegin();
3589  while (i != parameters.constEnd()) {
3590  setMetaData(QLatin1String("content-disposition-") + i.key(), i.value());
3591  kDebug(7113) << "Content-Disposition:" << i.key() << "=" << i.value();
3592  ++i;
3593  }
3594 }
3595 
3596 void HTTPProtocol::addEncoding(const QString &_encoding, QStringList &encs)
3597 {
3598  QString encoding = _encoding.trimmed().toLower();
3599  // Identity is the same as no encoding
3600  if (encoding == QLatin1String("identity")) {
3601  return;
3602  } else if (encoding == QLatin1String("8bit")) {
3603  // Strange encoding returned by http://linac.ikp.physik.tu-darmstadt.de
3604  return;
3605  } else if (encoding == QLatin1String("chunked")) {
3606  m_isChunked = true;
3607  // Anyone know of a better way to handle unknown sizes possibly/ideally with unsigned ints?
3608  //if ( m_cmd != CMD_COPY )
3609  m_iSize = NO_SIZE;
3610  } else if ((encoding == QLatin1String("x-gzip")) || (encoding == QLatin1String("gzip"))) {
3611  encs.append(QLatin1String("gzip"));
3612  } else if ((encoding == QLatin1String("x-bzip2")) || (encoding == QLatin1String("bzip2"))) {
3613  encs.append(QLatin1String("bzip2")); // Not yet supported!
3614  } else if ((encoding == QLatin1String("x-deflate")) || (encoding == QLatin1String("deflate"))) {
3615  encs.append(QLatin1String("deflate"));
3616  } else {
3617  kDebug(7113) << "Unknown encoding encountered. "
3618  << "Please write code. Encoding =" << encoding;
3619  }
3620 }
3621 
3622 void HTTPProtocol::cacheParseResponseHeader(const HeaderTokenizer &tokenizer)
3623 {
3624  if (!m_request.cacheTag.useCache)
3625  return;
3626 
3627  // might have to add more response codes
3628  if (m_request.responseCode != 200 && m_request.responseCode != 304) {
3629  return;
3630  }
3631 
3632  // -1 is also the value returned by KDateTime::toTime_t() from an invalid instance.
3633  m_request.cacheTag.servedDate = -1;
3634  m_request.cacheTag.lastModifiedDate = -1;
3635  m_request.cacheTag.expireDate = -1;
3636 
3637  const qint64 currentDate = time(0);
3638  bool mayCache = m_request.cacheTag.ioMode != NoCache;
3639 
3640  TokenIterator tIt = tokenizer.iterator("last-modified");
3641  if (tIt.hasNext()) {
3642  m_request.cacheTag.lastModifiedDate =
3643  KDateTime::fromString(toQString(tIt.next()), KDateTime::RFCDate).toTime_t();
3644 
3645  //### might be good to canonicalize the date by using KDateTime::toString()
3646  if (m_request.cacheTag.lastModifiedDate != -1) {
3647  setMetaData(QLatin1String("modified"), toQString(tIt.current()));
3648  }
3649  }
3650 
3651  // determine from available information when the response was served by the origin server
3652  {
3653  qint64 dateHeader = -1;
3654  tIt = tokenizer.iterator("date");
3655  if (tIt.hasNext()) {
3656  dateHeader = KDateTime::fromString(toQString(tIt.next()), KDateTime::RFCDate).toTime_t();
3657  // -1 on error
3658  }
3659 
3660  qint64 ageHeader = 0;
3661  tIt = tokenizer.iterator("age");
3662  if (tIt.hasNext()) {
3663  ageHeader = tIt.next().toLongLong();
3664  // 0 on error
3665  }
3666 
3667  if (dateHeader != -1) {
3668  m_request.cacheTag.servedDate = dateHeader;
3669  } else if (ageHeader) {
3670  m_request.cacheTag.servedDate = currentDate - ageHeader;
3671  } else {
3672  m_request.cacheTag.servedDate = currentDate;
3673  }
3674  }
3675 
3676  bool hasCacheDirective = false;
3677  // determine when the response "expires", i.e. becomes stale and needs revalidation
3678  {
3679  // (we also parse other cache directives here)
3680  qint64 maxAgeHeader = 0;
3681  tIt = tokenizer.iterator("cache-control");
3682  while (tIt.hasNext()) {
3683  QByteArray cacheStr = tIt.next().toLower();
3684  if (cacheStr.startsWith("no-cache") || cacheStr.startsWith("no-store")) { // krazy:exclude=strings
3685  // Don't put in cache
3686  mayCache = false;
3687  hasCacheDirective = true;
3688  } else if (cacheStr.startsWith("max-age=")) { // krazy:exclude=strings
3689  QByteArray ba = cacheStr.mid(qstrlen("max-age=")).trimmed();
3690  bool ok = false;
3691  maxAgeHeader = ba.toLongLong(&ok);
3692  if (ok) {
3693  hasCacheDirective = true;
3694  }
3695  }
3696  }
3697 
3698  qint64 expiresHeader = -1;
3699  tIt = tokenizer.iterator("expires");
3700  if (tIt.hasNext()) {
3701  expiresHeader = KDateTime::fromString(toQString(tIt.next()), KDateTime::RFCDate).toTime_t();
3702  kDebug(7113) << "parsed expire date from 'expires' header:" << tIt.current();
3703  }
3704 
3705  if (maxAgeHeader) {
3706  m_request.cacheTag.expireDate = m_request.cacheTag.servedDate + maxAgeHeader;
3707  } else if (expiresHeader != -1) {
3708  m_request.cacheTag.expireDate = expiresHeader;
3709  } else {
3710  // heuristic expiration date
3711  if (m_request.cacheTag.lastModifiedDate != -1) {
3712  // expAge is following the RFC 2616 suggestion for heuristic expiration
3713  qint64 expAge = (m_request.cacheTag.servedDate -
3714  m_request.cacheTag.lastModifiedDate) / 10;
3715  // not in the RFC: make sure not to have a huge heuristic cache lifetime
3716  expAge = qMin(expAge, qint64(3600 * 24));
3717  m_request.cacheTag.expireDate = m_request.cacheTag.servedDate + expAge;
3718  } else {
3719  m_request.cacheTag.expireDate = m_request.cacheTag.servedDate +
3720  DEFAULT_CACHE_EXPIRE;
3721  }
3722  }
3723  // make sure that no future clock monkey business causes the cache entry to un-expire
3724  if (m_request.cacheTag.expireDate < currentDate) {
3725  m_request.cacheTag.expireDate = 0; // January 1, 1970 :)
3726  }
3727  }
3728 
3729  tIt = tokenizer.iterator("etag");
3730  if (tIt.hasNext()) {
3731  QString prevEtag = m_request.cacheTag.etag;
3732  m_request.cacheTag.etag = toQString(tIt.next());
3733  if (m_request.cacheTag.etag != prevEtag && m_request.responseCode == 304) {
3734  kDebug(7103) << "304 Not Modified but new entity tag - I don't think this is legal HTTP.";
3735  }
3736  }
3737 
3738  // whoops.. we received a warning
3739  tIt = tokenizer.iterator("warning");
3740  if (tIt.hasNext()) {
3741  //Don't use warning() here, no need to bother the user.
3742  //Those warnings are mostly about caches.
3743  infoMessage(toQString(tIt.next()));
3744  }
3745 
3746  // Cache management (HTTP 1.0)
3747  tIt = tokenizer.iterator("pragma");
3748  while (tIt.hasNext()) {
3749  if (tIt.next().toLower().startsWith("no-cache")) { // krazy:exclude=strings
3750  mayCache = false;
3751  hasCacheDirective = true;
3752  }
3753  }
3754 
3755  // The deprecated Refresh Response
3756  tIt = tokenizer.iterator("refresh");
3757  if (tIt.hasNext()) {
3758  mayCache = false;
3759  setMetaData(QLatin1String("http-refresh"), toQString(tIt.next().trimmed()));
3760  }
3761 
3762  // We don't cache certain text objects
3763  if (m_mimeType.startsWith(QLatin1String("text/")) && (m_mimeType != QLatin1String("text/css")) &&
3764  (m_mimeType != QLatin1String("text/x-javascript")) && !hasCacheDirective) {
3765  // Do not cache secure pages or pages
3766  // originating from password protected sites
3767  // unless the webserver explicitly allows it.
3768  if (isUsingSsl() || m_wwwAuth) {
3769  mayCache = false;
3770  }
3771  }
3772 
3773  // note that we've updated cacheTag, so the plan() is with current data
3774  if (m_request.cacheTag.plan(m_maxCacheAge) == CacheTag::ValidateCached) {
3775  kDebug(7113) << "Cache needs validation";
3776  if (m_request.responseCode == 304) {
3777  kDebug(7113) << "...was revalidated by response code but not by updated expire times. "
3778  "We're going to set the expire date to 60 seconds in the future...";
3779  m_request.cacheTag.expireDate = currentDate + 60;
3780  if (m_request.cacheTag.policy == CC_Verify &&
3781  m_request.cacheTag.plan(m_maxCacheAge) != CacheTag::UseCached) {
3782  // "apparently" because we /could/ have made an error ourselves, but the errors I
3783  // witnessed were all the server's fault.
3784  kDebug(7113) << "this proxy or server apparently sends bogus expiry information.";
3785  }
3786  }
3787  }
3788 
3789  // validation handling
3790  if (mayCache && m_request.responseCode == 200 && !m_mimeType.isEmpty()) {
3791  kDebug(7113) << "Cache, adding" << m_request.url;
3792  // ioMode can still be ReadFromCache here if we're performing a conditional get
3793  // aka validation
3794  m_request.cacheTag.ioMode = WriteToCache;
3795  if (!cacheFileOpenWrite()) {
3796  kDebug(7113) << "Error creating cache entry for " << m_request.url << "!\n";
3797  }
3798  m_maxCacheSize = config()->readEntry("MaxCacheSize", DEFAULT_MAX_CACHE_SIZE);
3799  } else if (m_request.responseCode == 304 && m_request.cacheTag.file) {
3800  if (!mayCache) {
3801  kDebug(7113) << "This webserver is confused about the cacheability of the data it sends.";
3802  }
3803  // the cache file should still be open for reading, see satisfyRequestFromCache().
3804  Q_ASSERT(m_request.cacheTag.file->openMode() == QIODevice::ReadOnly);
3805  Q_ASSERT(m_request.cacheTag.ioMode == ReadFromCache);
3806  } else {
3807  cacheFileClose();
3808  }
3809 
3810  setCacheabilityMetadata(mayCache);
3811 }
3812 
3813 void HTTPProtocol::setCacheabilityMetadata(bool cachingAllowed)
3814 {
3815  if (!cachingAllowed) {
3816  setMetaData(QLatin1String("no-cache"), QLatin1String("true"));
3817  setMetaData(QLatin1String("expire-date"), QLatin1String("1")); // Expired
3818  } else {
3819  QString tmp;
3820  tmp.setNum(m_request.cacheTag.expireDate);
3821  setMetaData(QLatin1String("expire-date"), tmp);
3822  // slightly changed semantics from old creationDate, probably more correct now
3823  tmp.setNum(m_request.cacheTag.servedDate);
3824  setMetaData(QLatin1String("cache-creation-date"), tmp);
3825  }
3826 }
3827 
3828 bool HTTPProtocol::sendCachedBody()
3829 {
3830  infoMessage(i18n("Sending data to %1" , m_request.url.host()));
3831 
3832  QByteArray cLength ("Content-Length: ");
3833  cLength += QByteArray::number(m_POSTbuf->size());
3834  cLength += "\r\n\r\n";
3835 
3836  kDebug(7113) << "sending cached data (size=" << m_POSTbuf->size() << ")";
3837 
3838  // Send the content length...
3839  bool sendOk = (write(cLength.data(), cLength.size()) == (ssize_t) cLength.size());
3840  if (!sendOk) {
3841  kDebug( 7113 ) << "Connection broken when sending "
3842  << "content length: (" << m_request.url.host() << ")";
3843  error( ERR_CONNECTION_BROKEN, m_request.url.host() );
3844  return false;
3845  }
3846 
3847  // Make sure the read head is at the beginning...
3848  m_POSTbuf->reset();
3849 
3850  // Send the data...
3851  while (!m_POSTbuf->atEnd()) {
3852  const QByteArray buffer = m_POSTbuf->read(s_MaxInMemPostBufSize);
3853  sendOk = (write(buffer.data(), buffer.size()) == (ssize_t) buffer.size());
3854  if (!sendOk) {
3855  kDebug(7113) << "Connection broken when sending message body: ("
3856  << m_request.url.host() << ")";
3857  error( ERR_CONNECTION_BROKEN, m_request.url.host() );
3858  return false;
3859  }
3860  }
3861 
3862  return true;
3863 }
3864 
3865 bool HTTPProtocol::sendBody()
3866 {
3867  // If we have cached data, the it is either a repost or a DAV request so send
3868  // the cached data...
3869  if (m_POSTbuf)
3870  return sendCachedBody();
3871 
3872  if (m_iPostDataSize == NO_SIZE) {
3873  // Try the old approach of retireving content data from the job
3874  // before giving up.
3875  if (retrieveAllData())
3876  return sendCachedBody();
3877 
3878  error(ERR_POST_NO_SIZE, m_request.url.host());
3879  return false;
3880  }
3881 
3882  kDebug(7113) << "sending data (size=" << m_iPostDataSize << ")";
3883 
3884  infoMessage(i18n("Sending data to %1", m_request.url.host()));
3885 
3886  QByteArray cLength ("Content-Length: ");
3887  cLength += QByteArray::number(m_iPostDataSize);
3888  cLength += "\r\n\r\n";
3889 
3890  kDebug(7113) << cLength.trimmed();
3891 
3892  // Send the content length...
3893  bool sendOk = (write(cLength.data(), cLength.size()) == (ssize_t) cLength.size());
3894  if (!sendOk) {
3895  // The server might have closed the connection due to a timeout, or maybe
3896  // some transport problem arose while the connection was idle.
3897  if (m_request.isKeepAlive)
3898  {
3899  httpCloseConnection();
3900  return true; // Try again
3901  }
3902 
3903  kDebug(7113) << "Connection broken while sending POST content size to" << m_request.url.host();
3904  error( ERR_CONNECTION_BROKEN, m_request.url.host() );
3905  return false;
3906  }
3907 
3908  // Send the amount
3909  totalSize(m_iPostDataSize);
3910 
3911  // If content-length is 0, then do nothing but simply return true.
3912  if (m_iPostDataSize == 0)
3913  return true;
3914 
3915  sendOk = true;
3916  KIO::filesize_t bytesSent = 0;
3917 
3918  while (true) {
3919  dataReq();
3920 
3921  QByteArray buffer;
3922  const int bytesRead = readData(buffer);
3923 
3924  // On done...
3925  if (bytesRead == 0) {
3926  sendOk = (bytesSent == m_iPostDataSize);
3927  break;
3928  }
3929 
3930  // On error return false...
3931  if (bytesRead < 0) {
3932  error(ERR_ABORTED, m_request.url.host());
3933  sendOk = false;
3934  break;
3935  }
3936 
3937  // Cache the POST data in case of a repost request.
3938  cachePostData(buffer);
3939 
3940  // This will only happen if transmitting the data fails, so we will simply
3941  // cache the content locally for the potential re-transmit...
3942  if (!sendOk)
3943  continue;
3944 
3945  if (write(buffer.data(), bytesRead) == static_cast<ssize_t>(bytesRead)) {
3946  bytesSent += bytesRead;
3947  processedSize(bytesSent); // Send update status...
3948  continue;
3949  }
3950 
3951  kDebug(7113) << "Connection broken while sending POST content to" << m_request.url.host();
3952  error(ERR_CONNECTION_BROKEN, m_request.url.host());
3953  sendOk = false;
3954  }
3955 
3956  return sendOk;
3957 }
3958 
3959 void HTTPProtocol::httpClose( bool keepAlive )
3960 {
3961  kDebug(7113) << "keepAlive =" << keepAlive;
3962 
3963  cacheFileClose();
3964 
3965  // Only allow persistent connections for GET requests.
3966  // NOTE: we might even want to narrow this down to non-form
3967  // based submit requests which will require a meta-data from
3968  // khtml.
3969  if (keepAlive) {
3970  if (!m_request.keepAliveTimeout)
3971  m_request.keepAliveTimeout = DEFAULT_KEEP_ALIVE_TIMEOUT;
3972  else if (m_request.keepAliveTimeout > 2*DEFAULT_KEEP_ALIVE_TIMEOUT)
3973  m_request.keepAliveTimeout = 2*DEFAULT_KEEP_ALIVE_TIMEOUT;
3974 
3975  kDebug(7113) << "keep alive (" << m_request.keepAliveTimeout << ")";
3976  QByteArray data;
3977  QDataStream stream( &data, QIODevice::WriteOnly );
3978  stream << int(99); // special: Close connection
3979  setTimeoutSpecialCommand(m_request.keepAliveTimeout, data);
3980 
3981  return;
3982  }
3983 
3984  httpCloseConnection();
3985 }
3986 
3987 void HTTPProtocol::closeConnection()
3988 {
3989  kDebug(7113);
3990  httpCloseConnection();
3991 }
3992 
3993 void HTTPProtocol::httpCloseConnection()
3994 {
3995  kDebug(7113);
3996  m_server.clear();
3997  disconnectFromHost();
3998  clearUnreadBuffer();
3999  setTimeoutSpecialCommand(-1); // Cancel any connection timeout
4000 }
4001 
4002 void HTTPProtocol::slave_status()
4003 {
4004  kDebug(7113);
4005 
4006  if ( !isConnected() )
4007  httpCloseConnection();
4008 
4009  slaveStatus( m_server.url.host(), isConnected() );
4010 }
4011 
4012 void HTTPProtocol::mimetype( const KUrl& url )
4013 {
4014  kDebug(7113) << url;
4015 
4016  if (!maybeSetRequestUrl(url))
4017  return;
4018  resetSessionSettings();
4019 
4020  m_request.method = HTTP_HEAD;
4021  m_request.cacheTag.policy= CC_Cache;
4022 
4023  if (proceedUntilResponseHeader()) {
4024  httpClose(m_request.isKeepAlive);
4025  finished();
4026  }
4027 
4028  kDebug(7113) << m_mimeType;
4029 }
4030 
4031 void HTTPProtocol::special( const QByteArray &data )
4032 {
4033  kDebug(7113);
4034 
4035  int tmp;
4036  QDataStream stream(data);
4037 
4038  stream >> tmp;
4039  switch (tmp) {
4040  case 1: // HTTP POST
4041  {
4042  KUrl url;
4043  qint64 size;
4044  stream >> url >> size;
4045  post( url, size );
4046  break;
4047  }
4048  case 2: // cache_update
4049  {
4050  KUrl url;
4051  bool no_cache;
4052  qint64 expireDate;
4053  stream >> url >> no_cache >> expireDate;
4054  if (no_cache) {
4055  QString filename = cacheFilePathFromUrl(url);
4056  // there is a tiny risk of deleting the wrong file due to hash collisions here.
4057  // this is an unimportant performance issue.
4058  // FIXME on Windows we may be unable to delete the file if open
4059  QFile::remove(filename);
4060  finished();
4061  break;
4062  }
4063  // let's be paranoid and inefficient here...
4064  HTTPRequest savedRequest = m_request;
4065 
4066  m_request.url = url;
4067  if (cacheFileOpenRead()) {
4068  m_request.cacheTag.expireDate = expireDate;
4069  cacheFileClose(); // this sends an update command to the cache cleaner process
4070  }
4071 
4072  m_request = savedRequest;
4073  finished();
4074  break;
4075  }
4076  case 5: // WebDAV lock
4077  {
4078  KUrl url;
4079  QString scope, type, owner;
4080  stream >> url >> scope >> type >> owner;
4081  davLock( url, scope, type, owner );
4082  break;
4083  }
4084  case 6: // WebDAV unlock
4085  {
4086  KUrl url;
4087  stream >> url;
4088  davUnlock( url );
4089  break;
4090  }
4091  case 7: // Generic WebDAV
4092  {
4093  KUrl url;
4094  int method;
4095  qint64 size;
4096  stream >> url >> method >> size;
4097  davGeneric( url, (KIO::HTTP_METHOD) method, size );
4098  break;
4099  }
4100  case 99: // Close Connection
4101  {
4102  httpCloseConnection();
4103  break;
4104  }
4105  default:
4106  // Some command we don't understand.
4107  // Just ignore it, it may come from some future version of KDE.
4108  break;
4109  }
4110 }
4111 
4115 int HTTPProtocol::readChunked()
4116 {
4117  if ((m_iBytesLeft == 0) || (m_iBytesLeft == NO_SIZE))
4118  {
4119  // discard CRLF from previous chunk, if any, and read size of next chunk
4120 
4121  int bufPos = 0;
4122  m_receiveBuf.resize(4096);
4123 
4124  bool foundCrLf = readDelimitedText(m_receiveBuf.data(), &bufPos, m_receiveBuf.size(), 1);
4125 
4126  if (foundCrLf && bufPos == 2) {
4127  // The previous read gave us the CRLF from the previous chunk. As bufPos includes
4128  // the trailing CRLF it has to be > 2 to possibly include the next chunksize.
4129  bufPos = 0;
4130  foundCrLf = readDelimitedText(m_receiveBuf.data(), &bufPos, m_receiveBuf.size(), 1);
4131  }
4132  if (!foundCrLf) {
4133  kDebug(7113) << "Failed to read chunk header.";
4134  return -1;
4135  }
4136  Q_ASSERT(bufPos > 2);
4137 
4138  long long nextChunkSize = STRTOLL(m_receiveBuf.data(), 0, 16);
4139  if (nextChunkSize < 0)
4140  {
4141  kDebug(7113) << "Negative chunk size";
4142  return -1;
4143  }
4144  m_iBytesLeft = nextChunkSize;
4145 
4146  kDebug(7113) << "Chunk size =" << m_iBytesLeft << "bytes";
4147 
4148  if (m_iBytesLeft == 0)
4149  {
4150  // Last chunk; read and discard chunk trailer.
4151  // The last trailer line ends with CRLF and is followed by another CRLF
4152  // so we have CRLFCRLF like at the end of a standard HTTP header.
4153  // Do not miss a CRLFCRLF spread over two of our 4K blocks: keep three previous bytes.
4154  //NOTE the CRLF after the chunksize also counts if there is no trailer. Copy it over.
4155  char trash[4096];
4156  trash[0] = m_receiveBuf.constData()[bufPos - 2];
4157  trash[1] = m_receiveBuf.constData()[bufPos - 1];
4158  int trashBufPos = 2;
4159  bool done = false;
4160  while (!done && !m_isEOF) {
4161  if (trashBufPos > 3) {
4162  // shift everything but the last three bytes out of the buffer
4163  for (int i = 0; i < 3; i++) {
4164  trash[i] = trash[trashBufPos - 3 + i];
4165  }
4166  trashBufPos = 3;
4167  }
4168  done = readDelimitedText(trash, &trashBufPos, 4096, 2);
4169  }
4170  if (m_isEOF && !done) {
4171  kDebug(7113) << "Failed to read chunk trailer.";
4172  return -1;
4173  }
4174 
4175  return 0;
4176  }
4177  }
4178 
4179  int bytesReceived = readLimited();
4180  if (!m_iBytesLeft) {
4181  m_iBytesLeft = NO_SIZE; // Don't stop, continue with next chunk
4182  }
4183  return bytesReceived;
4184 }
4185 
4186 int HTTPProtocol::readLimited()
4187 {
4188  if (!m_iBytesLeft)
4189  return 0;
4190 
4191  m_receiveBuf.resize(4096);
4192 
4193  int bytesToReceive;
4194  if (m_iBytesLeft > KIO::filesize_t(m_receiveBuf.size()))
4195  bytesToReceive = m_receiveBuf.size();
4196  else
4197  bytesToReceive = m_iBytesLeft;
4198 
4199  const int bytesReceived = readBuffered(m_receiveBuf.data(), bytesToReceive, false);
4200 
4201  if (bytesReceived <= 0)
4202  return -1; // Error: connection lost
4203 
4204  m_iBytesLeft -= bytesReceived;
4205  return bytesReceived;
4206 }
4207 
4208 int HTTPProtocol::readUnlimited()
4209 {
4210  if (m_request.isKeepAlive)
4211  {
4212  kDebug(7113) << "Unbounded datastream on a Keep-alive connection!";
4213  m_request.isKeepAlive = false;
4214  }
4215 
4216  m_receiveBuf.resize(4096);
4217 
4218  int result = readBuffered(m_receiveBuf.data(), m_receiveBuf.size());
4219  if (result > 0)
4220  return result;
4221 
4222  m_isEOF = true;
4223  m_iBytesLeft = 0;
4224  return 0;
4225 }
4226 
4227 void HTTPProtocol::slotData(const QByteArray &_d)
4228 {
4229  if (!_d.size())
4230  {
4231  m_isEOD = true;
4232  return;
4233  }
4234 
4235  if (m_iContentLeft != NO_SIZE)
4236  {
4237  if (m_iContentLeft >= KIO::filesize_t(_d.size()))
4238  m_iContentLeft -= _d.size();
4239  else
4240  m_iContentLeft = NO_SIZE;
4241  }
4242 
4243  QByteArray d = _d;
4244  if ( !m_dataInternal )
4245  {
4246  // If a broken server does not send the mime-type,
4247  // we try to id it from the content before dealing
4248  // with the content itself.
4249  if ( m_mimeType.isEmpty() && !m_isRedirection &&
4250  !( m_request.responseCode >= 300 && m_request.responseCode <=399) )
4251  {
4252  kDebug(7113) << "Determining mime-type from content...";
4253  int old_size = m_mimeTypeBuffer.size();
4254  m_mimeTypeBuffer.resize( old_size + d.size() );
4255  memcpy( m_mimeTypeBuffer.data() + old_size, d.data(), d.size() );
4256  if ( (m_iBytesLeft != NO_SIZE) && (m_iBytesLeft > 0)
4257  && (m_mimeTypeBuffer.size() < 1024) )
4258  {
4259  m_cpMimeBuffer = true;
4260  return; // Do not send up the data since we do not yet know its mimetype!
4261  }
4262 
4263  kDebug(7113) << "Mimetype buffer size:" << m_mimeTypeBuffer.size();
4264 
4265  KMimeType::Ptr mime = KMimeType::findByNameAndContent(m_request.url.fileName(), m_mimeTypeBuffer);
4266  if( mime && !mime->isDefault() )
4267  {
4268  m_mimeType = mime->name();
4269  kDebug(7113) << "Mimetype from content:" << m_mimeType;
4270  }
4271 
4272  if ( m_mimeType.isEmpty() )
4273  {
4274  m_mimeType = QLatin1String( DEFAULT_MIME_TYPE );
4275  kDebug(7113) << "Using default mimetype:" << m_mimeType;
4276  }
4277 
4278  //### we could also open the cache file here
4279 
4280  if ( m_cpMimeBuffer )
4281  {
4282  d.resize(0);
4283  d.resize(m_mimeTypeBuffer.size());
4284  memcpy(d.data(), m_mimeTypeBuffer.data(), d.size());
4285  }
4286  mimeType(m_mimeType);
4287  m_mimeTypeBuffer.resize(0);
4288  }
4289 
4290  //kDebug(7113) << "Sending data of size" << d.size();
4291  data( d );
4292  if (m_request.cacheTag.ioMode == WriteToCache) {
4293  cacheFileWritePayload(d);
4294  }
4295  }
4296  else
4297  {
4298  uint old_size = m_webDavDataBuf.size();
4299  m_webDavDataBuf.resize (old_size + d.size());
4300  memcpy (m_webDavDataBuf.data() + old_size, d.data(), d.size());
4301  }
4302 }
4303 
4313 bool HTTPProtocol::readBody( bool dataInternal /* = false */ )
4314 {
4315  // special case for reading cached body since we also do it in this function. oh well.
4316  if (!canHaveResponseBody(m_request.responseCode, m_request.method) &&
4317  !(m_request.cacheTag.ioMode == ReadFromCache && m_request.responseCode == 304 &&
4318  m_request.method != HTTP_HEAD)) {
4319  return true;
4320  }
4321 
4322  m_isEOD = false;
4323  // Note that when dataInternal is true, we are going to:
4324  // 1) save the body data to a member variable, m_webDavDataBuf
4325  // 2) _not_ advertise the data, speed, size, etc., through the
4326  // corresponding functions.
4327  // This is used for returning data to WebDAV.
4328  m_dataInternal = dataInternal;
4329  if (dataInternal) {
4330  m_webDavDataBuf.clear();
4331  }
4332 
4333  // Check if we need to decode the data.
4334  // If we are in copy mode, then use only transfer decoding.
4335  bool useMD5 = !m_contentMD5.isEmpty();
4336 
4337  // Deal with the size of the file.
4338  KIO::filesize_t sz = m_request.offset;
4339  if ( sz )
4340  m_iSize += sz;
4341 
4342  if (!m_isRedirection) {
4343  // Update the application with total size except when
4344  // it is compressed, or when the data is to be handled
4345  // internally (webDAV). If compressed we have to wait
4346  // until we uncompress to find out the actual data size
4347  if ( !dataInternal ) {
4348  if ((m_iSize > 0) && (m_iSize != NO_SIZE)) {
4349  totalSize(m_iSize);
4350  infoMessage(i18n("Retrieving %1 from %2...", KIO::convertSize(m_iSize),
4351  m_request.url.host()));
4352  } else {
4353  totalSize(0);
4354  }
4355  }
4356 
4357  if (m_request.cacheTag.ioMode == ReadFromCache) {
4358  kDebug(7113) << "reading data from cache...";
4359 
4360  m_iContentLeft = NO_SIZE;
4361 
4362  QByteArray d;
4363  while (true) {
4364  d = cacheFileReadPayload(MAX_IPC_SIZE);
4365  if (d.isEmpty()) {
4366  break;
4367  }
4368  slotData(d);
4369  sz += d.size();
4370  if (!dataInternal) {
4371  processedSize(sz);
4372  }
4373  }
4374 
4375  m_receiveBuf.resize(0);
4376 
4377  if (!dataInternal) {
4378  data(QByteArray());
4379  }
4380 
4381  return true;
4382  }
4383  }
4384 
4385  if (m_iSize != NO_SIZE)
4386  m_iBytesLeft = m_iSize - sz;
4387  else
4388  m_iBytesLeft = NO_SIZE;
4389 
4390  m_iContentLeft = m_iBytesLeft;
4391 
4392  if (m_isChunked)
4393  m_iBytesLeft = NO_SIZE;
4394 
4395  kDebug(7113) << KIO::number(m_iBytesLeft) << "bytes left.";
4396 
4397  // Main incoming loop... Gather everything while we can...
4398  m_cpMimeBuffer = false;
4399  m_mimeTypeBuffer.resize(0);
4400 
4401  HTTPFilterChain chain;
4402 
4403  // redirection ignores the body
4404  if (!m_isRedirection) {
4405  QObject::connect(&chain, SIGNAL(output(QByteArray)),
4406  this, SLOT(slotData(QByteArray)));
4407  }
4408  QObject::connect(&chain, SIGNAL(error(QString)),
4409  this, SLOT(slotFilterError(QString)));
4410 
4411  // decode all of the transfer encodings
4412  while (!m_transferEncodings.isEmpty())
4413  {
4414  QString enc = m_transferEncodings.takeLast();
4415  if ( enc == QLatin1String("gzip") )
4416  chain.addFilter(new HTTPFilterGZip);
4417  else if ( enc == QLatin1String("deflate") )
4418  chain.addFilter(new HTTPFilterDeflate);
4419  }
4420 
4421  // From HTTP 1.1 Draft 6:
4422  // The MD5 digest is computed based on the content of the entity-body,
4423  // including any content-coding that has been applied, but not including
4424  // any transfer-encoding applied to the message-body. If the message is
4425  // received with a transfer-encoding, that encoding MUST be removed
4426  // prior to checking the Content-MD5 value against the received entity.
4427  HTTPFilterMD5 *md5Filter = 0;
4428  if ( useMD5 )
4429  {
4430  md5Filter = new HTTPFilterMD5;
4431  chain.addFilter(md5Filter);
4432  }
4433 
4434  // now decode all of the content encodings
4435  // -- Why ?? We are not
4436  // -- a proxy server, be a client side implementation!! The applications
4437  // -- are capable of determinig how to extract the encoded implementation.
4438  // WB: That's a misunderstanding. We are free to remove the encoding.
4439  // WB: Some braindead www-servers however, give .tgz files an encoding
4440  // WB: of "gzip" (or even "x-gzip") and a content-type of "applications/tar"
4441  // WB: They shouldn't do that. We can work around that though...
4442  while (!m_contentEncodings.isEmpty())
4443  {
4444  QString enc = m_contentEncodings.takeLast();
4445  if ( enc == QLatin1String("gzip") )
4446  chain.addFilter(new HTTPFilterGZip);
4447  else if ( enc == QLatin1String("deflate") )
4448  chain.addFilter(new HTTPFilterDeflate);
4449  }
4450 
4451  while (!m_isEOF)
4452  {
4453  int bytesReceived;
4454 
4455  if (m_isChunked)
4456  bytesReceived = readChunked();
4457  else if (m_iSize != NO_SIZE)
4458  bytesReceived = readLimited();
4459  else
4460  bytesReceived = readUnlimited();
4461 
4462  // make sure that this wasn't an error, first
4463  // kDebug(7113) << "bytesReceived:"
4464  // << (int) bytesReceived << " m_iSize:" << (int) m_iSize << " Chunked:"
4465  // << m_isChunked << " BytesLeft:"<< (int) m_iBytesLeft;
4466  if (bytesReceived == -1)
4467  {
4468  if (m_iContentLeft == 0)
4469  {
4470  // gzip'ed data sometimes reports a too long content-length.
4471  // (The length of the unzipped data)
4472  m_iBytesLeft = 0;
4473  break;
4474  }
4475  // Oh well... log an error and bug out
4476  kDebug(7113) << "bytesReceived==-1 sz=" << (int)sz
4477  << " Connection broken !";
4478  error(ERR_CONNECTION_BROKEN, m_request.url.host());
4479  return false;
4480  }
4481 
4482  // I guess that nbytes == 0 isn't an error.. but we certainly
4483  // won't work with it!
4484  if (bytesReceived > 0)
4485  {
4486  // Important: truncate the buffer to the actual size received!
4487  // Otherwise garbage will be passed to the app
4488  m_receiveBuf.truncate( bytesReceived );
4489 
4490  chain.slotInput(m_receiveBuf);
4491 
4492  if (m_iError)
4493  return false;
4494 
4495  sz += bytesReceived;
4496  if (!dataInternal)
4497  processedSize( sz );
4498  }
4499  m_receiveBuf.resize(0); // res
4500 
4501  if (m_iBytesLeft && m_isEOD && !m_isChunked)
4502  {
4503  // gzip'ed data sometimes reports a too long content-length.
4504  // (The length of the unzipped data)
4505  m_iBytesLeft = 0;
4506  }
4507 
4508  if (m_iBytesLeft == 0)
4509  {
4510  kDebug(7113) << "EOD received! Left ="<< KIO::number(m_iBytesLeft);
4511  break;
4512  }
4513  }
4514  chain.slotInput(QByteArray()); // Flush chain.
4515 
4516  if ( useMD5 )
4517  {
4518  QString calculatedMD5 = md5Filter->md5();
4519 
4520  if ( m_contentMD5 != calculatedMD5 )
4521  kWarning(7113) << "MD5 checksum MISMATCH! Expected:"
4522  << calculatedMD5 << ", Got:" << m_contentMD5;
4523  }
4524 
4525  // Close cache entry
4526  if (m_iBytesLeft == 0) {
4527  cacheFileClose(); // no-op if not necessary
4528  }
4529 
4530  if (!dataInternal && sz <= 1)
4531  {
4532  if (m_request.responseCode >= 500 && m_request.responseCode <= 599) {
4533  error(ERR_INTERNAL_SERVER, m_request.url.host());
4534  return false;
4535  } else if (m_request.responseCode >= 400 && m_request.responseCode <= 499 &&
4536  !isAuthenticationRequired(m_request.responseCode)) {
4537  error(ERR_DOES_NOT_EXIST, m_request.url.host());
4538  return false;
4539  }
4540  }
4541 
4542  if (!dataInternal && !m_isRedirection)
4543  data( QByteArray() );
4544 
4545  return true;
4546 }
4547 
4548 void HTTPProtocol::slotFilterError(const QString &text)
4549 {
4550  error(KIO::ERR_SLAVE_DEFINED, text);
4551 }
4552 
4553 void HTTPProtocol::error( int _err, const QString &_text )
4554 {
4555  // Close the connection only on connection errors. Otherwise, honor the
4556  // keep alive flag.
4557  if (_err == ERR_CONNECTION_BROKEN || _err == ERR_COULD_NOT_CONNECT)
4558  httpClose(false);
4559  else
4560  httpClose(m_request.isKeepAlive);
4561 
4562  if (!m_request.id.isEmpty())
4563  {
4564  forwardHttpResponseHeader();
4565  sendMetaData();
4566  }
4567 
4568  // It's over, we don't need it anymore
4569  clearPostDataBuffer();
4570 
4571  SlaveBase::error( _err, _text );
4572  m_iError = _err;
4573 }
4574 
4575 
4576 void HTTPProtocol::addCookies( const QString &url, const QByteArray &cookieHeader )
4577 {
4578  qlonglong windowId = m_request.windowId.toLongLong();
4579  QDBusInterface kcookiejar( QLatin1String("org.kde.kded"), QLatin1String("/modules/kcookiejar"), QLatin1String("org.kde.KCookieServer") );
4580  (void)kcookiejar.call( QDBus::NoBlock, QLatin1String("addCookies"), url,
4581  cookieHeader, windowId );
4582 }
4583 
4584 QString HTTPProtocol::findCookies( const QString &url)
4585 {
4586  qlonglong windowId = m_request.windowId.toLongLong();
4587  QDBusInterface kcookiejar( QLatin1String("org.kde.kded"), QLatin1String("/modules/kcookiejar"), QLatin1String("org.kde.KCookieServer") );
4588  QDBusReply<QString> reply = kcookiejar.call( QLatin1String("findCookies"), url, windowId );
4589 
4590  if ( !reply.isValid() )
4591  {
4592  kWarning(7113) << "Can't communicate with kded_kcookiejar!";
4593  return QString();
4594  }
4595  return reply;
4596 }
4597 
4598 /******************************* CACHING CODE ****************************/
4599 
4600 HTTPProtocol::CacheTag::CachePlan HTTPProtocol::CacheTag::plan(time_t maxCacheAge) const
4601 {
4602  //notable omission: we're not checking cache file presence or integrity
4603  switch (policy) {
4604  case KIO::CC_Refresh:
4605  // Conditional GET requires the presence of either an ETag or
4606  // last modified date.
4607  if (lastModifiedDate != -1 || !etag.isEmpty()) {
4608  return ValidateCached;
4609  }
4610  break;
4611  case KIO::CC_Reload:
4612  return IgnoreCached;
4613  case KIO::CC_CacheOnly:
4614  case KIO::CC_Cache:
4615  return UseCached;
4616  default:
4617  break;
4618  }
4619 
4620  Q_ASSERT((policy == CC_Verify || policy == CC_Refresh));
4621  time_t currentDate = time(0);
4622  if ((servedDate != -1 && currentDate > (servedDate + maxCacheAge)) ||
4623  (expireDate != -1 && currentDate > expireDate)) {
4624  return ValidateCached;
4625  }
4626  return UseCached;
4627 }
4628 
4629 // !START SYNC!
4630 // The following code should be kept in sync
4631 // with the code in http_cache_cleaner.cpp
4632 
4633 // we use QDataStream; this is just an illustration
4634 struct BinaryCacheFileHeader
4635 {
4636  quint8 version[2];
4637  quint8 compression; // for now fixed to 0
4638  quint8 reserved; // for now; also alignment
4639  qint32 useCount;
4640  qint64 servedDate;
4641  qint64 lastModifiedDate;
4642  qint64 expireDate;
4643  qint32 bytesCached;
4644  // packed size should be 36 bytes; we explicitly set it here to make sure that no compiler
4645  // padding ruins it. We write the fields to disk without any padding.
4646  static const int size = 36;
4647 };
4648 
4649 enum CacheCleanerCommandCode {
4650  InvalidCommand = 0,
4651  CreateFileNotificationCommand,
4652  UpdateFileCommand
4653 };
4654 
4655 // illustration for cache cleaner update "commands"
4656 struct CacheCleanerCommand
4657 {
4658  BinaryCacheFileHeader header;
4659  quint32 commandCode;
4660  // filename in ASCII, binary isn't worth the coding and decoding
4661  quint8 filename[s_hashedUrlNibbles];
4662 };
4663 
4664 QByteArray HTTPProtocol::CacheTag::serialize() const
4665 {
4666  QByteArray ret;
4667  QDataStream stream(&ret, QIODevice::WriteOnly);
4668  stream << quint8('A');
4669  stream << quint8('\n');
4670  stream << quint8(0);
4671  stream << quint8(0);
4672 
4673  stream << fileUseCount;
4674 
4675  // time_t overflow will only be checked when reading; we have no way to tell here.
4676  stream << qint64(servedDate);
4677  stream << qint64(lastModifiedDate);
4678  stream << qint64(expireDate);
4679 
4680  stream << bytesCached;
4681  Q_ASSERT(ret.size() == BinaryCacheFileHeader::size);
4682  return ret;
4683 }
4684 
4685 
4686 static bool compareByte(QDataStream *stream, quint8 value)
4687 {
4688  quint8 byte;
4689  *stream >> byte;
4690  return byte == value;
4691 }
4692 
4693 static bool readTime(QDataStream *stream, time_t *time)
4694 {
4695  qint64 intTime = 0;
4696  *stream >> intTime;
4697  *time = static_cast<time_t>(intTime);
4698 
4699  qint64 check = static_cast<qint64>(*time);
4700  return check == intTime;
4701 }
4702 
4703 // If starting a new file cacheFileWriteVariableSizeHeader() must have been called *before*
4704 // calling this! This is to fill in the headerEnd field.
4705 // If the file is not new headerEnd has already been read from the file and in fact the variable
4706 // size header *may* not be rewritten because a size change would mess up the file layout.
4707 bool HTTPProtocol::CacheTag::deserialize(const QByteArray &d)
4708 {
4709  if (d.size() != BinaryCacheFileHeader::size) {
4710  return false;
4711  }
4712  QDataStream stream(d);
4713  stream.setVersion(QDataStream::Qt_4_5);
4714 
4715  bool ok = true;
4716  ok = ok && compareByte(&stream, 'A');
4717  ok = ok && compareByte(&stream, '\n');
4718  ok = ok && compareByte(&stream, 0);
4719  ok = ok && compareByte(&stream, 0);
4720  if (!ok) {
4721  return false;
4722  }
4723 
4724  stream >> fileUseCount;
4725 
4726  // read and check for time_t overflow
4727  ok = ok && readTime(&stream, &servedDate);
4728  ok = ok && readTime(&stream, &lastModifiedDate);
4729  ok = ok && readTime(&stream, &expireDate);
4730  if (!ok) {
4731  return false;
4732  }
4733 
4734  stream >> bytesCached;
4735 
4736  return true;
4737 }
4738 
4739 /* Text part of the header, directly following the binary first part:
4740 URL\n
4741 etag\n
4742 mimetype\n
4743 header line\n
4744 header line\n
4745 ...
4746 \n
4747 */
4748 
4749 static KUrl storableUrl(const KUrl &url)
4750 {
4751  KUrl ret(url);
4752  ret.setPassword(QString());
4753  ret.setFragment(QString());
4754  return ret;
4755 }
4756 
4757 static void writeLine(QIODevice *dev, const QByteArray &line)
4758 {
4759  static const char linefeed = '\n';
4760  dev->write(line);
4761  dev->write(&linefeed, 1);
4762 }
4763 
4764 void HTTPProtocol::cacheFileWriteTextHeader()
4765 {
4766  QFile *&file = m_request.cacheTag.file;
4767  Q_ASSERT(file);
4768  Q_ASSERT(file->openMode() & QIODevice::WriteOnly);
4769 
4770  file->seek(BinaryCacheFileHeader::size);
4771  writeLine(file, storableUrl(m_request.url).toEncoded());
4772  writeLine(file, m_request.cacheTag.etag.toLatin1());
4773  writeLine(file, m_mimeType.toLatin1());
4774  writeLine(file, m_responseHeaders.join(QString(QLatin1Char('\n'))).toLatin1());
4775  // join("\n") adds no \n to the end, but writeLine() does.
4776  // Add another newline to mark the end of text.
4777  writeLine(file, QByteArray());
4778 }
4779 
4780 static bool readLineChecked(QIODevice *dev, QByteArray *line)
4781 {
4782  *line = dev->readLine(MAX_IPC_SIZE);
4783  // if nothing read or the line didn't fit into 8192 bytes(!)
4784  if (line->isEmpty() || !line->endsWith('\n')) {
4785  return false;
4786  }
4787  // we don't actually want the newline!
4788  line->chop(1);
4789  return true;
4790 }
4791 
4792 bool HTTPProtocol::cacheFileReadTextHeader1(const KUrl &desiredUrl)
4793 {
4794  QFile *&file = m_request.cacheTag.file;
4795  Q_ASSERT(file);
4796  Q_ASSERT(file->openMode() == QIODevice::ReadOnly);
4797 
4798  QByteArray readBuf;
4799  bool ok = readLineChecked(file, &readBuf);
4800  if (storableUrl(desiredUrl).toEncoded() != readBuf) {
4801  kDebug(7103) << "You have witnessed a very improbable hash collision!";
4802  return false;
4803  }
4804 
4805  ok = ok && readLineChecked(file, &readBuf);
4806  m_request.cacheTag.etag = toQString(readBuf);
4807 
4808  return ok;
4809 }
4810 
4811 bool HTTPProtocol::cacheFileReadTextHeader2()
4812 {
4813  QFile *&file = m_request.cacheTag.file;
4814  Q_ASSERT(file);
4815  Q_ASSERT(file->openMode() == QIODevice::ReadOnly);
4816 
4817  bool ok = true;
4818  QByteArray readBuf;
4819 #ifndef NDEBUG
4820  // we assume that the URL and etag have already been read
4821  qint64 oldPos = file->pos();
4822  file->seek(BinaryCacheFileHeader::size);
4823  ok = ok && readLineChecked(file, &readBuf);
4824  ok = ok && readLineChecked(file, &readBuf);
4825  Q_ASSERT(file->pos() == oldPos);
4826 #endif
4827  ok = ok && readLineChecked(file, &readBuf);
4828  m_mimeType = toQString(readBuf);
4829 
4830  m_responseHeaders.clear();
4831  // read as long as no error and no empty line found
4832  while (true) {
4833  ok = ok && readLineChecked(file, &readBuf);
4834  if (ok && !readBuf.isEmpty()) {
4835  m_responseHeaders.append(toQString(readBuf));
4836  } else {
4837  break;
4838  }
4839  }
4840  return ok; // it may still be false ;)
4841 }
4842 
4843 static QString filenameFromUrl(const KUrl &url)
4844 {
4845  QCryptographicHash hash(QCryptographicHash::Sha1);
4846  hash.addData(storableUrl(url).toEncoded());
4847  return toQString(hash.result().toHex());
4848 }
4849 
4850 QString HTTPProtocol::cacheFilePathFromUrl(const KUrl &url) const
4851 {
4852  QString filePath = m_strCacheDir;
4853  if (!filePath.endsWith(QLatin1Char('/'))) {
4854  filePath.append(QLatin1Char('/'));
4855  }
4856  filePath.append(filenameFromUrl(url));
4857  return filePath;
4858 }
4859 
4860 bool HTTPProtocol::cacheFileOpenRead()
4861 {
4862  kDebug(7113);
4863  QString filename = cacheFilePathFromUrl(m_request.url);
4864 
4865  QFile *&file = m_request.cacheTag.file;
4866  if (file) {
4867  kDebug(7113) << "File unexpectedly open; old file is" << file->fileName()
4868  << "new name is" << filename;
4869  Q_ASSERT(file->fileName() == filename);
4870  }
4871  Q_ASSERT(!file);
4872  file = new QFile(filename);
4873  if (file->open(QIODevice::ReadOnly)) {
4874  QByteArray header = file->read(BinaryCacheFileHeader::size);
4875  if (!m_request.cacheTag.deserialize(header)) {
4876  kDebug(7103) << "Cache file header is invalid.";
4877 
4878  file->close();
4879  }
4880  }
4881 
4882  if (file->isOpen() && !cacheFileReadTextHeader1(m_request.url)) {
4883  file->close();
4884  }
4885 
4886  if (!file->isOpen()) {
4887  cacheFileClose();
4888  return false;
4889  }
4890  return true;
4891 }
4892 
4893 
4894 bool HTTPProtocol::cacheFileOpenWrite()
4895 {
4896  kDebug(7113);
4897  QString filename = cacheFilePathFromUrl(m_request.url);
4898 
4899  // if we open a cache file for writing while we have a file open for reading we must have
4900  // found out that the old cached content is obsolete, so delete the file.
4901  QFile *&file = m_request.cacheTag.file;
4902  if (file) {
4903  // ensure that the file is in a known state - either open for reading or null
4904  Q_ASSERT(!qobject_cast<QTemporaryFile *>(file));
4905  Q_ASSERT((file->openMode() & QIODevice::WriteOnly) == 0);
4906  Q_ASSERT(file->fileName() == filename);
4907  kDebug(7113) << "deleting expired cache entry and recreating.";
4908  file->remove();
4909  delete file;
4910  file = 0;
4911  }
4912 
4913  // note that QTemporaryFile will automatically append random chars to filename
4914  file = new QTemporaryFile(filename);
4915  file->open(QIODevice::WriteOnly);
4916 
4917  // if we have started a new file we have not initialized some variables from disk data.
4918  m_request.cacheTag.fileUseCount = 0; // the file has not been *read* yet
4919  m_request.cacheTag.bytesCached = 0;
4920 
4921  if ((file->openMode() & QIODevice::WriteOnly) == 0) {
4922  kDebug(7113) << "Could not open file for writing:" << file->fileName()
4923  << "due to error" << file->error();
4924  cacheFileClose();
4925  return false;
4926  }
4927  return true;
4928 }
4929 
4930 static QByteArray makeCacheCleanerCommand(const HTTPProtocol::CacheTag &cacheTag,
4931  CacheCleanerCommandCode cmd)
4932 {
4933  QByteArray ret = cacheTag.serialize();
4934  QDataStream stream(&ret, QIODevice::WriteOnly);
4935  stream.setVersion(QDataStream::Qt_4_5);
4936 
4937  stream.skipRawData(BinaryCacheFileHeader::size);
4938  // append the command code
4939  stream << quint32(cmd);
4940  // append the filename
4941  QString fileName = cacheTag.file->fileName();
4942  int basenameStart = fileName.lastIndexOf(QLatin1Char('/')) + 1;
4943  QByteArray baseName = fileName.mid(basenameStart, s_hashedUrlNibbles).toLatin1();
4944  stream.writeRawData(baseName.constData(), baseName.size());
4945 
4946  Q_ASSERT(ret.size() == BinaryCacheFileHeader::size + sizeof(quint32) + s_hashedUrlNibbles);
4947  return ret;
4948 }
4949 
4950 //### not yet 100% sure when and when not to call this
4951 void HTTPProtocol::cacheFileClose()
4952 {
4953  kDebug(7113);
4954 
4955  QFile *&file = m_request.cacheTag.file;
4956  if (!file) {
4957  return;
4958  }
4959 
4960  m_request.cacheTag.ioMode = NoCache;
4961 
4962  QByteArray ccCommand;
4963  QTemporaryFile *tempFile = qobject_cast<QTemporaryFile *>(file);
4964 
4965  if (file->openMode() & QIODevice::WriteOnly) {
4966  Q_ASSERT(tempFile);
4967 
4968  if (m_request.cacheTag.bytesCached && !m_iError) {
4969  QByteArray header = m_request.cacheTag.serialize();
4970  tempFile->seek(0);
4971  tempFile->write(header);
4972 
4973  ccCommand = makeCacheCleanerCommand(m_request.cacheTag, CreateFileNotificationCommand);
4974 
4975  QString oldName = tempFile->fileName();
4976  QString newName = oldName;
4977  int basenameStart = newName.lastIndexOf(QLatin1Char('/')) + 1;
4978  // remove the randomized name part added by QTemporaryFile
4979  newName.chop(newName.length() - basenameStart - s_hashedUrlNibbles);
4980  kDebug(7113) << "Renaming temporary file" << oldName << "to" << newName;
4981 
4982  // on windows open files can't be renamed
4983  tempFile->setAutoRemove(false);
4984  delete tempFile;
4985  file = 0;
4986 
4987  if (!QFile::rename(oldName, newName)) {
4988  // ### currently this hides a minor bug when force-reloading a resource. We
4989  // should not even open a new file for writing in that case.
4990  kDebug(7113) << "Renaming temporary file failed, deleting it instead.";
4991  QFile::remove(oldName);
4992  ccCommand.clear(); // we have nothing of value to tell the cache cleaner
4993  }
4994  } else {
4995  // oh, we've never written payload data to the cache file.
4996  // the temporary file is closed and removed and no proper cache entry is created.
4997  }
4998  } else if (file->openMode() == QIODevice::ReadOnly) {
4999  Q_ASSERT(!tempFile);
5000  ccCommand = makeCacheCleanerCommand(m_request.cacheTag, UpdateFileCommand);
5001  }
5002  delete file;
5003  file = 0;
5004 
5005  if (!ccCommand.isEmpty()) {
5006  sendCacheCleanerCommand(ccCommand);
5007  }
5008 }
5009 
5010 void HTTPProtocol::sendCacheCleanerCommand(const QByteArray &command)
5011 {
5012  kDebug(7113);
5013  Q_ASSERT(command.size() == BinaryCacheFileHeader::size + s_hashedUrlNibbles + sizeof(quint32));
5014  int attempts = 0;
5015  while (m_cacheCleanerConnection.state() != QLocalSocket::ConnectedState && attempts < 6) {
5016  if (attempts == 2) {
5017  KToolInvocation::startServiceByDesktopPath(QLatin1String("http_cache_cleaner.desktop"));
5018  }
5019  QString socketFileName = KStandardDirs::locateLocal("socket", QLatin1String("kio_http_cache_cleaner"));
5020  m_cacheCleanerConnection.connectToServer(socketFileName, QIODevice::WriteOnly);
5021  m_cacheCleanerConnection.waitForConnected(1500);
5022  attempts++;
5023  }
5024 
5025  if (m_cacheCleanerConnection.state() == QLocalSocket::ConnectedState) {
5026  m_cacheCleanerConnection.write(command);
5027  m_cacheCleanerConnection.flush();
5028  } else {
5029  // updating the stats is not vital, so we just give up.
5030  kDebug(7113) << "Could not connect to cache cleaner, not updating stats of this cache file.";
5031  }
5032 }
5033 
5034 QByteArray HTTPProtocol::cacheFileReadPayload(int maxLength)
5035 {
5036  Q_ASSERT(m_request.cacheTag.file);
5037  Q_ASSERT(m_request.cacheTag.ioMode == ReadFromCache);
5038  Q_ASSERT(m_request.cacheTag.file->openMode() == QIODevice::ReadOnly);
5039  QByteArray ret = m_request.cacheTag.file->read(maxLength);
5040  if (ret.isEmpty()) {
5041  cacheFileClose();
5042  }
5043  return ret;
5044 }
5045 
5046 
5047 void HTTPProtocol::cacheFileWritePayload(const QByteArray &d)
5048 {
5049  if (!m_request.cacheTag.file) {
5050  return;
5051  }
5052 
5053  // If the file being downloaded is so big that it exceeds the max cache size,
5054  // do not cache it! See BR# 244215. NOTE: this can be improved upon in the
5055  // future...
5056  if (m_iSize >= KIO::filesize_t(m_maxCacheSize * 1024)) {
5057  kDebug(7113) << "Caching disabled because content size is too big.";
5058  cacheFileClose();
5059  return;
5060  }
5061 
5062  Q_ASSERT(m_request.cacheTag.ioMode == WriteToCache);
5063  Q_ASSERT(m_request.cacheTag.file->openMode() & QIODevice::WriteOnly);
5064 
5065  if (d.isEmpty()) {
5066  cacheFileClose();
5067  }
5068 
5069  //TODO: abort if file grows too big!
5070 
5071  // write the variable length text header as soon as we start writing to the file
5072  if (!m_request.cacheTag.bytesCached) {
5073  cacheFileWriteTextHeader();
5074  }
5075  m_request.cacheTag.bytesCached += d.size();
5076  m_request.cacheTag.file->write(d);
5077 }
5078 
5079 void HTTPProtocol::cachePostData(const QByteArray& data)
5080 {
5081  if (!m_POSTbuf) {
5082  m_POSTbuf = createPostBufferDeviceFor(qMax(m_iPostDataSize, static_cast<KIO::filesize_t>(data.size())));
5083  if (!m_POSTbuf)
5084  return;
5085  }
5086 
5087  m_POSTbuf->write (data.constData(), data.size());
5088 }
5089 
5090 void HTTPProtocol::clearPostDataBuffer()
5091 {
5092  if (!m_POSTbuf)
5093  return;
5094 
5095  delete m_POSTbuf;
5096  m_POSTbuf = 0;
5097 }
5098 
5099 bool HTTPProtocol::retrieveAllData()
5100 {
5101  if (!m_POSTbuf) {
5102  m_POSTbuf = createPostBufferDeviceFor(s_MaxInMemPostBufSize + 1);
5103  }
5104 
5105  if (!m_POSTbuf) {
5106  error (ERR_OUT_OF_MEMORY, m_request.url.host());
5107  return false;
5108  }
5109 
5110  while (true) {
5111  dataReq();
5112  QByteArray buffer;
5113  const int bytesRead = readData(buffer);
5114 
5115  if (bytesRead < 0) {
5116  error(ERR_ABORTED, m_request.url.host());
5117  return false;
5118  }
5119 
5120  if (bytesRead == 0) {
5121  break;
5122  }
5123 
5124  m_POSTbuf->write(buffer.constData(), buffer.size());
5125  }
5126 
5127  return true;
5128 }
5129 
5130 // The above code should be kept in sync
5131 // with the code in http_cache_cleaner.cpp
5132 // !END SYNC!
5133 
5134 //************************** AUTHENTICATION CODE ********************/
5135 
5136 QString HTTPProtocol::authenticationHeader()
5137 {
5138  QByteArray ret;
5139 
5140  // If the internal meta-data "cached-www-auth" is set, then check for cached
5141  // authentication data and preemtively send the authentication header if a
5142  // matching one is found.
5143  if (!m_wwwAuth && config()->readEntry("cached-www-auth", false)) {
5144  KIO::AuthInfo authinfo;
5145  authinfo.url = m_request.url;
5146  authinfo.realmValue = config()->readEntry("www-auth-realm", QString());
5147  // If no relam metadata, then make sure path matching is turned on.
5148  authinfo.verifyPath = (authinfo.realmValue.isEmpty());
5149 
5150  const bool useCachedAuth = (m_request.responseCode == 401 || !config()->readEntry("no-preemptive-auth-reuse", false));
5151 
5152  if (useCachedAuth && checkCachedAuthentication(authinfo)) {
5153  const QByteArray cachedChallenge = config()->readEntry("www-auth-challenge", QByteArray());
5154  if (!cachedChallenge.isEmpty()) {
5155  m_wwwAuth = KAbstractHttpAuthentication::newAuth(cachedChallenge, config());
5156  if (m_wwwAuth) {
5157  kDebug(7113) << "creating www authentcation header from cached info";
5158  m_wwwAuth->setChallenge(cachedChallenge, m_request.url, m_request.methodString());
5159  m_wwwAuth->generateResponse(authinfo.username, authinfo.password);
5160  }
5161  }
5162  }
5163  }
5164 
5165  // If the internal meta-data "cached-proxy-auth" is set, then check for cached
5166  // authentication data and preemtively send the authentication header if a
5167  // matching one is found.
5168  if (!m_proxyAuth && config()->readEntry("cached-proxy-auth", false)) {
5169  KIO::AuthInfo authinfo;
5170  authinfo.url = m_request.proxyUrl;
5171  authinfo.realmValue = config()->readEntry("proxy-auth-realm", QString());
5172  // If no relam metadata, then make sure path matching is turned on.
5173  authinfo.verifyPath = (authinfo.realmValue.isEmpty());
5174 
5175  if (checkCachedAuthentication(authinfo)) {
5176  const QByteArray cachedChallenge = config()->readEntry("proxy-auth-challenge", QByteArray());
5177  if (!cachedChallenge.isEmpty()) {
5178  m_proxyAuth = KAbstractHttpAuthentication::newAuth(cachedChallenge, config());
5179  if (m_proxyAuth) {
5180  kDebug(7113) << "creating proxy authentcation header from cached info";
5181  m_proxyAuth->setChallenge(cachedChallenge, m_request.proxyUrl, m_request.methodString());
5182  m_proxyAuth->generateResponse(authinfo.username, authinfo.password);
5183  }
5184  }
5185  }
5186  }
5187 
5188  // the authentication classes don't know if they are for proxy or webserver authentication...
5189  if (m_wwwAuth && !m_wwwAuth->isError()) {
5190  ret += "Authorization: ";
5191  ret += m_wwwAuth->headerFragment();
5192  }
5193 
5194  if (m_proxyAuth && !m_proxyAuth->isError()) {
5195  ret += "Proxy-Authorization: ";
5196  ret += m_proxyAuth->headerFragment();
5197  }
5198 
5199  return toQString(ret); // ## encoding ok?
5200 }
5201 
5202 static QString protocolForProxyType(QNetworkProxy::ProxyType type)
5203 {
5204  switch (type) {
5205  case QNetworkProxy::DefaultProxy:
5206  break;
5207  case QNetworkProxy::Socks5Proxy:
5208  return QLatin1String("socks");
5209  case QNetworkProxy::NoProxy:
5210  break;
5211  case QNetworkProxy::HttpProxy:
5212  case QNetworkProxy::HttpCachingProxy:
5213  case QNetworkProxy::FtpCachingProxy:
5214  default:
5215  break;
5216  }
5217 
5218  return QLatin1String("http");
5219 }
5220 
5221 void HTTPProtocol::proxyAuthenticationForSocket(const QNetworkProxy &proxy, QAuthenticator *authenticator)
5222 {
5223  kDebug(7113) << "realm:" << authenticator->realm() << "user:" << authenticator->user();
5224 
5225  // Set the proxy URL...
5226  m_request.proxyUrl.setProtocol(protocolForProxyType(proxy.type()));
5227  m_request.proxyUrl.setUser(proxy.user());
5228  m_request.proxyUrl.setHost(proxy.hostName());
5229  m_request.proxyUrl.setPort(proxy.port());
5230 
5231  AuthInfo info;
5232  info.url = m_request.proxyUrl;
5233  info.realmValue = authenticator->realm();
5234  info.username = authenticator->user();
5235  info.verifyPath = info.realmValue.isEmpty();
5236 
5237  const bool haveCachedCredentials = checkCachedAuthentication(info);
5238  const bool retryAuth = (m_socketProxyAuth != 0);
5239 
5240  // if m_socketProxyAuth is a valid pointer then authentication has been attempted before,
5241  // and it was not successful. see below and saveProxyAuthenticationForSocket().
5242  if (!haveCachedCredentials || retryAuth) {
5243  // Save authentication info if the connection succeeds. We need to disconnect
5244  // this after saving the auth data (or an error) so we won't save garbage afterwards!
5245  connect(socket(), SIGNAL(connected()),
5246  this, SLOT(saveProxyAuthenticationForSocket()));
5247  //### fillPromptInfo(&info);
5248  info.prompt = i18n("You need to supply a username and a password for "
5249  "the proxy server listed below before you are allowed "
5250  "to access any sites.");
5251  info.keepPassword = true;
5252  info.commentLabel = i18n("Proxy:");
5253  info.comment = i18n("<b>%1</b> at <b>%2</b>", htmlEscape(info.realmValue), m_request.proxyUrl.host());
5254 
5255  const QString errMsg ((retryAuth ? i18n("Proxy Authentication Failed.") : QString()));
5256 
5257  if (!openPasswordDialog(info, errMsg)) {
5258  kDebug(7113) << "looks like the user canceled proxy authentication.";
5259  error(ERR_USER_CANCELED, m_request.proxyUrl.host());
5260  delete m_proxyAuth;
5261  m_proxyAuth = 0;
5262  return;
5263  }
5264  }
5265  authenticator->setUser(info.username);
5266  authenticator->setPassword(info.password);
5267  authenticator->setOption(QLatin1String("keepalive"), info.keepPassword);
5268 
5269  if (m_socketProxyAuth) {
5270  *m_socketProxyAuth = *authenticator;
5271  } else {
5272  m_socketProxyAuth = new QAuthenticator(*authenticator);
5273  }
5274 
5275  if (!m_request.proxyUrl.user().isEmpty()) {
5276  m_request.proxyUrl.setUser(info.username);
5277  }
5278 }
5279 
5280 void HTTPProtocol::saveProxyAuthenticationForSocket()
5281 {
5282  kDebug(7113) << "Saving authenticator";
5283  disconnect(socket(), SIGNAL(connected()),
5284  this, SLOT(saveProxyAuthenticationForSocket()));
5285  Q_ASSERT(m_socketProxyAuth);
5286  if (m_socketProxyAuth) {
5287  kDebug(7113) << "realm:" << m_socketProxyAuth->realm() << "user:" << m_socketProxyAuth->user();
5288  KIO::AuthInfo a;
5289  a.verifyPath = true;
5290  a.url = m_request.proxyUrl;
5291  a.realmValue = m_socketProxyAuth->realm();
5292  a.username = m_socketProxyAuth->user();
5293  a.password = m_socketProxyAuth->password();
5294  a.keepPassword = m_socketProxyAuth->option(QLatin1String("keepalive")).toBool();
5295  cacheAuthentication(a);
5296  }
5297  delete m_socketProxyAuth;
5298  m_socketProxyAuth = 0;
5299 }
5300 
5301 void HTTPProtocol::saveAuthenticationData()
5302 {
5303  KIO::AuthInfo authinfo;
5304  bool alreadyCached = false;
5305  KAbstractHttpAuthentication *auth = 0;
5306  switch (m_request.prevResponseCode) {
5307  case 401:
5308  auth = m_wwwAuth;
5309  alreadyCached = config()->readEntry("cached-www-auth", false);
5310  break;
5311  case 407:
5312  auth = m_proxyAuth;
5313  alreadyCached = config()->readEntry("cached-proxy-auth", false);
5314  break;
5315  default:
5316  Q_ASSERT(false); // should never happen!
5317  }
5318 
5319  // Prevent recaching of the same credentials over and over again.
5320  if (auth && (!auth->realm().isEmpty() || !alreadyCached)) {
5321  auth->fillKioAuthInfo(&authinfo);
5322  if (auth == m_wwwAuth) {
5323  setMetaData(QLatin1String("{internal~currenthost}cached-www-auth"), QLatin1String("true"));
5324  if (!authinfo.realmValue.isEmpty())
5325  setMetaData(QLatin1String("{internal~currenthost}www-auth-realm"), authinfo.realmValue);
5326  if (!authinfo.digestInfo.isEmpty())
5327  setMetaData(QLatin1String("{internal~currenthost}www-auth-challenge"), authinfo.digestInfo);
5328  } else {
5329  setMetaData(QLatin1String("{internal~allhosts}cached-proxy-auth"), QLatin1String("true"));
5330  if (!authinfo.realmValue.isEmpty())
5331  setMetaData(QLatin1String("{internal~allhosts}proxy-auth-realm"), authinfo.realmValue);
5332  if (!authinfo.digestInfo.isEmpty())
5333  setMetaData(QLatin1String("{internal~allhosts}proxy-auth-challenge"), authinfo.digestInfo);
5334  }
5335 
5336  kDebug(7113) << "Cache authentication info ?" << authinfo.keepPassword;
5337 
5338  if (authinfo.keepPassword) {
5339  cacheAuthentication(authinfo);
5340  kDebug(7113) << "Cached authentication for" << m_request.url;
5341  }
5342  }
5343  // Update our server connection state which includes www and proxy username and password.
5344  m_server.updateCredentials(m_request);
5345 }
5346 
5347 bool HTTPProtocol::handleAuthenticationHeader(const HeaderTokenizer* tokenizer)
5348 {
5349  KIO::AuthInfo authinfo;
5350  QList<QByteArray> authTokens;
5351  KAbstractHttpAuthentication **auth;
5352 
5353  if (m_request.responseCode == 401) {
5354  auth = &m_wwwAuth;
5355  authTokens = tokenizer->iterator("www-authenticate").all();
5356  authinfo.url = m_request.url;
5357  authinfo.username = m_server.url.user();
5358  authinfo.prompt = i18n("You need to supply a username and a "
5359  "password to access this site.");
5360  authinfo.commentLabel = i18n("Site:");
5361  } else {
5362  // make sure that the 407 header hasn't escaped a lower layer when it shouldn't.
5363  // this may break proxy chains which were never tested anyway, and AFAIK they are
5364  // rare to nonexistent in the wild.
5365  Q_ASSERT(QNetworkProxy::applicationProxy().type() == QNetworkProxy::NoProxy);
5366  auth = &m_proxyAuth;
5367  authTokens = tokenizer->iterator("proxy-authenticate").all();
5368  authinfo.url = m_request.proxyUrl;
5369  authinfo.username = m_request.proxyUrl.user();
5370  authinfo.prompt = i18n("You need to supply a username and a password for "
5371  "the proxy server listed below before you are allowed "
5372  "to access any sites." );
5373  authinfo.commentLabel = i18n("Proxy:");
5374  }
5375 
5376  bool authRequiresAnotherRoundtrip = false;
5377 
5378  // Workaround brain dead server responses that violate the spec and
5379  // incorrectly return a 401/407 without the required WWW/Proxy-Authenticate
5380  // header fields. See bug 215736...
5381  if (!authTokens.isEmpty()) {
5382  QString errorMsg;
5383  authRequiresAnotherRoundtrip = true;
5384 
5385  if (m_request.responseCode == m_request.prevResponseCode && *auth) {
5386  // Authentication attempt failed. Retry...
5387  if ((*auth)->wasFinalStage()) {
5388  errorMsg = (m_request.responseCode == 401 ?
5389  i18n("Authentication Failed.") :
5390  i18n("Proxy Authentication Failed."));
5391  delete *auth;
5392  *auth = 0;
5393  } else { // Create authentication header
5394  // WORKAROUND: The following piece of code prevents brain dead IIS
5395  // servers that send back multiple "WWW-Authenticate" headers from
5396  // screwing up our authentication logic during the challenge
5397  // phase (Type 2) of NTLM authenticaiton.
5398  QMutableListIterator<QByteArray> it (authTokens);
5399  const QByteArray authScheme ((*auth)->scheme().trimmed());
5400  while (it.hasNext()) {
5401  if (qstrnicmp(authScheme.constData(), it.next().constData(), authScheme.length()) != 0) {
5402  it.remove();
5403  }
5404  }
5405  }
5406  }
5407 
5408 try_next_auth_scheme:
5409  QByteArray bestOffer = KAbstractHttpAuthentication::bestOffer(authTokens);
5410  if (*auth) {
5411  const QByteArray authScheme ((*auth)->scheme().trimmed());
5412  if (qstrnicmp(authScheme.constData(), bestOffer.constData(), authScheme.length()) != 0) {
5413  // huh, the strongest authentication scheme offered has changed.
5414  delete *auth;
5415  *auth = 0;
5416  }
5417  }
5418 
5419  if (!(*auth)) {
5420  *auth = KAbstractHttpAuthentication::newAuth(bestOffer, config());
5421  }
5422 
5423  if (*auth) {
5424  kDebug(7113) << "Trying authentication scheme:" << (*auth)->scheme();
5425 
5426  // remove trailing space from the method string, or digest auth will fail
5427  (*auth)->setChallenge(bestOffer, authinfo.url, m_request.methodString());
5428 
5429  QString username, password;
5430  bool generateAuthHeader = true;
5431  if ((*auth)->needCredentials()) {
5432  // use credentials supplied by the application if available
5433  if (!m_request.url.user().isEmpty() && !m_request.url.pass().isEmpty()) {
5434  username = m_request.url.user();
5435  password = m_request.url.pass();
5436  // don't try this password any more
5437  m_request.url.setPass(QString());
5438  } else {
5439  // try to get credentials from kpasswdserver's cache, then try asking the user.
5440  authinfo.verifyPath = false; // we have realm, no path based checking please!
5441  authinfo.realmValue = (*auth)->realm();
5442  if (authinfo.realmValue.isEmpty() && !(*auth)->supportsPathMatching())
5443  authinfo.realmValue = QLatin1String((*auth)->scheme());
5444 
5445  // Save the current authinfo url because it can be modified by the call to
5446  // checkCachedAuthentication. That way we can restore it if the call
5447  // modified it.
5448  const KUrl reqUrl = authinfo.url;
5449  if (!errorMsg.isEmpty() || !checkCachedAuthentication(authinfo)) {
5450  // Reset url to the saved url...
5451  authinfo.url = reqUrl;
5452  authinfo.keepPassword = true;
5453  authinfo.comment = i18n("<b>%1</b> at <b>%2</b>",
5454  htmlEscape(authinfo.realmValue), authinfo.url.host());
5455 
5456  if (!openPasswordDialog(authinfo, errorMsg)) {
5457  generateAuthHeader = false;
5458  authRequiresAnotherRoundtrip = false;
5459  if (!sendErrorPageNotification()) {
5460  error(ERR_ACCESS_DENIED, reqUrl.host());
5461  }
5462  kDebug(7113) << "looks like the user canceled the authentication dialog";
5463  delete *auth;
5464  *auth = 0;
5465  }
5466  }
5467  username = authinfo.username;
5468  password = authinfo.password;
5469  }
5470  }
5471 
5472  if (generateAuthHeader) {
5473  (*auth)->generateResponse(username, password);
5474  (*auth)->setCachePasswordEnabled(authinfo.keepPassword);
5475 
5476  kDebug(7113) << "isError=" << (*auth)->isError()
5477  << "needCredentials=" << (*auth)->needCredentials()
5478  << "forceKeepAlive=" << (*auth)->forceKeepAlive()
5479  << "forceDisconnect=" << (*auth)->forceDisconnect();
5480 
5481  if ((*auth)->isError()) {
5482  authTokens.removeOne(bestOffer);
5483  if (!authTokens.isEmpty()) {
5484  goto try_next_auth_scheme;
5485  } else {
5486  error(ERR_UNSUPPORTED_ACTION, i18n("Authorization failed."));
5487  authRequiresAnotherRoundtrip = false;
5488  }
5489  //### return false; ?
5490  } else if ((*auth)->forceKeepAlive()) {
5491  //### think this through for proxied / not proxied
5492  m_request.isKeepAlive = true;
5493  } else if ((*auth)->forceDisconnect()) {
5494  //### think this through for proxied / not proxied
5495  m_request.isKeepAlive = false;
5496  httpCloseConnection();
5497  }
5498  }
5499  } else {
5500  authRequiresAnotherRoundtrip = false;
5501  if (!sendErrorPageNotification()) {
5502  error(ERR_UNSUPPORTED_ACTION, i18n("Unknown Authorization method."));
5503  }
5504  }
5505  }
5506 
5507  return authRequiresAnotherRoundtrip;
5508 }
5509 
5510 
5511 #include "http.moc"
This file is part of the KDE documentation.
Documentation copyright © 1996-2013 The KDE developers.
Generated on Sat Jun 1 2013 22:05:29 by doxygen 1.8.3.1 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.

KIOSlave

Skip menu "KIOSlave"
  • Main Page
  • Alphabetical List
  • Class List
  • Class Hierarchy
  • Class Members
  • File List
  • File Members
  • Related Pages

kdelibs-4.10.4 API Reference

Skip menu "kdelibs-4.10.4 API Reference"
  • DNSSD
  • Interfaces
  •   KHexEdit
  •   KMediaPlayer
  •   KSpeech
  •   KTextEditor
  • kconf_update
  • KDE3Support
  •   KUnitTest
  • KDECore
  • KDED
  • KDEsu
  • KDEUI
  • KDEWebKit
  • KDocTools
  • KFile
  • KHTML
  • KImgIO
  • KInit
  • kio
  • KIOSlave
  • KJS
  •   KJS-API
  •   WTF
  • kjsembed
  • KNewStuff
  • KParts
  • KPty
  • Kross
  • KUnitConversion
  • KUtils
  • Nepomuk
  • Plasma
  • Solid
  • Sonnet
  • ThreadWeaver
Report problems with this website to our bug tracking system.
Contact the specific authors with questions and comments about the page contents.

KDE® and the K Desktop Environment® logo are registered trademarks of KDE e.V. | Legal