http.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2025 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 11 Hypertext Transfer Protocol (HTTP) */
10 
11 /*
12  * Anonymizing patch by lutz@as-node.jena.thur.de
13  * have a look into http-anon.c to get more information.
14  */
15 
16 #include "squid.h"
17 #include "acl/FilledChecklist.h"
18 #include "base/AsyncJobCalls.h"
19 #include "base/DelayedAsyncCalls.h"
20 #include "base/Raw.h"
21 #include "base/TextException.h"
22 #include "base64.h"
23 #include "CachePeer.h"
24 #include "client_side.h"
25 #include "comm/Connection.h"
26 #include "comm/Read.h"
27 #include "comm/Write.h"
28 #include "error/Detail.h"
29 #include "errorpage.h"
30 #include "fd.h"
31 #include "fde.h"
32 #include "globals.h"
33 #include "HeaderMangling.h"
34 #include "http.h"
37 #include "http/StatusCode.h"
38 #include "http/Stream.h"
39 #include "HttpControlMsg.h"
40 #include "HttpHdrCc.h"
41 #include "HttpHdrContRange.h"
42 #include "HttpHdrSc.h"
43 #include "HttpHdrScTarget.h"
44 #include "HttpHeaderTools.h"
45 #include "HttpReply.h"
46 #include "HttpRequest.h"
48 #include "log/access_log.h"
49 #include "MemBuf.h"
50 #include "MemObject.h"
51 #include "neighbors.h"
52 #include "pconn.h"
54 #include "refresh.h"
55 #include "RefreshPattern.h"
56 #include "rfc1738.h"
57 #include "SquidConfig.h"
58 #include "SquidMath.h"
59 #include "StatCounters.h"
60 #include "Store.h"
61 #include "StrList.h"
62 #include "tools.h"
63 #include "util.h"
64 
65 #if USE_AUTH
66 #include "auth/UserRequest.h"
67 #endif
68 #if USE_DELAY_POOLS
69 #include "DelayPools.h"
70 #endif
71 
73 
74 static const char *const crlf = "\r\n";
75 
77 static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request,
78  HttpHeader * hdr_out, const int we_do_ranges, const Http::StateFlags &);
79 
81  AsyncJob("HttpStateData"),
82  Client(theFwdState)
83 {
84  debugs(11,5, "HttpStateData " << this << " created");
86 
87  if (fwd->serverConnection() != nullptr)
88  _peer = cbdataReference(fwd->serverConnection()->getPeer()); /* might be NULL */
89 
93 
94  if (_peer) {
95  /*
96  * This NEIGHBOR_PROXY_ONLY check probably shouldn't be here.
97  * We might end up getting the object from somewhere else if,
98  * for example, the request to this neighbor fails.
99  */
101  entry->releaseRequest(true);
102 
103 #if USE_DELAY_POOLS
105 #endif
106  }
107 
108  /*
109  * register the handler to free HTTP state data when the FD closes
110  */
114 }
115 
117 {
118  /*
119  * don't forget that ~Client() gets called automatically
120  */
121 
122  if (httpChunkDecoder)
123  delete httpChunkDecoder;
124 
126 
127  delete upgradeHeaderOut;
128 
129  debugs(11,5, "HttpStateData " << this << " destroyed; " << serverConnection);
130 }
131 
134 {
135  return serverConnection;
136 }
137 
138 void
140 {
141  debugs(11, 5, "httpStateFree: FD " << params.fd << ", httpState=" << params.data);
142  doneWithFwd = "httpStateConnClosed()"; // assume FwdState is monitoring too
143  mustStop("HttpStateData::httpStateConnClosed");
144 }
145 
146 void
148 {
149  debugs(11, 4, serverConnection << ": '" << entry->url() << "'");
150 
151  if (entry->store_status == STORE_PENDING) {
153  }
154 
155  closeServer();
156  mustStop("HttpStateData::httpTimeout");
157 }
158 
159 static StoreEntry *
161  assert(newEntry->mem_obj);
162  return newEntry->mem_obj->request ?
164  storeGetPublic(newEntry->mem_obj->storeId(), newEntry->mem_obj->method);
165 }
166 
169 static void
171 {
172  int remove = 0;
173  int forbidden = 0;
174 
175  // If the incoming response already goes into a public entry, then there is
176  // nothing to remove. This protects ready-for-collapsing entries as well.
177  if (!EBIT_TEST(e->flags, KEY_PRIVATE))
178  return;
179 
180  // If the new/incoming response cannot be stored, then it does not
181  // compete with the old stored response for the public key, and the
182  // old stored response should be left as is.
183  if (e->mem_obj->request && !e->mem_obj->request->flags.cachable)
184  return;
185 
186  switch (status) {
187 
188  case Http::scOkay:
189 
191 
193 
195 
196  case Http::scFound:
197 
198  case Http::scSeeOther:
199 
200  case Http::scGone:
201 
202  case Http::scNotFound:
203  remove = 1;
204 
205  break;
206 
207  case Http::scForbidden:
208 
210  forbidden = 1;
211 
212  break;
213 
214 #if WORK_IN_PROGRESS
215 
217  forbidden = 1;
218 
219  break;
220 
221 #endif
222 
223  default:
224  break;
225  }
226 
227  if (!remove && !forbidden)
228  return;
229 
231 
232  if (pe != nullptr) {
233  assert(e != pe);
234 #if USE_HTCP
236 #endif
237  pe->release(true);
238  }
239 
244  if (e->mem_obj->request)
246  else
248 
249  if (pe != nullptr) {
250  assert(e != pe);
251 #if USE_HTCP
253 #endif
254  pe->release(true);
255  }
256 }
257 
258 void
260 {
261  if (request->flags.accelerated && reply->surrogate_control) {
263 
264  if (sctusable) {
265  if (sctusable->hasNoStore() ||
267  && sctusable->noStoreRemote())) {
268  surrogateNoStore = true;
269  // Be conservative for now and make it non-shareable because
270  // there is no enough information here to make the decision.
271  entry->makePrivate(false);
272  }
273 
274  /* The HttpHeader logic cannot tell if the header it's parsing is a reply to an
275  * accelerated request or not...
276  * Still, this is an abstraction breach. - RC
277  */
278  if (sctusable->hasMaxAge()) {
279  if (sctusable->maxAge() < sctusable->maxStale())
280  reply->expires = reply->date + sctusable->maxAge();
281  else
282  reply->expires = reply->date + sctusable->maxStale();
283 
284  /* And update the timestamps */
285  entry->timestampsSet();
286  }
287 
288  /* We ignore cache-control directives as per the Surrogate specification */
289  ignoreCacheControl = true;
290 
291  delete sctusable;
292  }
293  }
294 }
295 
298 {
299  HttpReply const *rep = finalReply();
300  HttpHeader const *hdr = &rep->header;
301  const char *v;
302 #if USE_HTTP_VIOLATIONS
303 
304  const RefreshPattern *R = nullptr;
305 
306  /* This strange looking define first looks up the refresh pattern
307  * and then checks if the specified flag is set. The main purpose
308  * of this is to simplify the refresh pattern lookup and USE_HTTP_VIOLATIONS
309  * condition
310  */
311 #define REFRESH_OVERRIDE(flag) \
312  ((R = (R ? R : refreshLimits(entry->mem_obj->storeId()))) , \
313  (R && R->flags.flag))
314 #else
315 #define REFRESH_OVERRIDE(flag) 0
316 #endif
317 
319  return decision.make(ReuseDecision::doNotCacheButShare, "the entry has been released");
320 
321  // RFC 9111 section 4:
322  // "When more than one suitable response is stored,
323  // a cache MUST use the most recent one
324  // (as determined by the Date header field)."
325  // TODO: whether such responses could be shareable?
326  if (sawDateGoBack)
327  return decision.make(ReuseDecision::reuseNot, "the response has an older date header");
328 
329  // Check for Surrogate/1.0 protocol conditions
330  // NP: reverse-proxy traffic our parent server has instructed us never to cache
331  if (surrogateNoStore)
332  return decision.make(ReuseDecision::reuseNot, "Surrogate-Control:no-store");
333 
334  // RFC 2616: HTTP/1.1 Cache-Control conditions
335  if (!ignoreCacheControl) {
336  // XXX: check to see if the request headers alone were enough to prevent caching earlier
337  // (ie no-store request header) no need to check those all again here if so.
338  // for now we are not reliably doing that so we waste CPU re-checking request CC
339 
340  // RFC 2616 section 14.9.2 - MUST NOT cache any response with request CC:no-store
342  !REFRESH_OVERRIDE(ignore_no_store))
343  return decision.make(ReuseDecision::reuseNot,
344  "client request Cache-Control:no-store");
345 
346  // NP: request CC:no-cache only means cache READ is forbidden. STORE is permitted.
348  /* TODO: we are allowed to cache when no-cache= has parameters.
349  * Provided we strip away any of the listed headers unless they are revalidated
350  * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
351  * That is a bit tricky for squid right now so we avoid caching entirely.
352  */
353  return decision.make(ReuseDecision::reuseNot,
354  "server reply Cache-Control:no-cache has parameters");
355  }
356 
357  // NP: request CC:private is undefined. We ignore.
358  // NP: other request CC flags are limiters on HIT/MISS. We don't care about here.
359 
360  // RFC 2616 section 14.9.2 - MUST NOT cache any response with CC:no-store
361  if (rep->cache_control && rep->cache_control->hasNoStore() &&
362  !REFRESH_OVERRIDE(ignore_no_store))
363  return decision.make(ReuseDecision::reuseNot,
364  "server reply Cache-Control:no-store");
365 
366  // RFC 2616 section 14.9.1 - MUST NOT cache any response with CC:private in a shared cache like Squid.
367  // CC:private overrides CC:public when both are present in a response.
368  // TODO: add a shared/private cache configuration possibility.
369  if (rep->cache_control &&
370  rep->cache_control->hasPrivate() &&
371  !REFRESH_OVERRIDE(ignore_private)) {
372  /* TODO: we are allowed to cache when private= has parameters.
373  * Provided we strip away any of the listed headers unless they are revalidated
374  * successfully (ie, must revalidate AND these headers are prohibited on stale replies).
375  * That is a bit tricky for squid right now so we avoid caching entirely.
376  */
377  return decision.make(ReuseDecision::reuseNot,
378  "server reply Cache-Control:private");
379  }
380  }
381 
382  // RFC 2068, sec 14.9.4 - MUST NOT cache any response with Authentication UNLESS certain CC controls are present
383  // allow HTTP violations to IGNORE those controls (ie re-block caching Auth)
384  if (request && (request->flags.auth || request->flags.authSent)) {
385  if (!rep->cache_control)
386  return decision.make(ReuseDecision::reuseNot,
387  "authenticated and server reply missing Cache-Control");
388 
389  if (ignoreCacheControl)
390  return decision.make(ReuseDecision::reuseNot,
391  "authenticated and ignoring Cache-Control");
392 
393  bool mayStore = false;
394  // HTTPbis pt6 section 3.2: a response CC:public is present
395  if (rep->cache_control->hasPublic()) {
396  debugs(22, 3, "Authenticated but server reply Cache-Control:public");
397  mayStore = true;
398 
399  // HTTPbis pt6 section 3.2: a response CC:must-revalidate is present
400  } else if (rep->cache_control->hasMustRevalidate()) {
401  debugs(22, 3, "Authenticated but server reply Cache-Control:must-revalidate");
402  mayStore = true;
403 
404 #if USE_HTTP_VIOLATIONS
405  // NP: given the must-revalidate exception we should also be able to exempt no-cache.
406  // HTTPbis WG verdict on this is that it is omitted from the spec due to being 'unexpected' by
407  // some. The caching+revalidate is not exactly unsafe though with Squids interpretation of no-cache
408  // (without parameters) as equivalent to must-revalidate in the reply.
409  } else if (rep->cache_control->hasNoCacheWithoutParameters()) {
410  debugs(22, 3, "Authenticated but server reply Cache-Control:no-cache (equivalent to must-revalidate)");
411  mayStore = true;
412 #endif
413 
414  // HTTPbis pt6 section 3.2: a response CC:s-maxage is present
415  } else if (rep->cache_control->hasSMaxAge()) {
416  debugs(22, 3, "Authenticated but server reply Cache-Control:s-maxage");
417  mayStore = true;
418  }
419 
420  if (!mayStore)
421  return decision.make(ReuseDecision::reuseNot, "authenticated transaction");
422 
423  // NP: response CC:no-cache is equivalent to CC:must-revalidate,max-age=0. We MAY cache, and do so.
424  // NP: other request CC flags are limiters on HIT/MISS/REFRESH. We don't care about here.
425  }
426 
427  /* HACK: The "multipart/x-mixed-replace" content type is used for
428  * continuous push replies. These are generally dynamic and
429  * probably should not be cachable
430  */
431  if ((v = hdr->getStr(Http::HdrType::CONTENT_TYPE)))
432  if (!strncasecmp(v, "multipart/x-mixed-replace", 25))
433  return decision.make(ReuseDecision::reuseNot, "Content-Type:multipart/x-mixed-replace");
434 
435  // TODO: if possible, provide more specific message for each status code
436  static const char *shareableError = "shareable error status code";
437  static const char *nonShareableError = "non-shareable error status code";
439  const char *statusReason = nonShareableError;
440 
441  switch (rep->sline.status()) {
442 
443  /* There are several situations when a non-cacheable response may be
444  * still shareable (e.g., among collapsed clients). We assume that these
445  * are 3xx and 5xx responses, indicating server problems and some of
446  * 4xx responses, common for all clients with a given cache key (e.g.,
447  * 404 Not Found or 414 URI Too Long). On the other hand, we should not
448  * share non-cacheable client-specific errors, such as 400 Bad Request
449  * or 406 Not Acceptable.
450  */
451 
452  /* Responses that are cacheable */
453 
454  case Http::scOkay:
455 
457 
459 
462 
463  case Http::scGone:
464  /*
465  * Don't cache objects that need to be refreshed on next request,
466  * unless we know how to refresh it.
467  */
468 
469  if (refreshIsCachable(entry) || REFRESH_OVERRIDE(store_stale))
470  decision.make(ReuseDecision::cachePositively, "refresh check returned cacheable");
471  else
472  decision.make(ReuseDecision::doNotCacheButShare, "refresh check returned non-cacheable");
473  break;
474 
475  /* Responses that only are cacheable if the server says so */
476 
477  case Http::scFound:
479  if (rep->date <= 0)
480  decision.make(ReuseDecision::doNotCacheButShare, "Date is missing/invalid");
481  else if (rep->expires > rep->date)
482  decision.make(ReuseDecision::cachePositively, "Expires > Date");
483  else
484  decision.make(ReuseDecision::doNotCacheButShare, "Expires <= Date");
485  break;
486 
487  /* These responses can be negatively cached. Most can also be shared. */
488  case Http::scNoContent:
489  case Http::scUseProxy:
490  case Http::scForbidden:
491  case Http::scNotFound:
493  case Http::scUriTooLong:
496  case Http::scBadGateway:
500  statusAnswer = ReuseDecision::doNotCacheButShare;
501  statusReason = shareableError;
502  [[fallthrough]]; // to the actual decision making below
503 
504  case Http::scBadRequest: // no sharing; perhaps the server did not like something specific to this request
505 #if USE_HTTP_VIOLATIONS
506  if (Config.negativeTtl > 0)
507  decision.make(ReuseDecision::cacheNegatively, "Config.negativeTtl > 0");
508  else
509 #endif
510  decision.make(statusAnswer, statusReason);
511  break;
512 
513  /* these responses can never be cached, some
514  of them can be shared though */
515  case Http::scSeeOther:
516  case Http::scNotModified:
521  // TODO: use more specific reason for non-error status codes
522  decision.make(ReuseDecision::doNotCacheButShare, shareableError);
523  break;
524 
525  case Http::scPartialContent: /* Not yet supported. TODO: make shareable for suitable ranges */
527  case Http::scRequestTimeout: // TODO: is this shareable?
528  case Http::scConflict: // TODO: is this shareable?
534  case Http::scLocked: // TODO: is this shareable?
538  case Http::scInvalidHeader: /* Squid header parsing error */
540  decision.make(ReuseDecision::reuseNot, nonShareableError);
541  break;
542 
543  default:
544  /* RFC 2616 section 6.1.1: an unrecognized response MUST NOT be cached. */
545  decision.make(ReuseDecision::reuseNot, "unknown status code");
546  break;
547  }
548 
549  return decision.answer;
550 }
551 
553 static void
554 assembleVaryKey(String &vary, SBuf &vstr, const HttpRequest &request)
555 {
556  static const SBuf asterisk("*");
557  const char *pos = nullptr;
558  const char *item = nullptr;
559  int ilen = 0;
560 
561  while (strListGetItem(&vary, ',', &item, &ilen, &pos)) {
562  SBuf name(item, ilen);
563  if (name == asterisk) {
564  vstr = asterisk;
565  break;
566  }
567  name.toLower();
568  if (!vstr.isEmpty())
569  vstr.append(", ", 2);
570  vstr.append(name);
571  String hdr(request.header.getByName(name));
572  const char *value = hdr.termedBuf();
573  if (value) {
574  value = rfc1738_escape_part(value);
575  vstr.append("=\"", 2);
576  vstr.append(value);
577  vstr.append("\"", 1);
578  }
579 
580  hdr.clean();
581  }
582 }
583 
584 /*
585  * For Vary, store the relevant request headers as
586  * virtual headers in the reply
587  * Returns an empty SBuf if the variance cannot be stored
588  */
589 SBuf
590 httpMakeVaryMark(HttpRequest * request, HttpReply const * reply)
591 {
592  SBuf vstr;
593  String vary;
594 
595  vary = reply->header.getList(Http::HdrType::VARY);
596  assembleVaryKey(vary, vstr, *request);
597 
598 #if X_ACCELERATOR_VARY
599  vary.clean();
601  assembleVaryKey(vary, vstr, *request);
602 #endif
603 
604  debugs(11, 3, vstr);
605  return vstr;
606 }
607 
608 void
610 {
611  if (flags.keepalive)
612  if (flags.peering && !flags.tunneling)
614 
615  if (reply->keep_alive) {
616  if (flags.peering && !flags.tunneling)
618 
620  && reply->bodySize(request->method) == -1 && !flags.chunked) {
621  debugs(11, DBG_IMPORTANT, "keepaliveAccounting: Impossible keep-alive header from '" << entry->url() << "'" );
622  // debugs(11, 2, "GOT HTTP REPLY HDR:\n---------\n" << readBuf->content() << "\n----------" );
623  flags.keepalive_broken = true;
624  }
625  }
626 }
627 
628 void
630 {
631  if (reply->date > -1 && flags.toOrigin) {
632  int skew = abs((int)(reply->date - squid_curtime));
633 
634  if (skew > 86400)
635  debugs(11, 3, "" << request->url.host() << "'s clock is skewed by " << skew << " seconds!");
636  }
637 }
638 
645 void
647 {
650  debugs(11, 3, "processReplyHeader: key '" << entry->getMD5Text() << "'");
651 
653 
654  if (!inBuf.length())
655  return;
656 
657  /* Attempt to parse the first line; this will define where the protocol, status, reason-phrase and header begin */
658  {
659  if (hp == nullptr)
661 
662  bool parsedOk = hp->parse(inBuf);
663  // remember the actual received status-code before returning on errors,
664  // overwriting any previously stored value from earlier forwarding attempts
665  request->hier.peer_reply_status = hp->messageStatus(); // may still be scNone
666 
667  // sync the buffers after parsing.
668  inBuf = hp->remaining();
669 
670  if (hp->needsMoreData()) {
671  if (eof) { // no more data coming
672  assert(!parsedOk);
673  // fall through to handle this premature EOF as an error
674  } else {
675  debugs(33, 5, "Incomplete response, waiting for end of response headers");
676  return;
677  }
678  }
679 
680  if (!parsedOk) {
681  // unrecoverable parsing error
682  // TODO: Use Raw! XXX: inBuf no longer has the [beginning of the] malformed header.
683  debugs(11, 3, "Non-HTTP-compliant header:\n---------\n" << inBuf << "\n----------");
684  flags.headers_parsed = true;
685  HttpReply *newrep = new HttpReply;
686  // hp->needsMoreData() means hp->parseStatusCode is unusable, but, here,
687  // it also means that the reply header got truncated by a premature EOF
688  assert(!hp->needsMoreData() || eof);
689  const auto scode = hp->needsMoreData() ? Http::scInvalidHeader : hp->parseStatusCode;
690  newrep->sline.set(Http::ProtocolVersion(), scode);
691  setVirginReply(newrep);
692  return;
693  }
694  }
695 
696  /* We know the whole response is in parser now */
697  debugs(11, 2, "HTTP Server " << serverConnection);
698  debugs(11, 2, "HTTP Server RESPONSE:\n---------\n" <<
699  hp->messageProtocol() << " " << hp->messageStatus() << " " << hp->reasonPhrase() << "\n" <<
700  hp->mimeHeader() <<
701  "----------");
702 
703  // reset payload tracking to begin after message headers
705 
706  const auto newrep = HttpReply::Pointer::Make();
707  // XXX: RFC 7230 indicates we MAY ignore the reason phrase,
708  // and use an empty string on unknown status.
709  // We do that now to avoid performance regression from using SBuf::c_str()
710  newrep->sline.set(hp->messageProtocol(), hp->messageStatus() /* , hp->reasonPhrase() */);
711 
712  // parse headers
713  if (!newrep->parseHeader(*hp)) {
714  newrep->sline.set(hp->messageProtocol(), Http::scInvalidHeader);
715  debugs(11, 2, "error parsing response headers mime block");
716  }
717 
718  // done with Parser, now process using the HttpReply
719  hp = nullptr;
720 
722 
723  if (newrep->sline.version.protocol == AnyP::PROTO_HTTP && Http::Is1xx(newrep->sline.status())) {
724  handle1xx(newrep.getRaw());
725  return;
726  }
727 
728  flags.chunked = false;
729  if (newrep->sline.version.protocol == AnyP::PROTO_HTTP && newrep->header.chunked()) {
730  flags.chunked = true;
732  }
733 
736 
737  const auto vrep = setVirginReply(newrep.getRaw());
738  flags.headers_parsed = true;
739 
740  keepaliveAccounting(vrep);
741 
742  checkDateSkew(vrep);
743 
745 }
746 
748 void
750 {
751  if (fwd->al)
752  fwd->al->reply = reply;
753 
754  // one 1xx at a time: we must not be called while waiting for previous 1xx
756  flags.handling1xx = true;
757 
758  const auto statusCode = reply->sline.status();
759 
760  // drop1xx() needs to handle HTTP 101 (Switching Protocols) responses
761  // specially because they indicate that the server has stopped speaking HTTP
764 
765  if (statusCode == Http::scContinue && request->forcedBodyContinuation)
766  return drop1xx("we have sent it already");
767 
768  if (!request->canHandle1xx())
769  return drop1xx("the client does not support it");
770 
771 #if USE_HTTP_VIOLATIONS
772  // check whether the 1xx response forwarding is allowed by squid.conf
773  if (Config.accessList.reply) {
775  ch.updateAle(fwd->al);
776  ch.updateReply(reply);
777  ch.syncAle(originalRequest().getRaw(), nullptr);
778  if (!ch.fastCheck().allowed()) // TODO: support slow lookups?
779  return drop1xx("http_reply_access blocked it");
780  }
781 #endif // USE_HTTP_VIOLATIONS
782 
784  if (const auto reason = blockSwitchingProtocols(*reply))
785  return drop1xx(reason);
786  }
787 
788  debugs(11, 2, "forwarding 1xx to client");
789 
790  // the Sink will use this to call us back after writing 1xx to the client
791  typedef NullaryMemFunT<HttpStateData> CbDialer;
792  const AsyncCall::Pointer cb = JobCallback(11, 3, CbDialer, this,
796  // If the call is not fired, then the Sink is gone, and HttpStateData
797  // will terminate due to an aborted store entry or another similar error.
798  // If we get stuck, it is not handle1xx fault if we could get stuck
799  // for similar reasons without a 1xx response.
800 }
801 
804 void
805 HttpStateData::drop1xx(const char *reason)
806 {
808  debugs(11, 2, "bad 101 because " << reason);
809  const auto err = new ErrorState(ERR_INVALID_RESP, Http::scBadGateway, request.getRaw(), fwd->al);
810  fwd->fail(err);
811  closeServer();
812  mustStop("prohibited HTTP/101 response");
813  return;
814  }
815 
816  debugs(11, 2, "ignoring 1xx because " << reason);
817  proceedAfter1xx();
818 }
819 
822 const char *
824 {
825  if (!upgradeHeaderOut)
826  return "Squid offered no Upgrade at all, but server switched to a tunnel";
827 
828  // See RFC 7230 section 6.7 for the corresponding MUSTs
829 
830  if (!reply.header.has(Http::HdrType::UPGRADE))
831  return "server did not send an Upgrade header field";
832 
833  if (!reply.header.hasListMember(Http::HdrType::CONNECTION, "upgrade", ','))
834  return "server did not send 'Connection: upgrade'";
835 
836  const auto acceptedProtos = reply.header.getList(Http::HdrType::UPGRADE);
837  const char *pos = nullptr;
838  const char *accepted = nullptr;
839  int acceptedLen = 0;
840  while (strListGetItem(&acceptedProtos, ',', &accepted, &acceptedLen, &pos)) {
841  debugs(11, 5, "server accepted at least" << Raw(nullptr, accepted, acceptedLen));
842  return nullptr; // OK: let the client validate server's selection
843  }
844 
845  return "server sent an essentially empty Upgrade header field";
846 }
847 
849 void
851 {
853 
855  // pass server connection ownership to request->clientConnectionManager
858  AsyncCall::Pointer call = asyncCall(11, 3, "ConnStateData::noteTakeServerConnectionControl",
861  ScheduleCallHere(call);
864  closeHandler = nullptr;
865  serverConnection = nullptr;
866  doneWithFwd = "switched protocols";
868  return;
869  }
870 
871  debugs(11, 2, "continuing with " << payloadSeen << " bytes in buffer after 1xx");
873 }
874 
878 bool
880 {
881  if (!_peer)
882  return true;
883 
884  // we are talking "through" rather than "to" our _peer
885  if (flags.tunneling)
886  return true;
887 
888  /*If this peer does not support connection pinning (authenticated
889  connections) return false
890  */
891  if (!_peer->connection_auth)
892  return false;
893 
894  const auto &rep = entry->mem().freshestReply();
895 
896  /*The peer supports connection pinning and the http reply status
897  is not unauthorized, so the related connection can be pinned
898  */
899  if (rep.sline.status() != Http::scUnauthorized)
900  return true;
901 
902  /*The server respond with Http::scUnauthorized and the peer configured
903  with "connection-auth=on" we know that the peer supports pinned
904  connections
905  */
906  if (_peer->connection_auth == 1)
907  return true;
908 
909  /*At this point peer has configured with "connection-auth=auto"
910  parameter so we need some extra checks to decide if we are going
911  to allow pinned connections or not
912  */
913 
914  /*if the peer configured with originserver just allow connection
915  pinning (squid 2.6 behaviour)
916  */
918  return true;
919 
920  /*if the connections it is already pinned it is OK*/
921  if (request->flags.pinned)
922  return true;
923 
924  /*Allow pinned connections only if the Proxy-support header exists in
925  reply and has in its list the "Session-Based-Authentication"
926  which means that the peer supports connection pinning.
927  */
928  if (rep.header.hasListMember(Http::HdrType::PROXY_SUPPORT, "Session-Based-Authentication", ','))
929  return true;
930 
931  return false;
932 }
933 
934 // Called when we parsed (and possibly adapted) the headers but
935 // had not starting storing (a.k.a., sending) the body yet.
936 void
938 {
940 
941  HttpReply *rep = finalReply();
942  const Http::StatusCode statusCode = rep->sline.status();
943 
944  entry->timestampsSet();
945 
946  /* Check if object is cacheable or not based on reply code */
947  debugs(11, 3, "HTTP CODE: " << statusCode);
948 
949  if (StoreEntry *oldEntry = findPreviouslyCachedEntry(entry)) {
950  oldEntry->lock("HttpStateData::haveParsedReplyHeaders");
951  sawDateGoBack = rep->olderThan(oldEntry->hasFreshestReply());
952  oldEntry->unlock("HttpStateData::haveParsedReplyHeaders");
953  }
954 
957 
958  bool varyFailure = false;
959  if (rep->header.has(Http::HdrType::VARY)
960 #if X_ACCELERATOR_VARY
962 #endif
963  ) {
964  const SBuf vary(httpMakeVaryMark(request.getRaw(), rep));
965 
966  if (vary.isEmpty()) {
967  // TODO: check whether such responses are shareable.
968  // Do not share for now.
969  entry->makePrivate(false);
972  varyFailure = true;
973  } else {
974  entry->mem_obj->vary_headers = vary;
975 
976  // RFC 7231 section 7.1.4
977  // Vary:* can be cached, but has mandatory revalidation
978  static const SBuf asterisk("*");
979  if (vary == asterisk)
981  }
982  }
983 
984  if (!varyFailure) {
985  /*
986  * If its not a reply that we will re-forward, then
987  * allow the client to get it.
988  */
991 
992  ReuseDecision decision(entry, statusCode);
993 
994  switch (reusableReply(decision)) {
995 
997  entry->makePrivate(false);
998  break;
999 
1001  if (!entry->makePublic()) {
1002  decision.make(ReuseDecision::doNotCacheButShare, "public key creation error");
1003  entry->makePrivate(true);
1004  }
1005  break;
1006 
1008  if (!entry->cacheNegatively()) {
1009  decision.make(ReuseDecision::doNotCacheButShare, "public key creation error");
1010  entry->makePrivate(true);
1011  }
1012  break;
1013 
1015  entry->makePrivate(true);
1016  break;
1017 
1018  default:
1019  assert(0);
1020  break;
1021  }
1022  debugs(11, 3, "decided: " << decision);
1023  }
1024 
1025  if (!ignoreCacheControl) {
1026  if (rep->cache_control) {
1027  // We are required to revalidate on many conditions.
1028  // For security reasons we do so even if storage was caused by refresh_pattern ignore-* option
1029 
1030  // CC:must-revalidate or CC:proxy-revalidate
1031  const bool ccMustRevalidate = (rep->cache_control->hasProxyRevalidate() || rep->cache_control->hasMustRevalidate());
1032 
1033  // CC:no-cache (only if there are no parameters)
1034  const bool ccNoCacheNoParams = rep->cache_control->hasNoCacheWithoutParameters();
1035 
1036  // CC:s-maxage=N
1037  const bool ccSMaxAge = rep->cache_control->hasSMaxAge();
1038 
1039  // CC:private (yes, these can sometimes be stored)
1040  const bool ccPrivate = rep->cache_control->hasPrivate();
1041 
1042  if (ccNoCacheNoParams || ccPrivate)
1044  else if (ccMustRevalidate || ccSMaxAge)
1046  }
1047 #if USE_HTTP_VIOLATIONS // response header Pragma::no-cache is undefined in HTTP
1048  else {
1049  // Expensive calculation. So only do it IF the CC: header is not present.
1050 
1051  /* HACK: Pragma: no-cache in _replies_ is not documented in HTTP,
1052  * but servers like "Active Imaging Webcast/2.0" sure do use it */
1053  if (rep->header.has(Http::HdrType::PRAGMA) &&
1054  rep->header.hasListMember(Http::HdrType::PRAGMA,"no-cache",','))
1056  }
1057 #endif
1058  }
1059 }
1060 
1063 {
1064  const HttpReply *rep = virginReply();
1068  static SBuf close("close", 5);
1069  if (httpHeaderHasConnDir(&rep->header, close))
1071 
1076  if (!flags.keepalive)
1078 
1082  if (flags.forceClose)
1084 
1089  if (!flags.request_sent) {
1090  debugs(11, 2, "Request not yet fully sent " << request->method << ' ' << entry->url());
1092  }
1093 
1097  /* XXX: BUG?
1098  * If the origin server (HTTP/1.0) does not send a keep-alive
1099  * header, but keeps the connection open anyway, what happens?
1100  * We'll return here and wait for an EOF before changing
1101  * store_status to STORE_OK. Combine this with ENTRY_FWD_HDR_WAIT
1102  * and an error status code, and we might have to wait until
1103  * the server times out the socket.
1104  */
1105  if (!rep->keep_alive)
1107 
1108  return COMPLETE_PERSISTENT_MSG;
1109 }
1110 
1113 {
1114  debugs(11, 3, serverConnection << " eof=" << eof);
1115  if (eof) // already reached EOF
1117 
1118  /* If server fd is closing (but we have not been notified yet), stop Comm
1119  I/O to avoid assertions. TODO: Change Comm API to handle callers that
1120  want more I/O after async closing (usually initiated by others). */
1121  // XXX: add canReceive or s/canSend/canTalkToServer/
1124 
1130  if (lastChunk && flags.chunked)
1131  return statusIfComplete();
1132 
1133  const HttpReply *vrep = virginReply();
1134  debugs(11, 5, "persistentConnStatus: content_length=" << vrep->content_length);
1135 
1136  const int64_t clen = vrep->bodySize(request->method);
1137 
1138  debugs(11, 5, "persistentConnStatus: clen=" << clen);
1139 
1140  /* If the body size is unknown we must wait for EOF */
1141  if (clen < 0)
1142  return INCOMPLETE_MSG;
1143 
1146  if (clen > 0) {
1147  debugs(11,5, "payloadSeen=" << payloadSeen << " content_length=" << vrep->content_length);
1148 
1149  if (payloadSeen < vrep->content_length)
1150  return INCOMPLETE_MSG;
1151 
1152  if (payloadTruncated > 0) // already read more than needed
1153  return COMPLETE_NONPERSISTENT_MSG; // disable pconns
1154  }
1155 
1158  return statusIfComplete();
1159 }
1160 
1161 void
1163 {
1166 }
1167 
1168 void
1170 {
1171  debugs(11, 5, io.conn);
1172  waitingForCommRead = false;
1173 
1174  // Bail out early on Comm::ERR_CLOSING - close handlers will tidy up for us
1175  if (io.flag == Comm::ERR_CLOSING) {
1176  debugs(11, 3, "http socket closing");
1177  return;
1178  }
1179 
1181  abortTransaction("store entry aborted while reading reply");
1182  return;
1183  }
1184 
1186  Must(io.conn->fd == serverConnection->fd);
1187 
1188  /*
1189  * Don't reset the timeout value here. The value should be
1190  * counting Config.Timeout.request and applies to the request
1191  * as a whole, not individual read() calls.
1192  * Plus, it breaks our lame *HalfClosed() detection
1193  */
1194 
1195  const auto moreDataPermission = canBufferMoreReplyBytes();
1196  if (!moreDataPermission) {
1197  abortTransaction("ready to read required data, but the read buffer is full and cannot be drained");
1198  return;
1199  }
1200 
1201  const auto readSizeMax = maybeMakeSpaceAvailable(moreDataPermission.value());
1202  // TODO: Move this logic inside maybeMakeSpaceAvailable():
1203  const auto readSizeWanted = readSizeMax ? entry->bytesWanted(Range<size_t>(0, readSizeMax)) : 0;
1204 
1205  if (readSizeWanted <= 0) {
1206  // XXX: If we avoid Comm::ReadNow(), we should not Comm::Read() again
1207  // when the wait is over. We should go straight to readReply() instead.
1208 
1209 #if USE_ADAPTATION
1210  // XXX: We are duplicating Client::calcBufferSpaceToReserve() logic.
1211  // XXX: Some other delayRead() cases may lack kickReads() guarantees.
1212  // TODO: Refactor maybeMakeSpaceAvailable() to properly treat each
1213  // no-read case instead of calling delayRead() for the remaining cases.
1214 
1215  if (responseBodyBuffer) {
1216  debugs(11, 5, "avoid delayRead() to give adaptation a chance to drain overflow buffer: " << responseBodyBuffer->contentSize());
1217  return; // wait for Client::noteMoreBodySpaceAvailable()
1218  }
1219 
1221  debugs(11, 5, "avoid delayRead() to give adaptation a chance to drain body pipe buffer: " << virginBodyDestination->buf().contentSize());
1222  return; // wait for Client::noteMoreBodySpaceAvailable()
1223  }
1224 #endif
1225 
1226  delayRead();
1227  return;
1228  }
1229 
1230  CommIoCbParams rd(this); // will be expanded with ReadNow results
1231  rd.conn = io.conn;
1232  rd.size = readSizeWanted;
1233  switch (Comm::ReadNow(rd, inBuf)) {
1234  case Comm::INPROGRESS:
1235  if (inBuf.isEmpty())
1236  debugs(33, 2, io.conn << ": no data to process, " << xstrerr(rd.xerrno));
1238  return;
1239 
1240  case Comm::OK:
1241  {
1242  payloadSeen += rd.size;
1243 #if USE_DELAY_POOLS
1244  DelayId delayId = entry->mem_obj->mostBytesAllowed();
1245  delayId.bytesIn(rd.size);
1246 #endif
1247 
1248  statCounter.server.all.kbytes_in += rd.size;
1249  statCounter.server.http.kbytes_in += rd.size;
1250  ++ IOStats.Http.reads;
1251 
1252  int bin = 0;
1253  for (int clen = rd.size - 1; clen; ++bin)
1254  clen >>= 1;
1255 
1256  ++ IOStats.Http.read_hist[bin];
1257 
1259  }
1260 
1261  /* Continue to process previously read data */
1262  break;
1263 
1264  case Comm::ENDFILE: // close detected by 0-byte read
1265  eof = 1;
1266 
1267  /* Continue to process previously read data */
1268  break;
1269 
1270  // case Comm::COMM_ERROR:
1271  default: // no other flags should ever occur
1272  debugs(11, 2, io.conn << ": read failure: " << xstrerr(rd.xerrno));
1273  const auto err = new ErrorState(ERR_READ_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1274  err->xerrno = rd.xerrno;
1275  fwd->fail(err);
1276  closeServer();
1277  mustStop("HttpStateData::readReply");
1278  return;
1279  }
1280 
1281  /* Process next response from buffer */
1282  processReply();
1283 }
1284 
1287 void
1289 {
1290 
1291  if (flags.handling1xx) { // we came back after handling a 1xx response
1292  debugs(11, 5, "done with 1xx handling");
1293  flags.handling1xx = false;
1295  }
1296 
1298  abortTransaction("store entry aborted while we were waiting for processReply()");
1299  return;
1300  }
1301 
1302  if (!flags.headers_parsed) { // have not parsed headers yet?
1304 
1305  if (!continueAfterParsingHeader()) // parsing error or need more data
1306  return; // TODO: send errors to ICAP
1307 
1308  adaptOrFinalizeReply(); // may write to, abort, or "close" the entry
1309  }
1310 
1311  // kick more reads if needed and/or process the response body, if any
1312  processReplyBody(); // may call serverComplete()
1313 }
1314 
1318 bool
1320 {
1321  if (flags.handling1xx) {
1322  debugs(11, 5, "wait for 1xx handling");
1324  return false;
1325  }
1326 
1327  if (!flags.headers_parsed && !eof) {
1328  debugs(11, 9, "needs more at " << inBuf.length());
1332  maybeReadVirginBody(); // schedules all kinds of reads; TODO: rename
1333  return false;
1334  }
1335 
1339 
1340  if (flags.headers_parsed) { // parsed headers, possibly with errors
1341  // check for header parsing errors
1342  if (HttpReply *vrep = virginReply()) {
1343  const Http::StatusCode s = vrep->sline.status();
1344  const AnyP::ProtocolVersion &v = vrep->sline.version;
1345  if (s == Http::scInvalidHeader && v != Http::ProtocolVersion(0,9)) {
1346  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Bad header encountered from " << entry->url() << " AKA " << request->url);
1348  } else if (s == Http::scHeaderTooLarge) {
1349  fwd->dontRetry(true);
1350  error = ERR_TOO_BIG;
1351  } else if (vrep->header.conflictingContentLength()) {
1352  fwd->dontRetry(true);
1354  } else if (vrep->header.unsupportedTe()) {
1355  fwd->dontRetry(true);
1357  } else {
1358  return true; // done parsing, got reply, and no error
1359  }
1360  } else {
1361  // parsed headers but got no reply
1362  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: No reply at all for " << entry->url() << " AKA " << request->url);
1364  }
1365  } else {
1366  assert(eof);
1367  if (inBuf.length()) {
1369  debugs(11, DBG_IMPORTANT, "WARNING: HTTP: Invalid Response: Headers did not parse at all for " << entry->url() << " AKA " << request->url);
1370  } else {
1372  debugs(11, (request->flags.accelerated?DBG_IMPORTANT:2), "WARNING: HTTP: Invalid Response: No object data received for " << entry->url() << " AKA " << request->url);
1373  }
1374  }
1375 
1376  assert(error != ERR_NONE);
1377  entry->reset();
1379  closeServer();
1380  mustStop("HttpStateData::continueAfterParsingHeader");
1381  return false; // quit on error
1382 }
1383 
1386 void
1388 {
1390 
1391  HttpReply *vrep = virginReply();
1392  int64_t clen = -1;
1393  if (!vrep->expectingBody(request->method, clen) || clen < 0)
1394  return; // no body or a body of unknown size, including chunked
1395 
1396  if (payloadSeen - payloadTruncated <= clen)
1397  return; // we did not read too much or already took care of the extras
1398 
1399  if (const int64_t extras = payloadSeen - payloadTruncated - clen) {
1400  // server sent more that the advertised content length
1401  debugs(11, 5, "payloadSeen=" << payloadSeen <<
1402  " clen=" << clen << '/' << vrep->content_length <<
1403  " truncated=" << payloadTruncated << '+' << extras);
1404 
1405  inBuf.chop(0, inBuf.length() - extras);
1406  payloadTruncated += extras;
1407  }
1408 }
1409 
1411 void
1413 {
1414  const auto err = new ErrorState(ERR_READ_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1415  static const auto d = MakeNamedErrorDetail("SRV_PREMATURE_EOF");
1416  err->detailError(d);
1417  fwd->fail(err);
1418 }
1419 
1424 void
1426 {
1427  truncateVirginBody(); // if needed
1428  const char *data = inBuf.rawContent();
1429  int len = inBuf.length();
1430  addVirginReplyBody(data, len);
1431  inBuf.consume(len);
1432 
1433  // after addVirginReplyBody() wrote (when not adapting) everything we have
1434  // received to Store, check whether we have received/parsed the entire reply
1435  int64_t clen = -1;
1436  const char *parsedWhole = nullptr;
1437  if (!virginReply()->expectingBody(request->method, clen))
1438  parsedWhole = "http parsed header-only reply";
1439  else if (clen >= 0 && clen == payloadSeen - payloadTruncated)
1440  parsedWhole = "http parsed Content-Length body bytes";
1441  else if (clen < 0 && eof)
1442  parsedWhole = "http parsed body ending with expected/required EOF";
1443 
1444  if (parsedWhole)
1445  markParsedVirginReplyAsWhole(parsedWhole);
1446  else if (eof)
1448 }
1449 
1450 bool
1452 {
1453  assert(flags.chunked);
1455  try {
1456  MemBuf decodedData;
1457  decodedData.init();
1458  httpChunkDecoder->setPayloadBuffer(&decodedData);
1459  const bool doneParsing = httpChunkDecoder->parse(inBuf);
1460  inBuf = httpChunkDecoder->remaining(); // sync buffers after parse
1461  addVirginReplyBody(decodedData.content(), decodedData.contentSize());
1462  if (doneParsing) {
1463  lastChunk = 1;
1464  markParsedVirginReplyAsWhole("http parsed last-chunk");
1465  } else if (eof) {
1467  }
1468  return true;
1469  }
1470  catch (...) {
1471  debugs (11, 2, "de-chunking failure: " << CurrentException);
1472  }
1473  return false;
1474 }
1475 
1483 void
1485 {
1486  if (!flags.headers_parsed) {
1488  return;
1489  }
1490 
1491 #if USE_ADAPTATION
1492  debugs(11,5, "adaptationAccessCheckPending=" << adaptationAccessCheckPending);
1494  return;
1495 
1496 #endif
1497 
1498  /*
1499  * At this point the reply headers have been parsed and consumed.
1500  * That means header content has been removed from readBuf and
1501  * it contains only body data.
1502  */
1503  if (entry->isAccepting()) {
1504  if (flags.chunked) {
1505  if (!decodeAndWriteReplyBody()) {
1506  serverComplete();
1507  return;
1508  }
1509  } else
1510  writeReplyBody();
1511  }
1512 
1513  // storing/sending methods like earlier adaptOrFinalizeReply() or
1514  // above writeReplyBody() may release/abort the store entry.
1516  // TODO: In some cases (e.g., 304), we should keep persistent conn open.
1517  // Detect end-of-reply (and, hence, pool our idle pconn) earlier (ASAP).
1518  abortTransaction("store entry aborted while storing reply");
1519  return;
1520  } else
1521  switch (persistentConnStatus()) {
1522  case INCOMPLETE_MSG: {
1523  debugs(11, 5, "processReplyBody: INCOMPLETE_MSG from " << serverConnection);
1524  /* Wait for more data or EOF condition */
1525  AsyncCall::Pointer nil;
1526  if (flags.keepalive_broken) {
1528  } else {
1530  }
1531  }
1532  break;
1533 
1534  case COMPLETE_PERSISTENT_MSG: {
1535  debugs(11, 5, "processReplyBody: COMPLETE_PERSISTENT_MSG from " << serverConnection);
1536 
1537  // TODO: Remove serverConnectionSaved but preserve exception safety.
1538 
1540 
1542  closeHandler = nullptr;
1543 
1544  Ip::Address client_addr; // XXX: Remove as unused. Why was it added?
1546  client_addr = request->client_addr;
1547 
1548  auto serverConnectionSaved = serverConnection;
1550  serverConnection = nullptr;
1551 
1552  bool ispinned = false; // TODO: Rename to isOrShouldBePinned
1553  if (request->flags.pinned) {
1554  ispinned = true;
1555  } else if (request->flags.connectionAuth && request->flags.authSent) {
1556  ispinned = true;
1557  }
1558 
1559  if (ispinned) {
1560  if (request->clientConnectionManager.valid()) {
1562  ConnStateData,
1563  notePinnedConnectionBecameIdle,
1564  ConnStateData::PinnedIdleContext(serverConnectionSaved, request));
1565  } else {
1566  // must not pool/share ispinned connections, even orphaned ones
1567  serverConnectionSaved->close();
1568  }
1569  } else {
1570  fwdPconnPool->push(serverConnectionSaved, request->url.host());
1571  }
1572 
1573  serverComplete();
1574  return;
1575  }
1576 
1578  debugs(11, 5, "processReplyBody: COMPLETE_NONPERSISTENT_MSG from " << serverConnection);
1579 
1580  serverComplete();
1581  return;
1582  }
1583 
1585 }
1586 
1587 bool
1589 {
1590  // TODO: Be more precise here. For example, if/when reading trailer, we may
1591  // not be doneWithServer() yet, but we should return false. Similarly, we
1592  // could still be writing the request body after receiving the whole reply.
1593  return !doneWithServer();
1594 }
1595 
1596 void
1598 {
1600  debugs(11, 3, "no, peer connection gone");
1601  return;
1602  }
1603 
1604  if (eof) {
1605  // tolerate hypothetical calls between Comm::ENDFILE and closeServer()
1606  debugs(11, 3, "no, saw EOF");
1607  return;
1608  }
1609 
1610  if (lastChunk) {
1611  // tolerate hypothetical calls between setting lastChunk and clearing serverConnection
1612  debugs(11, 3, "no, saw last-chunk");
1613  return;
1614  }
1615 
1616  if (!canBufferMoreReplyBytes()) {
1617  abortTransaction("more response bytes required, but the read buffer is full and cannot be drained");
1618  return;
1619  }
1620 
1622  debugs(11, 5, "no, still waiting for noteDelayAwareReadChance()");
1623  return;
1624  }
1625 
1626  if (waitingForCommRead) {
1627  debugs(11, 3, "no, already waiting for readReply()");
1628  return;
1629  }
1630 
1632 
1633  // wait for read(2) to be possible.
1635  AsyncCall::Pointer call = JobCallback(11, 5, Dialer, this, HttpStateData::readReply);
1637  waitingForCommRead = true;
1638 }
1639 
1645 size_t
1647 {
1648  if (!flags.headers_parsed)
1649  return Config.maxReplyHeaderSize;
1650 
1651  // XXX: Our inBuf is not used to maintain the read-ahead gap, and using
1652  // Config.readAheadGap like this creates huge read buffers for large
1653  // read_ahead_gap values. TODO: Switch to using tcp_recv_bufsize as the
1654  // primary read buffer capacity factor.
1655  //
1656  // TODO: Cannot reuse throwing NaturalCast() here. Consider removing
1657  // .value() dereference in NaturalCast() or add/use NaturalCastOrMax().
1658  const auto configurationPreferences = NaturalSum<size_t>(Config.readAheadGap).value_or(SBuf::maxSize);
1659 
1660  // TODO: Honor TeChunkedParser look-ahead and trailer parsing requirements
1661  // (when explicit configurationPreferences are set too low).
1662 
1663  return std::min<size_t>(configurationPreferences, SBuf::maxSize);
1664 }
1665 
1671 std::optional<size_t>
1673 {
1674 #if USE_ADAPTATION
1675  // If we do not check this now, we may say the final "no" prematurely below
1676  // because inBuf.length() will decrease as adaptation drains buffered bytes.
1677  if (responseBodyBuffer) {
1678  debugs(11, 3, "yes, but waiting for adaptation to drain read buffer");
1679  return 0; // yes, we may be able to buffer more (but later)
1680  }
1681 #endif
1682 
1683  const auto maxCapacity = calcReadBufferCapacityLimit();
1684  if (inBuf.length() >= maxCapacity) {
1685  debugs(11, 3, "no, due to a full buffer: " << inBuf.length() << '/' << inBuf.spaceSize() << "; limit: " << maxCapacity);
1686  return std::nullopt; // no, configuration prohibits buffering more
1687  }
1688 
1689  const auto maxReadSize = maxCapacity - inBuf.length(); // positive
1690  debugs(11, 7, "yes, may read up to " << maxReadSize << " into " << inBuf.length() << '/' << inBuf.spaceSize());
1691  return maxReadSize; // yes, can read up to this many bytes (subject to other constraints)
1692 }
1693 
1697 size_t
1699 {
1700  // how much we want to read
1701  const size_t read_size = calcBufferSpaceToReserve(inBuf.spaceSize(), maxReadSize);
1702 
1703  if (!read_size) {
1704  debugs(11, 7, "will not read up to " << read_size << " into buffer (" << inBuf.length() << "/" << inBuf.spaceSize() << ") from " << serverConnection);
1705  return 0;
1706  }
1707 
1708  // we may need to grow the buffer
1709  inBuf.reserveSpace(read_size);
1710  debugs(11, 7, "may read up to " << read_size << " bytes info buffer (" << inBuf.length() << "/" << inBuf.spaceSize() << ") from " << serverConnection);
1711  return read_size;
1712 }
1713 
1715 void
1717 {
1718  debugs(11, 5, serverConnection << ": size " << io.size << ": errflag " << io.flag << ".");
1719 #if URL_CHECKSUM_DEBUG
1720 
1721  entry->mem_obj->checkUrlChecksum();
1722 #endif
1723 
1724  // XXX: Keep in sync with Client::sentRequestBody().
1725  // TODO: Extract common parts.
1726 
1727  if (io.size > 0) {
1729  statCounter.server.all.kbytes_out += io.size;
1730  statCounter.server.http.kbytes_out += io.size;
1731  }
1732 
1733  if (io.flag == Comm::ERR_CLOSING)
1734  return;
1735 
1736  // both successful and failed writes affect response times
1738 
1739  if (io.flag) {
1740  const auto err = new ErrorState(ERR_WRITE_ERROR, Http::scBadGateway, fwd->request, fwd->al);
1741  err->xerrno = io.xerrno;
1742  fwd->fail(err);
1743  closeServer();
1744  mustStop("HttpStateData::wroteLast");
1745  return;
1746  }
1747 
1748  sendComplete();
1749 }
1750 
1752 void
1754 {
1755  /*
1756  * Set the read timeout here because it hasn't been set yet.
1757  * We only set the read timeout after the request has been
1758  * fully written to the peer. If we start the timeout
1759  * after connection establishment, then we are likely to hit
1760  * the timeout for POST/PUT requests that have very large
1761  * request bodies.
1762  */
1764  AsyncCall::Pointer timeoutCall = JobCallback(11, 5,
1765  TimeoutDialer, this, HttpStateData::httpTimeout);
1766 
1768  flags.request_sent = true;
1769 }
1770 
1771 void
1773 {
1774  debugs(11,5, "closing HTTP server " << serverConnection << " this " << this);
1775 
1779  closeHandler = nullptr;
1781  }
1782 }
1783 
1784 bool
1786 {
1788 }
1789 
1790 /*
1791  * Fixup authentication request headers for special cases
1792  */
1793 static void
1794 httpFixupAuthentication(HttpRequest * request, const HttpHeader * hdr_in, HttpHeader * hdr_out, const CachePeer * const peer, const Http::StateFlags &flags)
1795 {
1796  /* Nothing to do unless we are forwarding to a peer */
1797  if (!flags.peering)
1798  return;
1799 
1800  // do nothing if our cache_peer was reconfigured away
1801  if (!peer)
1802  return;
1803 
1804  // This request is going "through" rather than "to" our _peer.
1805  if (flags.tunneling)
1806  return;
1807 
1808  /* Needs to be explicitly enabled */
1809  if (!request->peer_login)
1810  return;
1811 
1813  /* Maybe already dealt with? */
1814  if (hdr_out->has(header))
1815  return;
1816 
1817  /* Nothing to do here for PASSTHRU */
1818  if (strcmp(request->peer_login, "PASSTHRU") == 0)
1819  return;
1820 
1821  // Dangerous and undocumented PROXYPASS is a single-signon to servers with
1822  // the proxy password. Only Basic Authentication can work this way. This
1823  // statement forwards a "basic" Proxy-Authorization value from our client
1824  // to an originserver peer. Other PROXYPASS cases are handled lower.
1825  if (flags.toOrigin &&
1826  strcmp(request->peer_login, "PROXYPASS") == 0 &&
1828 
1829  const char *auth = hdr_in->getStr(Http::HdrType::PROXY_AUTHORIZATION);
1830 
1831  if (auth && strncasecmp(auth, "basic ", 6) == 0) {
1832  hdr_out->putStr(header, auth);
1833  return;
1834  }
1835  }
1836 
1837  char loginbuf[base64_encode_len(MAX_LOGIN_SZ)];
1838  size_t blen;
1839  struct base64_encode_ctx ctx;
1840  base64_encode_init(&ctx);
1841 
1842  /* Special mode to pass the username to the upstream cache */
1843  if (*request->peer_login == '*') {
1844  const char *username = "-";
1845 
1846  if (request->extacl_user.size())
1847  username = request->extacl_user.termedBuf();
1848 #if USE_AUTH
1849  else if (request->auth_user_request != nullptr)
1850  username = request->auth_user_request->username();
1851 #endif
1852 
1853  blen = base64_encode_update(&ctx, loginbuf, strlen(username), reinterpret_cast<const uint8_t*>(username));
1854  blen += base64_encode_update(&ctx, loginbuf+blen, strlen(request->peer_login +1), reinterpret_cast<const uint8_t*>(request->peer_login +1));
1855  blen += base64_encode_final(&ctx, loginbuf+blen);
1856  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1857  return;
1858  }
1859 
1860  /* external_acl provided credentials */
1861  if (request->extacl_user.size() && request->extacl_passwd.size() &&
1862  (strcmp(request->peer_login, "PASS") == 0 ||
1863  strcmp(request->peer_login, "PROXYPASS") == 0)) {
1864 
1865  blen = base64_encode_update(&ctx, loginbuf, request->extacl_user.size(), reinterpret_cast<const uint8_t*>(request->extacl_user.rawBuf()));
1866  blen += base64_encode_update(&ctx, loginbuf+blen, 1, reinterpret_cast<const uint8_t*>(":"));
1867  blen += base64_encode_update(&ctx, loginbuf+blen, request->extacl_passwd.size(), reinterpret_cast<const uint8_t*>(request->extacl_passwd.rawBuf()));
1868  blen += base64_encode_final(&ctx, loginbuf+blen);
1869  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1870  return;
1871  }
1872  // if no external user credentials are available to fake authentication with PASS acts like PASSTHRU
1873  if (strcmp(request->peer_login, "PASS") == 0)
1874  return;
1875 
1876  /* Kerberos login to peer */
1877 #if HAVE_AUTH_MODULE_NEGOTIATE && HAVE_KRB5 && HAVE_GSSAPI
1878  if (strncmp(request->peer_login, "NEGOTIATE",strlen("NEGOTIATE")) == 0) {
1879  char *Token=nullptr;
1880  char *PrincipalName=nullptr,*p;
1881  int negotiate_flags = 0;
1882 
1883  if ((p=strchr(request->peer_login,':')) != nullptr ) {
1884  PrincipalName=++p;
1885  }
1886  if (request->flags.auth_no_keytab) {
1887  negotiate_flags |= PEER_PROXY_NEGOTIATE_NOKEYTAB;
1888  }
1889  Token = peer_proxy_negotiate_auth(PrincipalName, peer->host, negotiate_flags);
1890  if (Token) {
1891  httpHeaderPutStrf(hdr_out, header, "Negotiate %s",Token);
1892  }
1893  return;
1894  }
1895 #endif /* HAVE_KRB5 && HAVE_GSSAPI */
1896 
1897  blen = base64_encode_update(&ctx, loginbuf, strlen(request->peer_login), reinterpret_cast<const uint8_t*>(request->peer_login));
1898  blen += base64_encode_final(&ctx, loginbuf+blen);
1899  httpHeaderPutStrf(hdr_out, header, "Basic %.*s", (int)blen, loginbuf);
1900  return;
1901 }
1902 
1903 /*
1904  * build request headers and append them to a given MemBuf
1905  * used by buildRequestPrefix()
1906  * note: initialised the HttpHeader, the caller is responsible for Clean()-ing
1907  */
1908 void
1910  StoreEntry * entry,
1911  const AccessLogEntryPointer &al,
1912  HttpHeader * hdr_out,
1913  const CachePeer * const peer,
1914  const Http::StateFlags &flags)
1915 {
1916  /* building buffer for complex strings */
1917 #define BBUF_SZ (MAX_URL+32)
1918  LOCAL_ARRAY(char, bbuf, BBUF_SZ);
1919  LOCAL_ARRAY(char, ntoabuf, MAX_IPSTRLEN);
1920  const HttpHeader *hdr_in = &request->header;
1921  const HttpHeaderEntry *e = nullptr;
1923  assert (hdr_out->owner == hoRequest);
1924 
1925  /* use our IMS header if the cached entry has Last-Modified time */
1926  if (request->lastmod > -1)
1928 
1929  // Add our own If-None-Match field if the cached entry has a strong ETag.
1930  // copyOneHeaderFromClientsideRequestToUpstreamRequest() adds client ones.
1931  if (request->etag.size() > 0) {
1933  request->etag.termedBuf()));
1934  }
1935 
1936  bool we_do_ranges = decideIfWeDoRanges (request);
1937 
1938  String strConnection (hdr_in->getList(Http::HdrType::CONNECTION));
1939 
1940  while ((e = hdr_in->getEntry(&pos)))
1941  copyOneHeaderFromClientsideRequestToUpstreamRequest(e, strConnection, request, hdr_out, we_do_ranges, flags);
1942 
1943  /* Abstraction break: We should interpret multipart/byterange responses
1944  * into offset-length data, and this works around our inability to do so.
1945  */
1946  if (!we_do_ranges && request->multipartRangeRequest()) {
1947  /* don't cache the result */
1949  /* pretend it's not a range request */
1950  request->ignoreRange("want to request the whole object");
1951  request->flags.isRanged = false;
1952  }
1953 
1954  hdr_out->addVia(request->http_ver, hdr_in);
1955 
1956  if (request->flags.accelerated) {
1957  /* Append Surrogate-Capabilities */
1958  String strSurrogate(hdr_in->getList(Http::HdrType::SURROGATE_CAPABILITY));
1959  snprintf(bbuf, BBUF_SZ, "%s=\"Surrogate/1.0\"", Config.Accel.surrogate_id);
1960  strListAdd(&strSurrogate, bbuf, ',');
1962  hdr_out->putStr(Http::HdrType::SURROGATE_CAPABILITY, strSurrogate.termedBuf());
1963  }
1964 
1966  if (strcmp(opt_forwarded_for, "delete") != 0) {
1967 
1968  String strFwd = hdr_in->getList(Http::HdrType::X_FORWARDED_FOR);
1969 
1970  // Detect unreasonably long header values. And paranoidly check String
1971  // limits: a String ought to accommodate two reasonable-length values.
1972  if (strFwd.size() > 32*1024 || !strFwd.canGrowBy(strFwd.size())) {
1973  // There is probably a forwarding loop with Via detection disabled.
1974  // If we do nothing, String will assert on overflow soon.
1975  // TODO: Terminate all transactions with huge XFF?
1976  strFwd = "error";
1977 
1978  static int warnedCount = 0;
1979  if (warnedCount++ < 100) {
1980  const SBuf url(entry ? SBuf(entry->url()) : request->effectiveRequestUri());
1981  debugs(11, DBG_IMPORTANT, "WARNING: likely forwarding loop with " << url);
1982  }
1983  }
1984 
1985  if (strcmp(opt_forwarded_for, "on") == 0) {
1987  if ( request->client_addr.isNoAddr() )
1988  strListAdd(&strFwd, "unknown", ',');
1989  else
1990  strListAdd(&strFwd, request->client_addr.toStr(ntoabuf, MAX_IPSTRLEN), ',');
1991  } else if (strcmp(opt_forwarded_for, "off") == 0) {
1993  strListAdd(&strFwd, "unknown", ',');
1994  } else if (strcmp(opt_forwarded_for, "transparent") == 0) {
1996  } else if (strcmp(opt_forwarded_for, "truncate") == 0) {
1998  if ( request->client_addr.isNoAddr() )
1999  strFwd = "unknown";
2000  else
2001  strFwd = request->client_addr.toStr(ntoabuf, MAX_IPSTRLEN);
2002  }
2003  if (strFwd.size() > 0)
2004  hdr_out->putStr(Http::HdrType::X_FORWARDED_FOR, strFwd.termedBuf());
2005  }
2008  /* append Host if not there already */
2009  if (!hdr_out->has(Http::HdrType::HOST)) {
2010  if (request->peer_domain) {
2012  } else {
2013  SBuf authority = request->url.authority();
2014  hdr_out->putStr(Http::HdrType::HOST, authority.c_str());
2015  }
2016  }
2017 
2018  /* append Authorization if known in URL, not in header and going direct */
2019  if (!hdr_out->has(Http::HdrType::AUTHORIZATION)) {
2020  if (flags.toOrigin && !request->url.userInfo().isEmpty()) {
2021  static char result[base64_encode_len(MAX_URL*2)]; // should be big enough for a single URI segment
2022  struct base64_encode_ctx ctx;
2023  base64_encode_init(&ctx);
2024  size_t blen = base64_encode_update(&ctx, result, request->url.userInfo().length(), reinterpret_cast<const uint8_t*>(request->url.userInfo().rawContent()));
2025  blen += base64_encode_final(&ctx, result+blen);
2026  result[blen] = '\0';
2027  if (blen)
2028  httpHeaderPutStrf(hdr_out, Http::HdrType::AUTHORIZATION, "Basic %.*s", (int)blen, result);
2029  }
2030  }
2031 
2032  /* Fixup (Proxy-)Authorization special cases. Plain relaying dealt with above */
2033  httpFixupAuthentication(request, hdr_in, hdr_out, peer, flags);
2034 
2035  /* append Cache-Control, add max-age if not there already */
2036  {
2037  HttpHdrCc *cc = hdr_in->getCc();
2038 
2039  if (!cc)
2040  cc = new HttpHdrCc();
2041 
2042  /* Add max-age only without no-cache */
2043  if (!cc->hasMaxAge() && !cc->hasNoCache()) {
2044  // XXX: performance regression. c_str() reallocates
2046  cc->maxAge(getMaxAge(entry ? entry->url() : tmp.c_str()));
2047  }
2048 
2049  /* Enforce sibling relations */
2050  if (flags.only_if_cached)
2051  cc->onlyIfCached(true);
2052 
2053  hdr_out->putCc(*cc);
2054 
2055  delete cc;
2056  }
2057 
2058  // Always send Connection because HTTP/1.0 servers need explicit
2059  // "keep-alive", HTTP/1.1 servers need explicit "close", Upgrade recipients
2060  // need bare "upgrade", and we do not always know the server expectations.
2061  if (!hdr_out->has(Http::HdrType::CONNECTION)) // forwardUpgrade() may add it
2062  hdr_out->putStr(Http::HdrType::CONNECTION, flags.keepalive ? "keep-alive" : "close");
2063 
2064  /* append Front-End-Https */
2065  if (flags.front_end_https) {
2067  hdr_out->putStr(Http::HdrType::FRONT_END_HTTPS, "On");
2068  }
2069 
2070  if (flags.chunked_request) {
2071  // Do not just copy the original value so that if the client-side
2072  // starts decode other encodings, this code may remain valid.
2073  hdr_out->putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
2074  }
2075 
2076  /* Now mangle the headers. */
2077  httpHdrMangleList(hdr_out, request, al, ROR_REQUEST);
2078 
2079  strConnection.clean();
2080 }
2081 
2084 void
2086 {
2088  return; // forward nothing by default
2089 
2090  /* RFC 7230 section 6.7 paragraph 10:
2091  * A server MUST ignore an Upgrade header field that is received in
2092  * an HTTP/1.0 request.
2093  */
2094  if (request->http_ver == Http::ProtocolVersion(1,0))
2095  return;
2096 
2097  const auto &hdrIn = request->header;
2098  if (!hdrIn.has(Http::HdrType::UPGRADE))
2099  return;
2100  const auto upgradeIn = hdrIn.getList(Http::HdrType::UPGRADE);
2101 
2102  String upgradeOut;
2103 
2104  ACLFilledChecklist ch(nullptr, request.getRaw());
2105  ch.al = fwd->al;
2106  const char *pos = nullptr;
2107  const char *offeredStr = nullptr;
2108  int offeredStrLen = 0;
2109  while (strListGetItem(&upgradeIn, ',', &offeredStr, &offeredStrLen, &pos)) {
2110  const ProtocolView offeredProto(offeredStr, offeredStrLen);
2111  debugs(11, 5, "checks all rules applicable to " << offeredProto);
2112  Config.http_upgrade_request_protocols->forApplicable(offeredProto, [&ch, offeredStr, offeredStrLen, &upgradeOut] (const SBuf &cfgProto, const acl_access *guard) {
2113  debugs(11, 5, "checks " << cfgProto << " rule(s)");
2114  ch.changeAcl(guard);
2115  const auto &answer = ch.fastCheck();
2116  if (answer.implicit)
2117  return false; // keep looking for an explicit rule match
2118  if (answer.allowed())
2119  strListAdd(upgradeOut, offeredStr, offeredStrLen);
2120  // else drop the offer (explicitly denied cases and ACL errors)
2121  return true; // stop after an explicit rule match or an error
2122  });
2123  }
2124 
2125  if (upgradeOut.size()) {
2126  hdrOut.putStr(Http::HdrType::UPGRADE, upgradeOut.termedBuf());
2127 
2128  /* RFC 7230 section 6.7 paragraph 10:
2129  * When Upgrade is sent, the sender MUST also send a Connection header
2130  * field that contains an "upgrade" connection option, in
2131  * order to prevent Upgrade from being accidentally forwarded by
2132  * intermediaries that might not implement the listed protocols.
2133  *
2134  * NP: Squid does not truly implement the protocol(s) in this Upgrade.
2135  * For now we are treating an explicit blind tunnel as "implemented"
2136  * regardless of the security implications.
2137  */
2138  hdrOut.putStr(Http::HdrType::CONNECTION, "upgrade");
2139 
2140  // Connection:close and Connection:keepalive confuse some Upgrade
2141  // recipients, so we do not send those headers. Our Upgrade request
2142  // implicitly offers connection persistency per HTTP/1.1 defaults.
2143  // Update the keepalive flag to reflect that offer.
2144  // * If the server upgrades, then we would not be talking HTTP past the
2145  // HTTP 101 control message, and HTTP persistence would be irrelevant.
2146  // * Otherwise, our request will contradict onoff.server_pconns=off or
2147  // other no-keepalive conditions (if any). We compensate by copying
2148  // the original no-keepalive decision now and honoring it later.
2150  flags.keepalive = true; // should already be true in most cases
2151  }
2152 }
2153 
2158 void
2159 copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest * request, HttpHeader * hdr_out, const int we_do_ranges, const Http::StateFlags &flags)
2160 {
2161  debugs(11, 5, "httpBuildRequestHeader: " << e->name << ": " << e->value );
2162 
2163  switch (e->id) {
2164 
2172  if (!flags.toOrigin && request->peer_login &&
2173  (strcmp(request->peer_login, "PASS") == 0 ||
2174  strcmp(request->peer_login, "PROXYPASS") == 0 ||
2175  strcmp(request->peer_login, "PASSTHRU") == 0)) {
2176  hdr_out->addEntry(e->clone());
2177  }
2178  break;
2179 
2183  case Http::HdrType::TE:
2186  case Http::HdrType::TRAILER:
2188  break;
2189 
2192  break;
2193 
2200  if (!flags.toOriginPeer()) {
2201  hdr_out->addEntry(e->clone());
2202  } else {
2207  if (request->peer_login &&
2208  (strcmp(request->peer_login, "PASS") == 0 ||
2209  strcmp(request->peer_login, "PASSTHRU") == 0 ||
2210  strcmp(request->peer_login, "PROXYPASS") == 0)) {
2211  hdr_out->addEntry(e->clone());
2212  }
2213  }
2214 
2215  break;
2216 
2217  case Http::HdrType::HOST:
2224  if (request->peer_domain)
2225  hdr_out->putStr(Http::HdrType::HOST, request->peer_domain);
2226  else if (request->flags.redirected && !Config.onoff.redir_rewrites_host)
2227  hdr_out->addEntry(e->clone());
2228  else {
2229  SBuf authority = request->url.authority();
2230  hdr_out->putStr(Http::HdrType::HOST, authority.c_str());
2231  }
2232 
2233  break;
2234 
2243  // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2244  if (hdr_out->has(Http::HdrType::IF_MODIFIED_SINCE))
2245  break;
2246  else if (Config.onoff.cache_miss_revalidate || !request->flags.cachable || request->flags.auth)
2247  hdr_out->addEntry(e->clone());
2248  break;
2249 
2258  // XXX: need to check and cleanup the auth case so cacheable auth requests get cached.
2259  if (hdr_out->hasListMember(Http::HdrType::IF_MATCH, "*", ',') || Config.onoff.cache_miss_revalidate || !request->flags.cachable || request->flags.auth)
2260  hdr_out->addEntry(e->clone());
2261  break;
2262 
2266  if (request->method == Http::METHOD_TRACE || request->method == Http::METHOD_OPTIONS) {
2267  const int64_t hops = e->getInt64();
2268 
2269  if (hops > 0)
2270  hdr_out->putInt64(Http::HdrType::MAX_FORWARDS, hops - 1);
2271  }
2272 
2273  break;
2274 
2275  case Http::HdrType::VIA:
2280  if (!Config.onoff.via)
2281  hdr_out->addEntry(e->clone());
2282 
2283  break;
2284 
2285  case Http::HdrType::RANGE:
2286 
2288 
2292  if (!we_do_ranges)
2293  hdr_out->addEntry(e->clone());
2294 
2295  break;
2296 
2297  case Http::HdrType::PROXY_CONNECTION: // SHOULD ignore. But doing so breaks things.
2298  break;
2299 
2301  // pass through unless we chunk; also, keeping this away from default
2302  // prevents request smuggling via Connection: Content-Length tricks
2303  if (!flags.chunked_request)
2304  hdr_out->addEntry(e->clone());
2305  break;
2306 
2308 
2313  break;
2314 
2318  if (!flags.front_end_https)
2319  hdr_out->addEntry(e->clone());
2320 
2321  break;
2322 
2323  default:
2327  if (strConnection.size()>0 && strListIsMember(&strConnection, e->name, ',')) {
2328  debugs(11, 2, "'" << e->name << "' header cropped by Connection: definition");
2329  return;
2330  }
2331 
2332  hdr_out->addEntry(e->clone());
2333  }
2334 }
2335 
2336 bool
2338 {
2339  bool result = true;
2340  /* decide if we want to do Ranges ourselves
2341  * and fetch the whole object now)
2342  * We want to handle Ranges ourselves iff
2343  * - we can actually parse client Range specs
2344  * - the specs are expected to be simple enough (e.g. no out-of-order ranges)
2345  * - reply will be cachable
2346  * (If the reply will be uncachable we have to throw it away after
2347  * serving this request, so it is better to forward ranges to
2348  * the server and fetch only the requested content)
2349  */
2350 
2351  int64_t roffLimit = request->getRangeOffsetLimit();
2352 
2353  if (nullptr == request->range || !request->flags.cachable
2355  result = false;
2356 
2357  debugs(11, 8, "decideIfWeDoRanges: range specs: " <<
2358  request->range << ", cachable: " <<
2359  request->flags.cachable << "; we_do_ranges: " << result);
2360 
2361  return result;
2362 }
2363 
2364 /* build request prefix and append it to a given MemBuf;
2365  * return the length of the prefix */
2366 mb_size_t
2368 {
2369  const int offset = mb->size;
2370  /* Uses a local httpver variable to print the HTTP label
2371  * since the HttpRequest may have an older version label.
2372  * XXX: This could create protocol bugs as the headers sent and
2373  * flow control should all be based on the HttpRequest version
2374  * not the one we are sending. Needs checking.
2375  */
2376  const AnyP::ProtocolVersion httpver = Http::ProtocolVersion();
2378  mb->appendf(SQUIDSBUFPH " " SQUIDSBUFPH " %s/%d.%d\r\n",
2380  SQUIDSBUFPRINT(url),
2382  httpver.major,httpver.minor);
2383  /* build and pack headers */
2384  {
2385  HttpHeader hdr(hoRequest);
2386  forwardUpgrade(hdr); // before httpBuildRequestHeader() for CONNECTION
2387  const auto peer = cbdataReferenceValid(_peer) ? _peer : nullptr;
2388  httpBuildRequestHeader(request.getRaw(), entry, fwd->al, &hdr, peer, flags);
2389 
2391  request->flags.authSent = true;
2392  else if (hdr.has(Http::HdrType::AUTHORIZATION))
2393  request->flags.authSent = true;
2394 
2395  // The late placement of this check supports reply_header_add mangling,
2396  // but also complicates optimizing upgradeHeaderOut-like lookups.
2397  if (hdr.has(Http::HdrType::UPGRADE)) {
2400  }
2401 
2402  hdr.packInto(mb);
2403  hdr.clean();
2404  }
2405  /* append header terminator */
2406  mb->append(crlf, 2);
2407  return mb->size - offset;
2408 }
2409 
2410 /* This will be called when connect completes. Write request. */
2411 bool
2413 {
2414  MemBuf mb;
2415 
2416  debugs(11, 5, serverConnection << ", request " << request << ", this " << this << ".");
2417 
2419  debugs(11,3, "cannot send request to closing " << serverConnection);
2420  assert(closeHandler != nullptr);
2421  return false;
2422  }
2423 
2425  AsyncCall::Pointer timeoutCall = JobCallback(11, 5,
2426  TimeoutDialer, this, HttpStateData::httpTimeout);
2429 
2430  if (request->body_pipe != nullptr) {
2431  if (!startRequestBodyFlow()) // register to receive body data
2432  return false;
2434  requestSender = JobCallback(11,5,
2435  Dialer, this, HttpStateData::sentRequestBody);
2436 
2438  // use chunked encoding if we do not know the length
2439  if (request->content_length < 0)
2440  flags.chunked_request = true;
2441  } else {
2444  requestSender = JobCallback(11,5,
2445  Dialer, this, HttpStateData::wroteLast);
2446  }
2447 
2448  /*
2449  * Is keep-alive okay for all request methods?
2450  */
2452  flags.keepalive = true;
2453  else if (request->flags.pinned)
2455  else if (!Config.onoff.server_pconns)
2456  flags.keepalive = false;
2457  else if (flags.tunneling)
2458  // tunneled non pinned bumped requests must not keepalive
2460  else if (_peer == nullptr)
2461  flags.keepalive = true;
2462  else if (_peer->stats.n_keepalives_sent < 10)
2463  flags.keepalive = true;
2464  else if ((double) _peer->stats.n_keepalives_recv /
2465  (double) _peer->stats.n_keepalives_sent > 0.50)
2466  flags.keepalive = true;
2467 
2468  if (_peer && !flags.tunneling) {
2469  /*The old code here was
2470  if (neighborType(_peer, request->url) == PEER_SIBLING && ...
2471  which is equivalent to:
2472  if (neighborType(_peer, URL()) == PEER_SIBLING && ...
2473  or better:
2474  if (((_peer->type == PEER_MULTICAST && p->options.mcast_siblings) ||
2475  _peer->type == PEER_SIBLINGS ) && _peer->options.allow_miss)
2476  flags.only_if_cached = 1;
2477 
2478  But I suppose it was a bug
2479  */
2481  flags.only_if_cached = true;
2482 
2484  }
2485 
2486  mb.init();
2487  buildRequestPrefix(&mb);
2488 
2489  debugs(11, 2, "HTTP Server " << serverConnection);
2490  debugs(11, 2, "HTTP Server REQUEST:\n---------\n" << mb.buf << "\n----------");
2491 
2493  return true;
2494 }
2495 
2496 bool
2498 {
2499  // parent's implementation can handle the no-encoding case
2500  if (!flags.chunked_request)
2501  return Client::getMoreRequestBody(buf);
2502 
2503  MemBuf raw;
2504 
2505  Must(requestBodySource != nullptr);
2506  if (!requestBodySource->getMoreData(raw))
2507  return false; // no request body bytes to chunk yet
2508 
2509  // optimization: pre-allocate buffer size that should be enough
2510  const mb_size_t rawDataSize = raw.contentSize();
2511  // we may need to send: hex-chunk-size CRLF raw-data CRLF last-chunk
2512  buf.init(16 + 2 + rawDataSize + 2 + 5, raw.max_capacity);
2513 
2514  buf.appendf("%x\r\n", static_cast<unsigned int>(rawDataSize));
2515  buf.append(raw.content(), rawDataSize);
2516  buf.append("\r\n", 2);
2517 
2518  Must(rawDataSize > 0); // we did not accidentally created last-chunk above
2519 
2520  // Do not send last-chunk unless we successfully received everything
2523  flags.sentLastChunk = true;
2524  buf.append("0\r\n\r\n", 5);
2525  }
2526 
2527  return true;
2528 }
2529 
2530 void
2532 {
2533  debugs(11, 3, fwd->request->method << ' ' << fwd->entry->url());
2534  AsyncJob::Start(new HttpStateData(fwd));
2535 }
2536 
2537 void
2539 {
2540  if (!sendRequest()) {
2541  debugs(11, 3, "httpStart: aborted");
2542  mustStop("HttpStateData::start failed");
2543  return;
2544  }
2545 
2546  ++ statCounter.server.all.requests;
2547  ++ statCounter.server.http.requests;
2548 
2549  /*
2550  * We used to set the read timeout here, but not any more.
2551  * Now its set in httpSendComplete() after the full request,
2552  * including request body, has been written to the server.
2553  */
2554 }
2555 
2557 bool
2559 {
2560 #if USE_HTTP_VIOLATIONS
2561  if (!Config.accessList.brokenPosts) {
2562  debugs(11, 5, "No brokenPosts list");
2563  return false;
2564  }
2565 
2567  ch.al = fwd->al;
2568  ch.syncAle(originalRequest().getRaw(), nullptr);
2569  if (!ch.fastCheck().allowed()) {
2570  debugs(11, 5, "didn't match brokenPosts");
2571  return false;
2572  }
2573 
2575  debugs(11, 3, "ignoring broken POST for closed " << serverConnection);
2576  assert(closeHandler != nullptr);
2577  return true; // prevent caller from proceeding as if nothing happened
2578  }
2579 
2580  debugs(11, 3, "finishingBrokenPost: fixing broken POST");
2582  requestSender = JobCallback(11,5,
2583  Dialer, this, HttpStateData::wroteLast);
2584  Comm::Write(serverConnection, "\r\n", 2, requestSender, nullptr);
2585  return true;
2586 #else
2587  return false;
2588 #endif /* USE_HTTP_VIOLATIONS */
2589 }
2590 
2592 bool
2594 {
2595  if (flags.sentLastChunk) {
2596  debugs(11, 5, "already sent last-chunk");
2597  return false;
2598  }
2599 
2600  Must(receivedWholeRequestBody); // or we should not be sending last-chunk
2601  flags.sentLastChunk = true;
2602 
2604  requestSender = JobCallback(11,5, Dialer, this, HttpStateData::wroteLast);
2605  Comm::Write(serverConnection, "0\r\n\r\n", 5, requestSender, nullptr);
2606  return true;
2607 }
2608 
2609 void
2611 {
2613  debugs(11,5, serverConnection);
2614 
2615  // do we need to write something after the last body byte?
2617  return;
2619  return;
2620 
2621  sendComplete();
2622 }
2623 
2624 // more origin request body data is available
2625 void
2627 {
2629  // XXX: we should check this condition in other callbacks then!
2630  // TODO: Check whether this can actually happen: We should unsubscribe
2631  // as a body consumer when the above condition(s) are detected.
2632  debugs(11, DBG_IMPORTANT, "Transaction aborted while reading HTTP body");
2633  return;
2634  }
2635 
2636  assert(requestBodySource != nullptr);
2637 
2638  if (requestBodySource->buf().hasContent()) {
2639  // XXX: why does not this trigger a debug message on every request?
2640 
2642  flags.abuse_detected = true;
2643  debugs(11, DBG_IMPORTANT, "http handleMoreRequestBodyAvailable: Likely proxy abuse detected '" << request->client_addr << "' -> '" << entry->url() << "'" );
2644 
2645  if (virginReply()->sline.status() == Http::scInvalidHeader) {
2646  closeServer();
2647  mustStop("HttpStateData::handleMoreRequestBodyAvailable");
2648  return;
2649  }
2650  }
2651  }
2652 
2654 }
2655 
2656 // premature end of the request body
2657 void
2659 {
2661  if (entry->isEmpty()) {
2662  debugs(11, 3, "request body aborted: " << serverConnection);
2663  // We usually get here when ICAP REQMOD aborts during body processing.
2664  // We might also get here if client-side aborts, but then our response
2665  // should not matter because either client-side will provide its own or
2666  // there will be no response at all (e.g., if the the client has left).
2668  static const auto d = MakeNamedErrorDetail("SRV_REQMOD_REQ_BODY");
2669  err->detailError(d);
2670  fwd->fail(err);
2671  }
2672 
2673  abortTransaction("request body producer aborted");
2674 }
2675 
2676 // called when we wrote request headers(!) or a part of the body
2677 void
2679 {
2680  if (io.size > 0)
2681  statCounter.server.http.kbytes_out += io.size;
2682 
2684 }
2685 
2686 void
2687 HttpStateData::abortAll(const char *reason)
2688 {
2689  debugs(11,5, "aborting transaction for " << reason <<
2690  "; " << serverConnection << ", this " << this);
2691  mustStop(reason);
2692 }
2693 
2695  : answer(HttpStateData::ReuseDecision::reuseNot), reason(nullptr), entry(e), statusCode(code) {}
2696 
2699 {
2700  answer = ans;
2701  reason = why;
2702  return answer;
2703 }
2704 
2705 std::ostream &operator <<(std::ostream &os, const HttpStateData::ReuseDecision &d)
2706 {
2707  static const char *ReuseMessages[] = {
2708  "do not cache and do not share", // reuseNot
2709  "cache positively and share", // cachePositively
2710  "cache negatively and share", // cacheNegatively
2711  "do not cache but share" // doNotCacheButShare
2712  };
2713 
2716  return os << ReuseMessages[d.answer] << " because " << d.reason <<
2717  "; HTTP status " << d.statusCode << " " << *(d.entry);
2718 }
2719 
~HttpStateData() override
Definition: http.cc:116
const MemBuf & buf() const
Definition: BodyPipe.h:137
void drop1xx(const char *reason)
Definition: http.cc:805
void Read(const Comm::ConnectionPointer &conn, AsyncCall::Pointer &callback)
Definition: Read.cc:40
bool proxy_only
Definition: CachePeer.h:109
const char * xstrerr(int error)
Definition: xstrerror.cc:83
void makePrivate(const bool shareable)
Definition: store.cc:174
bool handling1xx
Whether we are processing an HTTP 1xx control message.
Definition: StateFlags.h:30
struct SquidConfig::@84 Accel
bool hasProxyRevalidate() const
Definition: HttpHdrCc.h:117
char * buf
Definition: MemBuf.h:134
AsyncCall::Pointer comm_add_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:942
int connection_auth
0 - off, 1 - on, 2 - auto
Definition: CachePeer.h:226
virtual bool getMoreRequestBody(MemBuf &buf)
either fill buf with available [encoded] request body bytes or return false
Definition: Client.cc:442
void truncateVirginBody()
Definition: http.cc:1387
void httpHdrMangleList(HttpHeader *l, HttpRequest *request, const AccessLogEntryPointer &al, req_or_rep_t req_or_rep)
static auto Make(Args &&... args)
Definition: RefCount.h:34
bool keepalive_broken
Definition: StateFlags.h:58
bool expectingBody(const HttpRequestMethod &, int64_t &) const override
Definition: HttpReply.cc:528
unsigned int major
major version number
StoreEntry * entry
Definition: Client.h:177
void markPrematureReplyBodyEofFailure()
called on a premature EOF discovered when reading response body
Definition: http.cc:1412
void commUnsetConnTimeout(const Comm::ConnectionPointer &conn)
Definition: comm.cc:618
bool keepalive
whether the Squid-sent request offers to keep the connection persistent
Definition: StateFlags.h:21
@ scMethodNotAllowed
Definition: StatusCode.h:50
bool hasPrivate(const String **val=nullptr) const
Definition: HttpHdrCc.h:74
@ scNonAuthoritativeInformation
Definition: StatusCode.h:30
AnyP::ProtocolVersion http_ver
Definition: Message.h:72
void userInfo(const SBuf &s)
Definition: Uri.h:70
#define base64_encode_len(length)
Definition: base64.h:169
@ KEY_PRIVATE
Definition: enums.h:97
@ HDR_X_ACCELERATOR_VARY
void ignoreRange(const char *reason)
forgets about the cached Range header (for a reason)
Definition: HttpRequest.cc:618
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
@ scUnauthorized
Definition: StatusCode.h:46
@ METHOD_HEAD
Definition: MethodType.h:28
void delayRead()
Definition: Client.cc:1042
const char * rawBuf() const
Definition: SquidString.h:87
void noteDelayAwareReadChance() override
Definition: http.cc:1162
struct StatCounters::@105 server
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
http_hdr_owner_type owner
Definition: HttpHeader.h:177
bool ignoreCacheControl
Definition: http.h:72
Definition: Client.h:29
@ ERR_READ_ERROR
Definition: forward.h:28
void fd_bytes(const int fd, const int len, const IoDirection direction)
Definition: fd.cc:221
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:97
bool spoofClientIp
Definition: RequestFlags.h:74
const char * storeId() const
Definition: MemObject.cc:53
void releaseRequest(const bool shareable=false)
Definition: store.cc:458
bool waitingForDelayAwareReadChance
whether we are waiting for MemObject::delayRead() to call us back
Definition: Client.h:203
@ scBadRequest
Definition: StatusCode.h:45
bool chunked
reading a chunked response; TODO: rename
Definition: StateFlags.h:61
const char * getStr(Http::HdrType id) const
Definition: HttpHeader.cc:1296
#define LOCAL_ARRAY(type, name, size)
Definition: squid.h:62
mb_size_t size
Definition: MemBuf.h:135
SupportOrVeto cachable
whether the response may be stored in the cache
Definition: RequestFlags.h:35
int reads
Definition: IoStats.h:19
bool makePublic(const KeyScope keyScope=ksDefault)
Definition: store.cc:167
HttpHeader header
Definition: Message.h:74
bool offsetLimitExceeded(const int64_t limit) const
#define EBIT_SET(flag, bit)
Definition: defines.h:65
int hasListMember(Http::HdrType id, const char *member, const char separator) const
Definition: HttpHeader.cc:1795
Http::StateFlags flags
Definition: http.h:70
@ scMisdirectedRequest
Definition: StatusCode.h:63
void doneSendingRequestBody() override
Definition: http.cc:2610
Http::StatusCode peer_reply_status
last HTTP status code received
bool MonitorsRead(int fd)
whether the FD socket is being monitored for read
Definition: Read.cc:29
Answers make(const Answers ans, const char *why)
stores the corresponding decision
Definition: http.cc:2698
bool isEmpty() const
Definition: SBuf.h:435
MemObject * mem_obj
Definition: Store.h:220
AsyncCall::Pointer requestSender
Definition: Client.h:183
struct CachePeer::@21::@27 flags
RequestFlags flags
Definition: HttpRequest.h:141
String * upgradeHeaderOut
Upgrade header value sent to the origin server or cache peer.
Definition: http.h:76
void checkDateSkew(HttpReply *)
Definition: http.cc:629
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
void sentRequestBody(const CommIoCbParams &io) override
Definition: http.cc:2678
int64_t payloadTruncated
positive when we read more than we wanted
Definition: http.h:154
void reserveSpace(size_type minSpace)
Definition: SBuf.h:444
@ scLengthRequired
Definition: StatusCode.h:56
const Http::StatusCode statusCode
HTTP status for debugging.
Definition: http.h:43
bool hasPotentialSpace() const
Definition: MemBuf.h:75
bool surrogateNoStore
Definition: http.h:73
const char * url() const
Definition: store.cc:1566
a reference to a protocol name[/version] string; no 0-termination is assumed
int lastChunk
Definition: http.h:69
Comm::Flag ReadNow(CommIoCbParams &params, SBuf &buf)
Definition: Read.cc:81
size_t base64_encode_final(struct base64_encode_ctx *ctx, char *dst)
Definition: base64.c:308
void addVia(const AnyP::ProtocolVersion &ver, const HttpHeader *from=nullptr)
Definition: HttpHeader.cc:1078
int64_t getRangeOffsetLimit()
Definition: HttpRequest.cc:591
MemObject & mem()
Definition: Store.h:47
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
@ ENTRY_ABORTED
Definition: enums.h:110
#define ScheduleCallHere(call)
Definition: AsyncCall.h:166
virtual void handleRequestBodyProducerAborted()=0
Definition: Client.cc:355
void error(char *format,...)
AccessLogEntryPointer al
info for the future access.log entry
Definition: FwdState.h:204
unsigned int minor
minor version number
@ PROXY_CONNECTION
#define PEER_PROXY_NEGOTIATE_NOKEYTAB
bool finishingBrokenPost()
if broken posts are enabled for the request, try to fix and return true
Definition: http.cc:2558
CachePeer * _peer
Definition: http.h:67
bool connectionAuth
Definition: RequestFlags.h:85
@ scUnsupportedMediaType
Definition: StatusCode.h:60
bool hasContent() const
Definition: MemBuf.h:54
HttpReplyPointer reply
void sendControlMsg(HttpControlMsg) override
called to send the 1xx message and notify the Source
int64_t bodySize(const HttpRequestMethod &) const
Definition: HttpReply.cc:377
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
const char * ProtocolType_str[]
@ INPROGRESS
Definition: Flag.h:21
Definition: SBuf.h:93
mb_size_t buildRequestPrefix(MemBuf *mb)
Definition: http.cc:2367
size_t maxReplyHeaderSize
Definition: SquidConfig.h:137
ConnectionStatus statusIfComplete() const
Definition: http.cc:1062
AsyncCall::Pointer closeHandler
Definition: http.h:96
char * opt_forwarded_for
ConnectionStatus
Definition: http.h:97
@ scUseProxy
Definition: StatusCode.h:42
HttpStateData(FwdState *)
Definition: http.cc:80
@ scMovedPermanently
Definition: StatusCode.h:38
void noStoreRemote(bool v)
static void assembleVaryKey(String &vary, SBuf &vstr, const HttpRequest &request)
assemble a variant key (vary-mark) from the given Vary header and HTTP request
Definition: http.cc:554
DelayId mostBytesAllowed() const
Definition: MemObject.cc:466
@ ERR_INVALID_RESP
Definition: forward.h:41
bool peering
Whether the next TCP hop is a cache_peer, including originserver.
Definition: StateFlags.h:40
BodyPipe::Pointer requestBodySource
Definition: Client.h:182
a representation of a refresh pattern.
const char * blockSwitchingProtocols(const HttpReply &) const
Definition: http.cc:823
PconnPool * fwdPconnPool
a collection of previously used persistent Squid-to-peer HTTP(S) connections
Definition: FwdState.cc:78
void handle1xx(const HttpReplyPointer &)
ignore or start forwarding the 1xx response (a.k.a., control message)
Definition: http.cc:749
bool getMoreRequestBody(MemBuf &buf) override
either fill buf with available [encoded] request body bytes or return false
Definition: http.cc:2497
String getList(Http::HdrType id) const
Definition: HttpHeader.cc:921
bool sendRequest()
Definition: http.cc:2412
size_t calcReadBufferCapacityLimit() const
Definition: http.cc:1646
int fd
FD which the call was about. Set by the async call creator.
Definition: CommCalls.h:85
bool chunked_request
writing a chunked request
Definition: StateFlags.h:62
Auth::UserRequest::Pointer auth_user_request
Definition: HttpRequest.h:127
void handleRequestBodyProducerAborted() override
Definition: http.cc:2658
int detect_broken_server_pconns
Definition: SquidConfig.h:314
C * getRaw() const
Definition: RefCount.h:89
size_type spaceSize() const
Definition: SBuf.h:397
uint16_t flags
Definition: Store.h:231
size_t maybeMakeSpaceAvailable(size_t maxReadSize)
Definition: http.cc:1698
int cbdataReferenceValid(const void *p)
Definition: cbdata.cc:270
AccessLogEntry::Pointer al
info for the future access.log, and external ACL
const SBuf & image() const
void toLower()
converts all characters to lower case;
Definition: SBuf.cc:811
bool IsConnOpen(const Comm::ConnectionPointer &conn)
Definition: Connection.cc:27
bool allow_miss
Definition: CachePeer.h:130
IoStats IOStats
@ OK
Definition: Flag.h:16
std::optional< size_t > canBufferMoreReplyBytes() const
Definition: http.cc:1672
void putCc(const HttpHdrCc &cc)
Definition: HttpHeader.cc:1144
bool isAccepting() const
Definition: store.cc:1988
Http::StatusLine sline
Definition: HttpReply.h:56
struct SquidConfig::@77 Timeout
time_t getMaxAge(const char *url)
Definition: refresh.cc:639
@ ENDFILE
Definition: Flag.h:26
Comm::ConnectionPointer serverConnection
Definition: http.h:95
@ ERR_NONE
Definition: forward.h:15
int surrogate_is_remote
Definition: SquidConfig.h:313
StatusCode
Definition: StatusCode.h:20
static const char *const crlf
Definition: http.cc:74
err_type
Definition: forward.h:14
void adaptOrFinalizeReply()
Definition: Client.cc:1009
MemBuf * responseBodyBuffer
Definition: Client.h:174
void unregister(Comm::ConnectionPointer &conn)
Definition: FwdState.cc:499
struct CachePeer::@20 options
bool multipartRangeRequest() const
Definition: HttpRequest.cc:431
void fail(ErrorState *err)
Definition: FwdState.cc:458
void keepaliveAccounting(HttpReply *)
Definition: http.cc:609
BodyPipe::Pointer virginBodyDestination
Definition: Client.h:186
bool startRequestBodyFlow()
Definition: Client.cc:243
@ scRequestedRangeNotSatisfied
Definition: StatusCode.h:61
@ ERR_CLOSING
Definition: Flag.h:24
virtual void haveParsedReplyHeaders()
called when we have final (possibly adapted) reply headers; kids extend
Definition: Client.cc:541
bool auth_no_keytab
Definition: RequestFlags.h:32
bool Is1xx(const int sc)
whether this is an informational 1xx response status code
Definition: StatusCode.h:94
#define cbdataReference(var)
Definition: cbdata.h:348
int front_end_https
0 - off, 1 - on, 2 - auto
Definition: CachePeer.h:225
@ ENTRY_REVALIDATE_STALE
Definition: enums.h:95
bool connectionAuthDisabled
Definition: RequestFlags.h:87
void reset()
Definition: store.cc:1621
char * name
Definition: CachePeer.h:61
@ IF_MODIFIED_SINCE
bool decodeAndWriteReplyBody()
Definition: http.cc:1451
int xerrno
The last errno to occur. non-zero if flag is Comm::COMM_ERROR.
Definition: CommCalls.h:83
String extacl_user
Definition: HttpRequest.h:176
static const size_type maxSize
Maximum size of a SBuf. By design it MUST be < MAX(size_type)/2. Currently 256Mb.
Definition: SBuf.h:103
void httpHeaderPutStrf(HttpHeader *hdr, Http::HdrType id, const char *fmt,...)
StoreEntry * storeGetPublicByRequest(HttpRequest *req, const KeyScope keyScope)
Definition: store.cc:516
ProtocolType protocol
which protocol this version is for
RefCount< AsyncCallT< Dialer > > asyncCall(int aDebugSection, int aDebugLevel, const char *aName, const Dialer &aDialer)
Definition: AsyncCall.h:156
void readReply(const CommIoCbParams &io)
Definition: http.cc:1169
static bool decideIfWeDoRanges(HttpRequest *orig_request)
Definition: http.cc:2337
size_t getMoreData(MemBuf &buf)
Definition: BodyPipe.cc:294
@ scSeeOther
Definition: StatusCode.h:40
SBuf httpMakeVaryMark(HttpRequest *request, HttpReply const *reply)
Definition: http.cc:590
void syncAle(HttpRequest *adaptedRequest, const char *logUri) const override
assigns uninitialized adapted_request and url ALE components
StoreEntry * storeGetPublicByRequestMethod(HttpRequest *req, const HttpRequestMethod &method, const KeyScope keyScope)
Definition: store.cc:510
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:804
void processReplyBody() override
Definition: http.cc:1484
void serverComplete()
Definition: Client.cc:167
@ METHOD_OPTIONS
Definition: MethodType.h:31
@ CONTENT_LENGTH
void updateReply(const HttpReply::Pointer &)
SBuf & chop(size_type pos, size_type n=npos)
Definition: SBuf.cc:530
mb_size_t max_capacity
Definition: MemBuf.h:142
peer_t neighborType(const CachePeer *p, const AnyP::Uri &url)
Definition: neighbors.cc:117
#define MAX_IPSTRLEN
Length of buffer that needs to be allocated to old a null-terminated IP-string.
Definition: forward.h:25
@ scNotAcceptable
Definition: StatusCode.h:51
@ COMPLETE_NONPERSISTENT_MSG
Definition: http.h:100
Definition: Range.h:18
@ scGatewayTimeout
Definition: StatusCode.h:77
Definition: Raw.h:20
#define MAX_URL
Definition: defines.h:76
bool persistent() const
Definition: Message.cc:236
void sendComplete()
successfully wrote the entire request (including body, last-chunk, etc.)
Definition: http.cc:1753
time_t lastmod
Definition: HttpRequest.h:165
void strListAdd(String &str, const char *item, const size_t itemSize, const char delimiter)
Appends the given item of a given size to a delimiter-separated list in str.
Definition: StrList.cc:18
bool canGrowBy(const size_type growthLen) const
whether appending growthLen characters is safe (i.e., unlikely to assert)
Definition: SquidString.h:126
@ ERR_ICAP_FAILURE
Definition: forward.h:64
@ HTCP_CLR_INVALIDATION
Definition: enums.h:236
void bytesIn(int qty)
Definition: DelayId.cc:135
HttpHdrScTarget * getMergedTarget(const char *ourtarget)
Definition: HttpHdrSc.cc:290
String etag
A strong etag of the cached entry. Used for refreshing that entry.
Definition: HttpRequest.h:189
void processReplyHeader()
Definition: http.cc:646
ReuseDecision(const StoreEntry *e, const Http::StatusCode code)
Definition: http.cc:2694
static void httpBuildRequestHeader(HttpRequest *request, StoreEntry *entry, const AccessLogEntryPointer &al, HttpHeader *hdr_out, const CachePeer *peer, const Http::StateFlags &flags)
Definition: http.cc:1909
bool httpHeaderHasConnDir(const HttpHeader *hdr, const SBuf &directive)
static void httpMaybeRemovePublic(StoreEntry *, Http::StatusCode)
Definition: http.cc:170
time_t expires
Definition: HttpReply.h:44
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
int64_t getInt64() const
Definition: HttpHeader.cc:1667
@ scTemporaryRedirect
Definition: StatusCode.h:43
parameters for the async notePinnedConnectionBecameIdle() call
Definition: client_side.h:182
unsigned int front_end_https
send "Front-End-Https: On" header (off/on/auto=2)
Definition: StateFlags.h:18
void append(const char *c, int sz) override
Definition: MemBuf.cc:209
@ COMPLETE_PERSISTENT_MSG
Definition: http.h:99
const char * rawContent() const
Definition: SBuf.cc:509
SBuf consume(size_type n=npos)
Definition: SBuf.cc:481
int n_keepalives_sent
Definition: CachePeer.h:78
@ ENTRY_FWD_HDR_WAIT
Definition: enums.h:106
@ scFailedDependency
Definition: StatusCode.h:66
void maxAge(int32_t v)
Definition: HttpHdrCc.h:123
@ scSwitchingProtocols
Definition: StatusCode.h:23
int strListIsMember(const String *list, const SBuf &m, char del)
Definition: StrList.cc:46
virtual void doneSendingRequestBody()=0
Definition: Client.cc:344
time_t read
Definition: SquidConfig.h:112
@ scForbidden
Definition: StatusCode.h:48
#define SQUIDSBUFPRINT(s)
Definition: SBuf.h:32
bool dontRetry()
Definition: FwdState.h:130
void mustStop(const char *aReason)
Definition: AsyncJob.cc:85
const HttpReply * virginReply() const
Definition: Client.cc:102
SBuf inBuf
I/O buffer for receiving server responses.
Definition: http.h:71
bundles HTTP 1xx reply and the "successfully forwarded" callback
@ scInsufficientStorage
Definition: StatusCode.h:80
SBuf vary_headers
Definition: MemObject.h:221
@ scPreconditionFailed
Definition: StatusCode.h:57
@ scNotImplemented
Definition: StatusCode.h:74
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
#define MAX_LOGIN_SZ
Definition: defines.h:77
bool hasNoCacheWithParameters() const
Definition: HttpHdrCc.h:87
const Comm::ConnectionPointer & serverConnection() const
Definition: FwdState.h:138
void forApplicable(const ProtocolView &, const Visitor &) const
bool hasNoStore() const
virtual void handleMoreRequestBodyAvailable()
Definition: http.cc:2626
static void httpFixupAuthentication(HttpRequest *request, const HttpHeader *hdr_in, HttpHeader *hdr_out, const CachePeer *const peer, const Http::StateFlags &flags)
Definition: http.cc:1794
const char * username() const
Definition: UserRequest.cc:33
char * peer_login
Definition: HttpRequest.h:163
const Acl::Answer & fastCheck()
Definition: Checklist.cc:298
@ srcHttp
http_port or HTTP server
Definition: Message.h:39
struct IoStats::@59 Http
@ scBadGateway
Definition: StatusCode.h:75
const StoreEntry * entry
entry for debugging
Definition: http.h:42
Definition: MemBuf.h:23
struct StatCounters::@105::@115 all
bool toOriginPeer() const
Whether the next TCP/HTTP hop is an originserver cache_peer.
Definition: StateFlags.h:56
#define EBIT_TEST(flag, bit)
Definition: defines.h:67
CachePeer * getPeer() const
Definition: Connection.cc:110
void push(const Comm::ConnectionPointer &serverConn, const char *domain)
Definition: pconn.cc:435
bool canHandle1xx() const
whether the client is likely to be able to handle a 1xx reply
Definition: HttpRequest.cc:632
@ scPartialContent
Definition: StatusCode.h:33
@ scGone
Definition: StatusCode.h:55
int delById(Http::HdrType id)
Definition: HttpHeader.cc:799
Http::HdrType id
Definition: HttpHeader.h:66
void proceedAfter1xx()
restores state and resumes processing after 1xx is ignored or forwarded
Definition: http.cc:850
ConnectionStatus persistentConnStatus() const
Definition: http.cc:1112
bool hasSMaxAge(int32_t *val=nullptr) const
Definition: HttpHdrCc.h:127
@ INCOMPLETE_MSG
Definition: http.h:98
#define CallJobHere(debugSection, debugLevel, job, Class, method)
Definition: AsyncJobCalls.h:59
char * peer_domain
Definition: HttpRequest.h:170
@ scUnprocessableEntity
Definition: StatusCode.h:64
#define BBUF_SZ
void set(const AnyP::ProtocolVersion &newVersion, Http::StatusCode newStatus, const char *newReason=nullptr)
Definition: StatusLine.cc:35
void httpTimeout(const CommTimeoutCbParams &params)
Definition: http.cc:147
Comm::ConnectionPointer conn
Definition: CommCalls.h:80
@ RELEASE_REQUEST
prohibits making the key public
Definition: enums.h:93
Answers answer
the decision id
Definition: http.h:40
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:869
bool hasNoCache(const String **val=nullptr) const
Definition: HttpHdrCc.h:89
HttpRequestMethod method
Definition: MemObject.h:147
store_status_t store_status
Definition: Store.h:243
@ scPermanentRedirect
Definition: StatusCode.h:44
void putTime(Http::HdrType id, time_t htime)
Definition: HttpHeader.cc:1119
int neighbors_do_private_keys
@ ERR_ZERO_SIZE_OBJECT
Definition: forward.h:46
HttpUpgradeProtocolAccess * http_upgrade_request_protocols
http_upgrade_request_protocols
Definition: SquidConfig.h:469
#define assert(EX)
Definition: assert.h:17
@ scUriTooLong
Definition: StatusCode.h:59
@ scContentTooLarge
Definition: StatusCode.h:58
void setPayloadBuffer(MemBuf *parsedContent)
set the buffer to be used to store decoded chunk data
@ scContinue
Definition: StatusCode.h:22
HttpHdrCc * cache_control
Definition: Message.h:76
FwdState::Pointer fwd
Definition: Client.h:178
void haveParsedReplyHeaders() override
called when we have final (possibly adapted) reply headers; kids extend
Definition: http.cc:937
HierarchyLogEntry hier
Definition: HttpRequest.h:157
void changeAcl(const acl_access *)
change the current ACL list
Definition: Checklist.cc:187
acl_access * brokenPosts
Definition: SquidConfig.h:375
void neighborsHtcpClear(StoreEntry *e, HttpRequest *req, const HttpRequestMethod &method, htcp_clr_reason reason)
Definition: neighbors.cc:1675
std::ostream & CurrentException(std::ostream &os)
prints active (i.e., thrown but not yet handled) exception
virtual HttpRequestPointer originalRequest()
a hack to reach HttpStateData::orignal_request
Definition: Client.cc:573
Comm::Flag flag
comm layer result status.
Definition: CommCalls.h:82
const SBuf & remaining() const
the remaining unprocessed section of buffer
Definition: Parser.h:98
void abortTransaction(const char *reason)
Definition: http.h:118
@ TRANSFER_ENCODING
bool no_delay
Definition: CachePeer.h:128
@ scServiceUnavailable
Definition: StatusCode.h:76
const AnyP::UriScheme & getScheme() const
Definition: Uri.h:58
@ METHOD_TRACE
Definition: MethodType.h:30
#define cbdataReferenceDone(var)
Definition: cbdata.h:357
@ ENTRY_REVALIDATE_ALWAYS
Definition: enums.h:80
bool adaptationAccessCheckPending
Definition: Client.h:190
void onlyIfCached(bool v)
Definition: HttpHdrCc.h:146
void writeReplyBody()
Definition: http.cc:1425
void closeServer() override
Definition: http.cc:1772
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:70
@ scInternalServerError
Definition: StatusCode.h:73
HttpHdrRange * range
Definition: HttpRequest.h:143
HttpReply * setVirginReply(HttpReply *r)
Definition: Client.cc:116
void httpStateConnClosed(const CommCloseCbParams &params)
Definition: http.cc:139
void packInto(Packable *p, bool mask_sensitive_info=false) const
Definition: HttpHeader.cc:672
bool sentLastChunk
do not try to write last-chunk again
Definition: StateFlags.h:63
bool cacheNegatively()
Definition: store.cc:188
const char * c_str()
Definition: SBuf.cc:516
void Write(const Comm::ConnectionPointer &conn, const char *buf, int size, AsyncCall::Pointer &callback, FREE *free_func)
Definition: Write.cc:33
bool olderThan(const HttpReply *them) const
Definition: HttpReply.cc:638
SBuf & authority(bool requirePort=false) const
Definition: Uri.cc:721
void base64_encode_init(struct base64_encode_ctx *ctx)
Definition: base64.c:232
bool receivedWholeRequestBody
handleRequestBodyProductionEnded called
Definition: Client.h:200
#define CBDATA_CLASS_INIT(type)
Definition: cbdata.h:325
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:419
time_t squid_curtime
Definition: stub_libtime.cc:20
bool mayReadVirginReplyBody() const override
whether we may receive more virgin response body bytes
Definition: http.cc:1588
size_t bytesWanted(Range< size_t > const aRange, bool ignoreDelayPool=false) const
Definition: store.cc:213
SBuf & append(const SBuf &S)
Definition: SBuf.cc:185
bool isNoAddr() const
Definition: Address.cc:304
int64_t content_length
Definition: Message.h:83
time_t date
Definition: HttpReply.h:40
HttpReply * finalReply()
Definition: Client.cc:129
@ scNotModified
Definition: StatusCode.h:41
ReuseDecision::Answers reusableReply(ReuseDecision &decision)
Definition: http.cc:297
void start() override
called by AsyncStart; do not call directly
Definition: http.cc:2538
void markParsedVirginReplyAsWhole(const char *reasonWeAreSure)
Definition: Client.cc:158
bool mustKeepalive
Definition: RequestFlags.h:83
Http1::TeChunkedParser * httpChunkDecoder
Definition: http.h:149
void veto()
makes decision() false regardless of past or future support() calls
Definition: SupportOrVeto.h:29
void wroteLast(const CommIoCbParams &io)
called after writing the very last request byte (body, last-chunk, etc)
Definition: http.cc:1716
#define fd_table
Definition: fde.h:189
Http1::ResponseParserPointer hp
Parser being used at present to parse the HTTP/ICY server response.
Definition: http.h:148
bool params
Definition: CachePeer.h:138
const Comm::ConnectionPointer & dataConnection() const override
Definition: http.cc:133
@ ERR_READ_TIMEOUT
Definition: forward.h:26
assists in making and relaying entry caching/sharing decision
Definition: http.h:31
@ scFound
Definition: StatusCode.h:39
String getByName(const SBuf &name) const
Definition: HttpHeader.cc:981
@ ROR_REQUEST
@ PROTO_HTTPS
Definition: ProtocolType.h:27
size_t calcBufferSpaceToReserve(const size_t space, const size_t wantSpace) const
determine how much space the buffer needs to reserve
Definition: Client.cc:1079
@ scRequestTimeout
Definition: StatusCode.h:53
bool accelerated
Definition: RequestFlags.h:62
void maxAge(int v)
HttpRequestMethod method
Definition: HttpRequest.h:114
const char * reason
the decision reason
Definition: http.h:41
@ scNotFound
Definition: StatusCode.h:49
acl_access * reply
Definition: SquidConfig.h:379
noteTakeServerConnectionControl() callback parameter
Definition: client_side.h:213
struct CachePeer::@18 stats
void commSetConnTimeout(const Comm::ConnectionPointer &conn, time_t timeout, AsyncCall::Pointer &callback)
Definition: comm.cc:594
@ PROTO_HTTP
Definition: ProtocolType.h:25
char * host
Definition: CachePeer.h:66
int eof
Definition: http.h:68
const char * termedBuf() const
Definition: SquidString.h:93
bool allowed() const
Definition: Acl.h:82
int cache_miss_revalidate
Definition: SquidConfig.h:319
bool sawDateGoBack
Definition: http.h:161
void addVirginReplyBody(const char *buf, ssize_t len)
Definition: Client.cc:1054
int server_pconns
Definition: SquidConfig.h:305
const char * doneWithFwd
Definition: Client.h:211
virtual void sentRequestBody(const CommIoCbParams &io)=0
Definition: Client.cc:368
bool peerSupportsConnectionPinning() const
Definition: http.cc:879
bool timestampsSet()
Definition: store.cc:1387
@ scNoContent
Definition: StatusCode.h:31
@ PROXY_AUTHENTICATE
@ ERR_TOO_BIG
Definition: forward.h:40
bool forcedBodyContinuation
whether we have responded with HTTP 100 or FTP 150 already
Definition: HttpRequest.h:192
int has(Http::HdrType id) const
Definition: HttpHeader.cc:1070
void processReply()
Definition: http.cc:1288
char * content()
start of the added data
Definition: MemBuf.h:41
bool hasMaxAge(int32_t *val=nullptr) const
Definition: HttpHdrCc.h:122
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:1128
@ ERR_WRITE_ERROR
Definition: forward.h:29
bool finishingChunkedRequest()
if needed, write last-chunk to end the request body and return true
Definition: http.cc:2593
size_type size() const
Definition: SquidString.h:74
struct SquidConfig::@91 accessList
bool doneWithServer() const override
Definition: http.cc:1785
bool isEmpty() const
Definition: Store.h:65
bool refreshIsCachable(const StoreEntry *entry)
Definition: refresh.cc:518
#define Must(condition)
Definition: TextException.h:75
int redir_rewrites_host
Definition: SquidConfig.h:297
@ SURROGATE_CAPABILITY
#define DBG_IMPORTANT
Definition: Stream.h:38
@ scProxyAuthenticationRequired
Definition: StatusCode.h:52
size_t base64_encode_update(struct base64_encode_ctx *ctx, char *dst, size_t length, const uint8_t *src)
Definition: base64.c:265
char * surrogate_id
Definition: SquidConfig.h:220
void abortAll(const char *reason) override
abnormal transaction termination; reason is for debugging only
Definition: http.cc:2687
StoreEntry * entry
Definition: FwdState.h:202
StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod &method)
Definition: store.cc:504
void release(const bool shareable=false)
Definition: store.cc:1146
const HttpReply & freshestReply() const
Definition: MemObject.h:68
ssize_t mb_size_t
Definition: MemBuf.h:17
HttpHdrSc * surrogate_control
Definition: HttpReply.h:48
bool parse(const SBuf &) override
@ scInvalidHeader
Squid header parsing error.
Definition: StatusCode.h:88
@ scPaymentRequired
Definition: StatusCode.h:47
@ scLocked
Definition: StatusCode.h:65
static StoreEntry * findPreviouslyCachedEntry(StoreEntry *newEntry)
Definition: http.cc:160
@ scConflict
Definition: StatusCode.h:54
int n_keepalives_recv
Definition: CachePeer.h:79
void maybeReadVirginBody() override
read response data from the network
Definition: http.cc:1597
#define REFRESH_OVERRIDE(flag)
@ srcHttps
https_port or bumped http_port tunnel; HTTPS server
Definition: Message.h:33
int64_t readAheadGap
Definition: SquidConfig.h:98
void setNoDelay(bool const)
Definition: store.cc:250
bool IsReforwardableStatus(StatusCode)
whether to send the request to another peer based on the current response status code
Definition: StatusCode.cc:281
bool hasNoCacheWithoutParameters() const
Definition: HttpHdrCc.h:88
bool hasMustRevalidate() const
Definition: HttpHdrCc.h:112
void processSurrogateControl(HttpReply *)
Definition: http.cc:259
int64_t payloadSeen
amount of message payload/body received so far.
Definition: http.h:152
bool continueAfterParsingHeader()
Definition: http.cc:1319
@ scMultipleChoices
Definition: StatusCode.h:37
bool serverSwitchedProtocols
Definition: StateFlags.h:35
@ scOkay
Definition: StatusCode.h:27
bool waitingForCommRead
whether we are waiting for our Comm::Read() handler to be called
Definition: http.h:157
std::ostream & operator<<(std::ostream &os, const HttpStateData::ReuseDecision &d)
Definition: http.cc:2705
HttpHeaderEntry * getEntry(HttpHeaderPos *pos) const
Definition: HttpHeader.cc:716
@ scHeaderTooLarge
Header too large to process.
Definition: StatusCode.h:89
#define rfc1738_escape_part(x)
Definition: rfc1738.h:55
time_t lifetime
Definition: SquidConfig.h:114
struct SquidConfig::@90 onoff
SBuf & originForm() const
The RFC 7230 origin-form URI for currently stored values.
Definition: Uri.h:146
const SBuf & effectiveRequestUri() const
RFC 7230 section 5.5 - Effective Request URI.
Definition: HttpRequest.cc:741
static void copyOneHeaderFromClientsideRequestToUpstreamRequest(const HttpHeaderEntry *e, const String strConnection, const HttpRequest *request, HttpHeader *hdr_out, const int we_do_ranges, const Http::StateFlags &)
Definition: http.cc:2159
HttpRequest * request
Definition: FwdState.h:203
Ip::Address client_addr
Definition: HttpRequest.h:149
bool hasNoStore() const
Definition: HttpHdrCc.h:102
void host(const char *src)
Definition: Uri.cc:142
void forwardUpgrade(HttpHeader &)
Definition: http.cc:2085
@ PEER_SIBLING
Definition: enums.h:24
@ scExpectationFailed
Definition: StatusCode.h:62
HttpRequestPointer request
Definition: Client.h:179
int read_hist[histSize]
Definition: IoStats.h:21
virtual void noteTakeServerConnectionControl(ServerConnectionContext)
Definition: client_side.h:229
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:192
short int keep_alive
Definition: HttpReply.h:53
@ hoRequest
Definition: HttpHeader.h:36
HttpRequestPointer request
Definition: MemObject.h:205
String extacl_passwd
Definition: HttpRequest.h:178
void clean()
Definition: HttpHeader.cc:318
void comm_remove_close_handler(int fd, CLCB *handler, void *data)
Definition: comm.cc:971
void updateAle(const AccessLogEntry::Pointer &)
const char * getMD5Text() const
Definition: store.cc:207
bool originserver
Definition: CachePeer.h:144
time_t negativeTtl
Definition: SquidConfig.h:102
HttpHdrCc * getCc() const
Definition: HttpHeader.cc:1327
#define SQUIDSBUFPH
Definition: SBuf.h:31
void putInt64(Http::HdrType id, int64_t number)
Definition: HttpHeader.cc:1110
HttpHeaderEntry * clone() const
Definition: HttpHeader.cc:1639
CbcPointer< ConnStateData > clientConnectionManager
Definition: HttpRequest.h:230
bool hasMaxAge() const
bool hasPublic() const
Definition: HttpHdrCc.h:69
class SquidConfig Config
Definition: SquidConfig.cc:12
void httpStart(FwdState *fwd)
Definition: http.cc:2531
int strListGetItem(const String *str, char del, const char **item, int *ilen, const char **pos)
Definition: StrList.cc:78
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
StatCounters statCounter
Definition: StatCounters.cc:12
void clean()
Definition: String.cc:104
static void Start(const Pointer &job)
Definition: AsyncJob.cc:37
AnyP::ProtocolVersion ProtocolVersion(unsigned int aMajor, unsigned int aMinor)
HTTP version label information.
@ PROXY_AUTHORIZATION
void maxStale(int v)
#define CallJobHere1(debugSection, debugLevel, job, Class, method, arg1)
Definition: AsyncJobCalls.h:64
struct StatCounters::@105::@115 http
@ STORE_PENDING
Definition: enums.h:46

 

Introduction

Documentation

Support

Miscellaneous