ModXact.cc
Go to the documentation of this file.
1 /*
2  * Copyright (C) 1996-2025 The Squid Software Foundation and contributors
3  *
4  * Squid software is distributed under GPLv2+ license and includes
5  * contributions from numerous individuals and organizations.
6  * Please see the COPYING and CONTRIBUTORS files for details.
7  */
8 
9 /* DEBUG: section 93 ICAP (RFC 3507) Client */
10 
11 #include "squid.h"
12 #include "AccessLogEntry.h"
13 #include "adaptation/Answer.h"
14 #include "adaptation/History.h"
15 #include "adaptation/icap/Client.h"
16 #include "adaptation/icap/Config.h"
21 #include "adaptation/Initiator.h"
22 #include "auth/UserRequest.h"
23 #include "base/TextException.h"
24 #include "base64.h"
25 #include "comm.h"
26 #include "comm/Connection.h"
27 #include "error/Detail.h"
30 #include "HttpHeaderTools.h"
31 #include "HttpReply.h"
32 #include "MasterXaction.h"
33 #include "parser/Tokenizer.h"
34 #include "sbuf/Stream.h"
35 
36 // flow and terminology:
37 // HTTP| --> receive --> encode --> write --> |network
38 // end | <-- send <-- parse <-- read <-- |end
39 
40 // TODO: replace gotEncapsulated() with something faster; we call it often
41 
44 
45 static constexpr auto TheBackupLimit = BodyPipe::MaxCapacity;
46 
48 
50 {
51  memset(this, 0, sizeof(*this));
52 }
53 
56  AsyncJob("Adaptation::Icap::ModXact"),
57  Adaptation::Icap::Xaction("Adaptation::Icap::ModXact", aService),
58  virginConsumed(0),
59  bodyParser(nullptr),
60  canStartBypass(false), // too early
61  protectGroupBypass(true),
64  adaptHistoryId(-1),
65  trailerParser(nullptr),
66  alMaster(alp)
67 {
68  assert(virginHeader);
69 
70  virgin.setHeader(virginHeader); // sets virgin.body_pipe if needed
71  virgin.setCause(virginCause); // may be NULL
72 
73  // adapted header and body are initialized when we parse them
74 
75  // writing and reading ends are handled by Adaptation::Icap::Xaction
76 
77  // encoding
78  // nothing to do because we are using temporary buffers
79 
80  // parsing; TODO: do not set until we parse, see ICAPOptXact
81  icapReply = new HttpReply;
82  icapReply->protoPrefix = "ICAP/"; // TODO: make an IcapReply class?
83 
84  debugs(93,7, "initialized." << status());
85 }
86 
87 // initiator wants us to start
89 {
91 
92  // reserve an adaptation history slot (attempts are known at this time)
93  Adaptation::History::Pointer ah = virginRequest().adaptLogHistory();
94  if (ah != nullptr)
95  adaptHistoryId = ah->recordXactStart(service().cfg().key, icap_tr_start, attempts > 1);
96 
97  estimateVirginBody(); // before virgin disappears!
98 
99  canStartBypass = service().cfg().bypass;
100 
101  // it is an ICAP violation to send request to a service w/o known OPTIONS
102  // and the service may is too busy for us: honor Max-Connections and such
103  if (service().up() && service().availableForNew())
104  startWriting();
105  else
106  waitForService();
107 }
108 
110 {
111  const char *comment;
112  Must(!state.serviceWaiting);
113 
114  if (!service().up()) {
115  AsyncCall::Pointer call = JobCallback(93,5,
117 
118  service().callWhenReady(call);
119  comment = "to be up";
120  } else {
121  //The service is unavailable because of max-connection or other reason
122 
123  if (service().cfg().onOverload != srvWait) {
124  // The service is overloaded, but waiting to be available prohibited by
125  // user configuration (onOverload is set to "block" or "bypass")
126  if (service().cfg().onOverload == srvBlock)
127  disableBypass("not available", true);
128  else //if (service().cfg().onOverload == srvBypass)
129  canStartBypass = true;
130 
131  disableRetries();
132  disableRepeats("ICAP service is not available");
133 
134  debugs(93, 7, "will not wait for the service to be available" <<
135  status());
136 
137  throw TexcHere("ICAP service is not available");
138  }
139 
140  AsyncCall::Pointer call = JobCallback(93,5,
142  service().callWhenAvailable(call, state.waitedForService);
143  comment = "to be available";
144  }
145 
146  debugs(93, 7, "will wait for the service " << comment << status());
147  state.serviceWaiting = true; // after callWhenReady() which may throw
148  state.waitedForService = true;
149 }
150 
152 {
153  Must(state.serviceWaiting);
154  state.serviceWaiting = false;
155 
156  if (!service().up()) {
157  disableRetries();
158  disableRepeats("ICAP service is unusable");
159  throw TexcHere("ICAP service is unusable");
160  }
161 
162  if (service().availableForOld())
163  startWriting();
164  else
165  waitForService();
166 }
167 
169 {
170  Must(state.serviceWaiting);
171  state.serviceWaiting = false;
172 
173  if (service().up() && service().availableForOld())
174  startWriting();
175  else
176  waitForService();
177 }
178 
180 {
181  state.writing = State::writingConnect;
182 
183  decideOnPreview(); // must be decided before we decideOnRetries
184  decideOnRetries();
185 
186  openConnection();
187 }
188 
190 {
191  Must(state.writing == State::writingConnect);
192 
193  startReading(); // wait for early errors from the ICAP server
194 
195  MemBuf requestBuf;
196  requestBuf.init();
197 
198  makeRequestHeaders(requestBuf);
199  debugs(93, 9, "will write" << status() << ":\n" <<
200  (requestBuf.terminate(), requestBuf.content()));
201 
202  // write headers
203  state.writing = State::writingHeaders;
204  icap_tio_start = current_time;
205  scheduleWrite(requestBuf);
206 }
207 
209 {
210  debugs(93, 5, "Wrote " << sz << " bytes");
211 
212  if (state.writing == State::writingHeaders)
213  handleCommWroteHeaders();
214  else
215  handleCommWroteBody();
216 }
217 
219 {
220  Must(state.writing == State::writingHeaders);
221 
222  // determine next step
223  if (preview.enabled()) {
224  if (preview.done())
225  decideWritingAfterPreview("zero-size");
226  else
227  state.writing = State::writingPreview;
228  } else if (virginBody.expected()) {
229  state.writing = State::writingPrime;
230  } else {
231  stopWriting(true);
232  return;
233  }
234 
235  writeMore();
236 }
237 
239 {
240  debugs(93, 5, "checking whether to write more" << status());
241 
242  if (writer != nullptr) // already writing something
243  return;
244 
245  switch (state.writing) {
246 
247  case State::writingInit: // waiting for service OPTIONS
248  Must(state.serviceWaiting);
249  return;
250 
251  case State::writingConnect: // waiting for the connection to establish
252  case State::writingHeaders: // waiting for the headers to be written
253  case State::writingPaused: // waiting for the ICAP server response
254  case State::writingReallyDone: // nothing more to write
255  return;
256 
257  case State::writingAlmostDone: // was waiting for the last write
258  stopWriting(false);
259  return;
260 
261  case State::writingPreview:
262  writePreviewBody();
263  return;
264 
265  case State::writingPrime:
266  writePrimeBody();
267  return;
268 
269  default:
270  throw TexcHere("Adaptation::Icap::ModXact in bad writing state");
271  }
272 }
273 
275 {
276  debugs(93, 8, "will write Preview body from " <<
277  virgin.body_pipe << status());
278  Must(state.writing == State::writingPreview);
279  Must(virgin.body_pipe != nullptr);
280 
281  const size_t sizeMax = (size_t)virgin.body_pipe->buf().contentSize();
282  const size_t size = min(preview.debt(), sizeMax);
283  writeSomeBody("preview body", size);
284 
285  // change state once preview is written
286 
287  if (preview.done())
288  decideWritingAfterPreview("body");
289 }
290 
293 {
294  if (preview.ieof()) // nothing more to write
295  stopWriting(true);
296  else if (state.parsing == State::psIcapHeader) // did not get a reply yet
297  state.writing = State::writingPaused; // wait for the ICAP server reply
298  else
299  stopWriting(true); // ICAP server reply implies no post-preview writing
300 
301  debugs(93, 6, "decided on writing after " << kind << " preview" <<
302  status());
303 }
304 
306 {
307  Must(state.writing == State::writingPrime);
308  Must(virginBodyWriting.active());
309 
310  const size_t size = (size_t)virgin.body_pipe->buf().contentSize();
311  writeSomeBody("prime virgin body", size);
312 
313  if (virginBodyEndReached(virginBodyWriting)) {
314  debugs(93, 5, "wrote entire body");
315  stopWriting(true);
316  }
317 }
318 
319 void Adaptation::Icap::ModXact::writeSomeBody(const char *label, size_t size)
320 {
321  Must(!writer && state.writing < state.writingAlmostDone);
322  Must(virgin.body_pipe != nullptr);
323  debugs(93, 8, "will write up to " << size << " bytes of " <<
324  label);
325 
326  MemBuf writeBuf; // TODO: suggest a min size based on size and lastChunk
327 
328  writeBuf.init(); // note: we assume that last-chunk will fit
329 
330  const size_t writableSize = virginContentSize(virginBodyWriting);
331  const size_t chunkSize = min(writableSize, size);
332 
333  if (chunkSize) {
334  debugs(93, 7, "will write " << chunkSize <<
335  "-byte chunk of " << label);
336 
337  openChunk(writeBuf, chunkSize, false);
338  writeBuf.append(virginContentData(virginBodyWriting), chunkSize);
339  closeChunk(writeBuf);
340 
341  virginBodyWriting.progress(chunkSize);
342  virginConsume();
343  } else {
344  debugs(93, 7, "has no writable " << label << " content");
345  }
346 
347  const bool wroteEof = virginBodyEndReached(virginBodyWriting);
348  bool lastChunk = wroteEof;
349  if (state.writing == State::writingPreview) {
350  preview.wrote(chunkSize, wroteEof); // even if wrote nothing
351  lastChunk = lastChunk || preview.done();
352  }
353 
354  if (lastChunk) {
355  debugs(93, 8, "will write last-chunk of " << label);
356  addLastRequestChunk(writeBuf);
357  }
358 
359  debugs(93, 7, "will write " << writeBuf.contentSize()
360  << " raw bytes of " << label);
361 
362  if (writeBuf.hasContent()) {
363  scheduleWrite(writeBuf); // comm will free the chunk
364  } else {
365  writeBuf.clean();
366  }
367 }
368 
370 {
371  const bool ieof = state.writing == State::writingPreview && preview.ieof();
372  openChunk(buf, 0, ieof);
373  closeChunk(buf);
374 }
375 
376 void Adaptation::Icap::ModXact::openChunk(MemBuf &buf, size_t chunkSize, bool ieof)
377 {
378  buf.appendf((ieof ? "%x; ieof\r\n" : "%x\r\n"), (int) chunkSize);
379 }
380 
382 {
383  buf.append(ICAP::crlf, 2); // chunk-terminating CRLF
384 }
385 
387 {
388  const HttpRequest *request = virgin.cause ?
389  virgin.cause : dynamic_cast<const HttpRequest*>(virgin.header);
390  Must(request);
391  return *request;
392 }
393 
394 // did the activity reached the end of the virgin body?
396 {
397  return
398  !act.active() || // did all (assuming it was originally planned)
399  !virgin.body_pipe->expectMoreAfter(act.offset()); // will not have more
400 }
401 
402 // the size of buffered virgin body data available for the specified activity
403 // if this size is zero, we may be done or may be waiting for more data
405 {
406  Must(act.active());
407  // asbolute start of unprocessed data
408  const uint64_t dataStart = act.offset();
409  // absolute end of buffered data
410  const uint64_t dataEnd = virginConsumed + virgin.body_pipe->buf().contentSize();
411  Must(virginConsumed <= dataStart && dataStart <= dataEnd);
412  return static_cast<size_t>(dataEnd - dataStart);
413 }
414 
415 // pointer to buffered virgin body data available for the specified activity
417 {
418  Must(act.active());
419  const uint64_t dataStart = act.offset();
420  Must(virginConsumed <= dataStart);
421  return virgin.body_pipe->buf().content() + static_cast<size_t>(dataStart-virginConsumed);
422 }
423 
425 {
426  debugs(93, 9, "consumption guards: " << !virgin.body_pipe << isRetriable <<
427  isRepeatable << canStartBypass << protectGroupBypass);
428 
429  if (!virgin.body_pipe)
430  return; // nothing to consume
431 
432  if (isRetriable)
433  return; // do not consume if we may have to retry later
434 
435  BodyPipe &bp = *virgin.body_pipe;
436  const bool wantToPostpone = isRepeatable || canStartBypass || protectGroupBypass;
437 
438  if (wantToPostpone && bp.buf().potentialSpaceSize() > 0) {
439  // Postponing may increase memory footprint and slow the HTTP side
440  // down. Not postponing may increase the number of ICAP errors
441  // if the ICAP service fails. Should the trade-off be configurable?
442  debugs(93, 8, "postponing consumption from " << bp.status());
443  return;
444  }
445 
446  const size_t have = static_cast<size_t>(bp.buf().contentSize());
447  const uint64_t end = virginConsumed + have;
448  uint64_t offset = end;
449 
450  debugs(93, 9, "max virgin consumption offset=" << offset <<
451  " acts " << virginBodyWriting.active() << virginBodySending.active() <<
452  " consumed=" << virginConsumed <<
453  " from " << virgin.body_pipe->status());
454 
455  if (virginBodyWriting.active())
456  offset = min(virginBodyWriting.offset(), offset);
457 
458  if (virginBodySending.active())
459  offset = min(virginBodySending.offset(), offset);
460 
461  Must(virginConsumed <= offset && offset <= end);
462 
463  if (const size_t size = static_cast<size_t>(offset - virginConsumed)) {
464  debugs(93, 8, "consuming " << size << " out of " << have <<
465  " virgin body bytes");
466  bp.consume(size);
467  virginConsumed += size;
468  Must(!isRetriable); // or we should not be consuming
469  disableRepeats("consumed content");
470  disableBypass("consumed content", true);
471  }
472 }
473 
475 {
476  writeMore();
477 }
478 
479 // Called when we do not expect to call comm_write anymore.
480 // We may have a pending write though.
481 // If stopping nicely, we will just wait for that pending write, if any.
483 {
484  if (state.writing == State::writingReallyDone)
485  return;
486 
487  if (writer != nullptr) {
488  if (nicely) {
489  debugs(93, 7, "will wait for the last write" << status());
490  state.writing = State::writingAlmostDone; // may already be set
491  checkConsuming();
492  return;
493  }
494  debugs(93, 3, "will NOT wait for the last write" << status());
495 
496  // Comm does not have an interface to clear the writer callback nicely,
497  // but without clearing the writer we cannot recycle the connection.
498  // We prevent connection reuse and hope that we can handle a callback
499  // call at any time, usually in the middle of the destruction sequence!
500  // Somebody should add comm_remove_write_handler() to comm API.
501  reuseConnection = false;
502  ignoreLastWrite = true;
503  }
504 
505  debugs(93, 7, "will no longer write" << status());
506  if (virginBodyWriting.active()) {
507  virginBodyWriting.disable();
508  virginConsume();
509  }
510  state.writing = State::writingReallyDone;
511  checkConsuming();
512 }
513 
515 {
516  if (!virginBodySending.active())
517  return;
518 
519  debugs(93, 7, "will no longer backup" << status());
520  virginBodySending.disable();
521  virginConsume();
522 }
523 
525 {
526  return Adaptation::Icap::Xaction::doneAll() && !state.serviceWaiting &&
527  doneSending() &&
528  doneReading() && state.doneWriting();
529 }
530 
532 {
533  Must(haveConnection());
534  Must(!reader);
535  Must(!adapted.header);
536  Must(!adapted.body_pipe);
537 
538  // we use the same buffer for headers and body and then consume headers
539  readMore();
540 }
541 
543 {
544  if (reader != nullptr || doneReading()) {
545  debugs(93,3, "returning from readMore because reader or doneReading()");
546  return;
547  }
548 
549  // do not fill readBuf if we have no space to store the result
550  if (adapted.body_pipe != nullptr &&
551  !adapted.body_pipe->buf().hasPotentialSpace()) {
552  debugs(93,3, "not reading because ICAP reply pipe is full");
553  return;
554  }
555 
556  if (readBuf.length() < SQUID_TCP_SO_RCVBUF)
557  scheduleRead();
558  else
559  debugs(93,3, "cannot read with a full buffer");
560 }
561 
562 // comm module read a portion of the ICAP response for us
564 {
565  Must(!state.doneParsing());
566  icap_tio_finish = current_time;
567  parseMore();
568  readMore();
569 }
570 
572 {
573  Must(state.sending == State::sendingVirgin);
574  Must(adapted.body_pipe != nullptr);
575  Must(virginBodySending.active());
576 
577  const size_t sizeMax = virginContentSize(virginBodySending);
578  debugs(93,5, "will echo up to " << sizeMax << " bytes from " <<
579  virgin.body_pipe->status());
580  debugs(93,5, "will echo up to " << sizeMax << " bytes to " <<
581  adapted.body_pipe->status());
582 
583  if (sizeMax > 0) {
584  const size_t size = adapted.body_pipe->putMoreData(virginContentData(virginBodySending), sizeMax);
585  debugs(93,5, "echoed " << size << " out of " << sizeMax <<
586  " bytes");
587  virginBodySending.progress(size);
588  disableRepeats("echoed content");
589  disableBypass("echoed content", true);
590  virginConsume();
591  }
592 
593  if (virginBodyEndReached(virginBodySending)) {
594  debugs(93, 5, "echoed all" << status());
595  stopSending(true);
596  } else {
597  debugs(93, 5, "has " <<
598  virgin.body_pipe->buf().contentSize() << " bytes " <<
599  "and expects more to echo" << status());
600  // TODO: timeout if virgin or adapted pipes are broken
601  }
602 }
603 
605 {
606  return state.sending == State::sendingDone;
607 }
608 
609 // stop (or do not start) sending adapted message body
611 {
612  debugs(93, 7, "Enter stop sending ");
613  if (doneSending())
614  return;
615  debugs(93, 7, "Proceed with stop sending ");
616 
617  if (state.sending != State::sendingUndecided) {
618  debugs(93, 7, "will no longer send" << status());
619  if (adapted.body_pipe != nullptr) {
620  virginBodySending.disable();
621  // we may leave debts if we were echoing and the virgin
622  // body_pipe got exhausted before we echoed all planned bytes
623  const bool leftDebts = adapted.body_pipe->needsMoreData();
624  stopProducingFor(adapted.body_pipe, nicely && !leftDebts);
625  }
626  } else {
627  debugs(93, 7, "will not start sending" << status());
628  Must(!adapted.body_pipe);
629  }
630 
631  state.sending = State::sendingDone;
632  checkConsuming();
633 }
634 
635 // should be called after certain state.writing or state.sending changes
637 {
638  // quit if we already stopped or are still using the pipe
639  if (!virgin.body_pipe || !state.doneConsumingVirgin())
640  return;
641 
642  debugs(93, 7, "will stop consuming" << status());
643  stopConsumingFrom(virgin.body_pipe);
644 }
645 
647 {
648  debugs(93, 5, "have " << readBuf.length() << " bytes to parse" << status());
649  debugs(93, 5, "\n" << readBuf);
650 
651  if (state.parsingHeaders())
652  parseHeaders();
653 
654  if (state.parsing == State::psBody)
655  parseBody();
656 
657  if (state.parsing == State::psIcapTrailer)
658  parseIcapTrailer();
659 }
660 
661 void Adaptation::Icap::ModXact::callException(const std::exception &e)
662 {
663  if (!canStartBypass || isRetriable) {
664  if (!isRetriable) {
665  if (const TextException *te = dynamic_cast<const TextException *>(&e))
666  detailError(new ExceptionErrorDetail(te->id()));
667  else
668  detailError(new ExceptionErrorDetail(Here().id()));
669  }
671  return;
672  }
673 
674  try {
675  debugs(93, 3, "bypassing " << inCall << " exception: " <<
676  e.what() << ' ' << status());
677  bypassFailure();
678  } catch (const TextException &bypassTe) {
679  detailError(new ExceptionErrorDetail(bypassTe.id()));
681  } catch (const std::exception &bypassE) {
682  detailError(new ExceptionErrorDetail(Here().id()));
684  }
685 }
686 
688 {
689  disableBypass("already started to bypass", false);
690 
691  Must(!isRetriable); // or we should not be bypassing
692  // TODO: should the same be enforced for isRepeatable? Check icap_repeat??
693 
694  prepEchoing();
695 
696  startSending();
697 
698  // end all activities associated with the ICAP server
699 
700  stopParsing(false);
701 
702  stopWriting(true); // or should we force it?
703  if (haveConnection()) {
704  reuseConnection = false; // be conservative
705  cancelRead(); // may not work; and we cannot stop connecting either
706  if (!doneWithIo())
707  debugs(93, 7, "Warning: bypass failed to stop I/O" << status());
708  }
709 
710  service().noteFailure(); // we are bypassing, but this is still a failure
711 }
712 
713 void Adaptation::Icap::ModXact::disableBypass(const char *reason, bool includingGroupBypass)
714 {
715  if (canStartBypass) {
716  debugs(93,7, "will never start bypass because " << reason);
717  canStartBypass = false;
718  }
719  if (protectGroupBypass && includingGroupBypass) {
720  debugs(93,7, "not protecting group bypass because " << reason);
721  protectGroupBypass = false;
722  }
723 }
724 
725 // note that allocation for echoing is done in handle204NoContent()
727 {
728  if (adapted.header) // already allocated
729  return;
730 
731  if (gotEncapsulated("res-hdr")) {
732  adapted.setHeader(new HttpReply);
733  setOutcome(service().cfg().method == ICAP::methodReqmod ?
735  } else if (gotEncapsulated("req-hdr")) {
736  adapted.setHeader(new HttpRequest(virginRequest().masterXaction));
737  setOutcome(xoModified);
738  } else
739  throw TexcHere("Neither res-hdr nor req-hdr in maybeAllocateHttpMsg()");
740 }
741 
743 {
744  Must(state.parsingHeaders());
745 
746  if (state.parsing == State::psIcapHeader) {
747  debugs(93, 5, "parse ICAP headers");
748  parseIcapHead();
749  }
750 
751  if (state.parsing == State::psHttpHeader) {
752  debugs(93, 5, "parse HTTP headers");
753  parseHttpHead();
754  }
755 
756  if (state.parsingHeaders()) { // need more data
757  Must(mayReadMore());
758  return;
759  }
760 
761  startSending();
762 }
763 
764 // called after parsing all headers or when bypassing an exception
766 {
767  disableRepeats("sent headers");
768  disableBypass("sent headers", true);
769  sendAnswer(Answer::Forward(adapted.header));
770 
771  if (state.sending == State::sendingVirgin)
772  echoMore();
773  else {
774  // If we are not using the virgin HTTP object update the
775  // Http::Message::sources flag.
776  // The state.sending may set to State::sendingVirgin in the case
777  // of 206 responses too, where we do not want to update Http::Message::sources
778  // flag. However even for 206 responses the state.sending is
779  // not set yet to sendingVirgin. This is done in later step
780  // after the parseBody method called.
781  updateSources();
782  }
783 }
784 
786 {
787  Must(state.sending == State::sendingUndecided);
788 
789  if (!parseHead(icapReply.getRaw()))
790  return;
791 
792  if (expectIcapTrailers()) {
793  Must(!trailerParser);
794  trailerParser = new TrailerParser;
795  }
796 
797  static SBuf close("close", 5);
798  if (httpHeaderHasConnDir(&icapReply->header, close)) {
799  debugs(93, 5, "found connection close");
800  reuseConnection = false;
801  }
802 
803  switch (icapReply->sline.status()) {
804 
805  case Http::scContinue:
806  handle100Continue();
807  break;
808 
809  case Http::scOkay:
810  case Http::scCreated: // Symantec Scan Engine 5.0 and later when modifying HTTP msg
811 
812  if (!validate200Ok()) {
813  throw TexcHere("Invalid ICAP Response");
814  } else {
815  handle200Ok();
816  }
817 
818  break;
819 
820  case Http::scNoContent:
821  handle204NoContent();
822  break;
823 
825  handle206PartialContent();
826  break;
827 
828  default:
829  debugs(93, 5, "ICAP status " << icapReply->sline.status());
830  handleUnknownScode();
831  break;
832  }
833 
834  const HttpRequest *request = dynamic_cast<HttpRequest*>(adapted.header);
835  if (!request)
836  request = &virginRequest();
837 
838  // update the cross-transactional database if needed (all status codes!)
839  if (const char *xxName = Adaptation::Config::masterx_shared_name) {
840  Adaptation::History::Pointer ah = request->adaptHistory(true);
841  if (ah != nullptr) { // TODO: reorder checks to avoid creating history
842  const String val = icapReply->header.getByName(xxName);
843  if (val.size() > 0) // XXX: HttpHeader lacks empty value detection
844  ah->updateXxRecord(xxName, val);
845  }
846  }
847 
848  // update the adaptation plan if needed (all status codes!)
849  if (service().cfg().routing) {
850  String services;
851  if (icapReply->header.getList(Http::HdrType::X_NEXT_SERVICES, &services)) {
852  Adaptation::History::Pointer ah = request->adaptHistory(true);
853  if (ah != nullptr)
854  ah->updateNextServices(services);
855  }
856  } // TODO: else warn (occasionally!) if we got Http::HdrType::X_NEXT_SERVICES
857 
858  // We need to store received ICAP headers for <icapLastHeader logformat option.
859  // If we already have stored headers from previous ICAP transaction related to this
860  // request, old headers will be replaced with the new one.
861 
863  if (ah != nullptr)
864  ah->recordMeta(&icapReply->header);
865 
866  // handle100Continue() manages state.writing on its own.
867  // Non-100 status means the server needs no postPreview data from us.
868  if (state.writing == State::writingPaused)
869  stopWriting(true);
870 }
871 
875 
876  if (parsePart(trailerParser, "trailer")) {
877  for (const auto &e: trailerParser->trailer.entries)
878  debugs(93, 5, "ICAP trailer: " << e->name << ": " << e->value);
879  stopParsing();
880  }
881 }
882 
884 {
885  if (service().cfg().method == ICAP::methodRespmod)
886  return gotEncapsulated("res-hdr");
887 
888  return service().cfg().method == ICAP::methodReqmod &&
889  expectHttpHeader();
890 }
891 
893 {
894  Must(state.writing == State::writingPaused);
895  // server must not respond before the end of preview: we may send ieof
896  Must(preview.enabled() && preview.done() && !preview.ieof());
897 
898  // 100 "Continue" cancels our Preview commitment,
899  // but not commitment to handle 204 or 206 outside Preview
900  if (!state.allowedPostview204 && !state.allowedPostview206)
901  stopBackup();
902 
903  state.parsing = State::psIcapHeader; // eventually
904  icapReply->reset();
905 
906  state.writing = State::writingPrime;
907 
908  writeMore();
909 }
910 
912 {
913  state.parsing = State::psHttpHeader;
914  state.sending = State::sendingAdapted;
915  stopBackup();
916  checkConsuming();
917 }
918 
920 {
921  stopParsing();
922  prepEchoing();
923 }
924 
926 {
927  if (state.writing == State::writingPaused) {
928  Must(preview.enabled());
929  Must(state.allowedPreview206);
930  debugs(93, 7, "206 inside preview");
931  } else {
932  Must(state.writing > State::writingPaused);
933  Must(state.allowedPostview206);
934  debugs(93, 7, "206 outside preview");
935  }
936  state.parsing = State::psHttpHeader;
937  state.sending = State::sendingAdapted;
938  state.readyForUob = true;
939  checkConsuming();
940 }
941 
942 // Called when we receive a 204 No Content response and
943 // when we are trying to bypass a service failure.
944 // We actually start sending (echoig or not) in startSending.
946 {
947  disableRepeats("preparing to echo content");
948  disableBypass("preparing to echo content", true);
949  setOutcome(xoEcho);
950 
951  // We want to clone the HTTP message, but we do not want
952  // to copy some non-HTTP state parts that Http::Message kids carry in them.
953  // Thus, we cannot use a smart pointer, copy constructor, or equivalent.
954  // Instead, we simply write the HTTP message and "clone" it by parsing.
955  // TODO: use Http::Message::clone()!
956 
957  Http::Message *oldHead = virgin.header;
958  debugs(93, 7, "cloning virgin message " << oldHead);
959 
960  MemBuf httpBuf;
961 
962  // write the virgin message into a memory buffer
963  httpBuf.init();
964  packHead(httpBuf, oldHead);
965 
966  // allocate the adapted message and copy metainfo
967  Must(!adapted.header);
968  {
969  Http::MessagePointer newHead;
970  if (const HttpRequest *r = dynamic_cast<const HttpRequest*>(oldHead)) {
971  newHead = new HttpRequest(r->masterXaction);
972  } else if (dynamic_cast<const HttpReply*>(oldHead)) {
973  newHead = new HttpReply;
974  }
975  Must(newHead);
976 
977  newHead->inheritProperties(oldHead);
978 
979  adapted.setHeader(newHead.getRaw());
980  }
981 
982  // parse the buffer back
984 
985  httpBuf.terminate(); // Http::Message::parse requires nil-terminated buffer
986  Must(adapted.header->parse(httpBuf.content(), httpBuf.contentSize(), true, &error));
987  Must(adapted.header->hdr_sz == httpBuf.contentSize()); // no leftovers
988 
989  httpBuf.clean();
990 
991  debugs(93, 7, "cloned virgin message " << oldHead << " to " <<
992  adapted.header);
993 
994  // setup adapted body pipe if needed
995  if (oldHead->body_pipe != nullptr) {
996  debugs(93, 7, "will echo virgin body from " <<
997  oldHead->body_pipe);
998  if (!virginBodySending.active())
999  virginBodySending.plan(); // will throw if not possible
1000  state.sending = State::sendingVirgin;
1001  checkConsuming();
1002 
1003  // TODO: optimize: is it possible to just use the oldHead pipe and
1004  // remove ICAP from the loop? This echoing is probably a common case!
1005  makeAdaptedBodyPipe("echoed virgin response");
1006  if (oldHead->body_pipe->bodySizeKnown())
1007  adapted.body_pipe->setBodySize(oldHead->body_pipe->bodySize());
1008  debugs(93, 7, "will echo virgin body to " <<
1009  adapted.body_pipe);
1010  } else {
1011  debugs(93, 7, "no virgin body to echo");
1012  stopSending(true);
1013  }
1014 }
1015 
1019 {
1020  Must(virginBodySending.active());
1021  Must(virgin.header->body_pipe != nullptr);
1022 
1023  setOutcome(xoPartEcho);
1024 
1025  debugs(93, 7, "will echo virgin body suffix from " <<
1026  virgin.header->body_pipe << " offset " << pos );
1027 
1028  // check that use-original-body=N does not point beyond buffered data
1029  const uint64_t virginDataEnd = virginConsumed +
1030  virgin.body_pipe->buf().contentSize();
1031  Must(pos <= virginDataEnd);
1032  virginBodySending.progress(static_cast<size_t>(pos));
1033 
1034  state.sending = State::sendingVirgin;
1035  checkConsuming();
1036 
1037  if (virgin.header->body_pipe->bodySizeKnown())
1038  adapted.body_pipe->expectProductionEndAfter(virgin.header->body_pipe->bodySize() - pos);
1039 
1040  debugs(93, 7, "will echo virgin body suffix to " <<
1041  adapted.body_pipe);
1042 
1043  // Start echoing data
1044  echoMore();
1045 }
1046 
1048 {
1049  stopParsing(false);
1050  stopBackup();
1051  // TODO: mark connection as "bad"
1052 
1053  // Terminate the transaction; we do not know how to handle this response.
1054  throw TexcHere("Unsupported ICAP status code");
1055 }
1056 
1058 {
1059  if (expectHttpHeader()) {
1060  replyHttpHeaderSize = 0;
1061  maybeAllocateHttpMsg();
1062 
1063  if (!parseHead(adapted.header))
1064  return; // need more header data
1065 
1066  if (adapted.header)
1067  replyHttpHeaderSize = adapted.header->hdr_sz;
1068 
1069  if (dynamic_cast<HttpRequest*>(adapted.header)) {
1070  const HttpRequest *oldR = dynamic_cast<const HttpRequest*>(virgin.header);
1071  Must(oldR);
1072  // TODO: the adapted request did not really originate from the
1073  // client; give proxy admin an option to prevent copying of
1074  // sensitive client information here. See the following thread:
1075  // http://www.squid-cache.org/mail-archive/squid-dev/200703/0040.html
1076  }
1077 
1078  // Maybe adapted.header==NULL if HttpReply and have Http 0.9 ....
1079  if (adapted.header)
1080  adapted.header->inheritProperties(virgin.header);
1081  }
1082 
1083  decideOnParsingBody();
1084 }
1085 
1086 template<class Part>
1087 bool Adaptation::Icap::ModXact::parsePart(Part *part, const char *description)
1088 {
1089  Must(part);
1090  debugs(93, 5, "have " << readBuf.length() << ' ' << description << " bytes to parse; state: " << state.parsing);
1092  // XXX: performance regression. c_str() data copies
1093  // XXX: Http::Message::parse requires a terminated string buffer
1094  const char *tmpBuf = readBuf.c_str();
1095  const bool parsed = part->parse(tmpBuf, readBuf.length(), commEof, &error);
1096  debugs(93, (!parsed && error) ? 2 : 5, description << " parsing result: " << parsed << " detail: " << error);
1097  Must(parsed || !error);
1098  if (parsed)
1099  readBuf.consume(part->hdr_sz);
1100  return parsed;
1101 }
1102 
1103 // parses both HTTP and ICAP headers
1104 bool
1106 {
1107  if (!parsePart(head, "head")) {
1108  head->reset();
1109  return false;
1110  }
1111  return true;
1112 }
1113 
1115 {
1116  return gotEncapsulated("res-hdr") || gotEncapsulated("req-hdr");
1117 }
1118 
1120 {
1121  return gotEncapsulated("res-body") || gotEncapsulated("req-body");
1122 }
1123 
1125 {
1126  String trailers;
1127  const bool promisesToSendTrailer = icapReply->header.getByIdIfPresent(Http::HdrType::TRAILER, &trailers);
1128  const bool supportsTrailers = icapReply->header.hasListMember(Http::HdrType::ALLOW, "trailers", ',');
1129  // ICAP Trailer specs require us to reject transactions having either Trailer
1130  // header or Allow:trailers
1131  Must((promisesToSendTrailer == supportsTrailers) || (!promisesToSendTrailer && supportsTrailers));
1132  if (promisesToSendTrailer && !trailers.size())
1133  debugs(93, DBG_IMPORTANT, "ERROR: ICAP Trailer response header field must not be empty (salvaged)");
1134  return promisesToSendTrailer;
1135 }
1136 
1138 {
1139  if (expectHttpBody()) {
1140  debugs(93, 5, "expecting a body");
1141  state.parsing = State::psBody;
1142  replyHttpBodySize = 0;
1143  bodyParser = new Http1::TeChunkedParser;
1144  bodyParser->parseExtensionValuesWith(&extensionParser);
1145  makeAdaptedBodyPipe("adapted response from the ICAP server");
1146  Must(state.sending == State::sendingAdapted);
1147  } else {
1148  debugs(93, 5, "not expecting a body");
1149  if (trailerParser)
1150  state.parsing = State::psIcapTrailer;
1151  else
1152  stopParsing();
1153  stopSending(true);
1154  }
1155 }
1156 
1158 {
1159  Must(state.parsing == State::psBody);
1160  Must(bodyParser);
1161 
1162  debugs(93, 5, "have " << readBuf.length() << " body bytes to parse");
1163 
1164  // the parser will throw on errors
1165  BodyPipeCheckout bpc(*adapted.body_pipe);
1166  bodyParser->setPayloadBuffer(&bpc.buf);
1167  const bool parsed = bodyParser->parse(readBuf);
1168  readBuf = bodyParser->remaining(); // sync buffers after parse
1169  bpc.checkIn();
1170 
1171  debugs(93, 5, "have " << readBuf.length() << " body bytes after parsed all: " << parsed);
1172  replyHttpBodySize += adapted.body_pipe->buf().contentSize();
1173 
1174  // TODO: expose BodyPipe::putSize() to make this check simpler and clearer
1175  // TODO: do we really need this if we disable when sending headers?
1176  if (adapted.body_pipe->buf().contentSize() > 0) { // parsed something sometime
1177  disableRepeats("sent adapted content");
1178  disableBypass("sent adapted content", true);
1179  }
1180 
1181  if (parsed) {
1182  if (state.readyForUob && extensionParser.sawUseOriginalBody())
1183  prepPartialBodyEchoing(extensionParser.useOriginalBody());
1184  else
1185  stopSending(true); // the parser succeeds only if all parsed data fits
1186  if (trailerParser)
1187  state.parsing = State::psIcapTrailer;
1188  else
1189  stopParsing();
1190  return;
1191  }
1192 
1193  debugs(93,3, this << " needsMoreData = " << bodyParser->needsMoreData());
1194 
1195  if (bodyParser->needsMoreData()) {
1196  debugs(93,3, this);
1197  Must(mayReadMore());
1198  readMore();
1199  }
1200 
1201  if (bodyParser->needsMoreSpace()) {
1202  Must(!doneSending()); // can hope for more space
1203  Must(adapted.body_pipe->buf().contentSize() > 0); // paranoid
1204  // TODO: there should be a timeout in case the sink is broken
1205  // or cannot consume partial content (while we need more space)
1206  }
1207 }
1208 
1209 void Adaptation::Icap::ModXact::stopParsing(const bool checkUnparsedData)
1210 {
1211  if (state.parsing == State::psDone)
1212  return;
1213 
1214  if (checkUnparsedData)
1215  Must(readBuf.isEmpty());
1216 
1217  debugs(93, 7, "will no longer parse" << status());
1218 
1219  delete bodyParser;
1220  bodyParser = nullptr;
1221 
1222  delete trailerParser;
1223  trailerParser = nullptr;
1224 
1225  state.parsing = State::psDone;
1226 }
1227 
1228 // HTTP side added virgin body data
1230 {
1231  writeMore();
1232 
1233  if (state.sending == State::sendingVirgin)
1234  echoMore();
1235 }
1236 
1237 // HTTP side sent us all virgin info
1239 {
1240  Must(virgin.body_pipe->productionEnded());
1241 
1242  // push writer and sender in case we were waiting for the last-chunk
1243  writeMore();
1244 
1245  if (state.sending == State::sendingVirgin)
1246  echoMore();
1247 }
1248 
1249 // body producer aborted, but the initiator may still want to know
1250 // the answer, even though the HTTP message has been truncated
1252 {
1253  Must(virgin.body_pipe->productionEnded());
1254 
1255  // push writer and sender in case we were waiting for the last-chunk
1256  writeMore();
1257 
1258  if (state.sending == State::sendingVirgin)
1259  echoMore();
1260 }
1261 
1262 // adapted body consumer wants more adapted data and
1263 // possibly freed some buffer space
1265 {
1266  if (state.sending == State::sendingVirgin)
1267  echoMore();
1268  else if (state.sending == State::sendingAdapted)
1269  parseMore();
1270  else
1271  Must(state.sending == State::sendingUndecided);
1272 }
1273 
1274 // adapted body consumer aborted
1276 {
1277  static const auto d = MakeNamedErrorDetail("ICAP_XACT_BODY_CONSUMER_ABORT");
1278  detailError(d);
1279  mustStop("adapted body consumer aborted");
1280 }
1281 
1283 {
1284  delete bodyParser;
1285  delete trailerParser;
1286 }
1287 
1288 // internal cleanup
1290 {
1291  debugs(93, 5, "swan sings" << status());
1292 
1293  stopWriting(false);
1294  stopSending(false);
1295 
1296  if (theInitiator.set()) { // we have not sent the answer to the initiator
1297  static const auto d = MakeNamedErrorDetail("ICAP_XACT_OTHER");
1298  detailError(d);
1299  }
1300 
1301  // update adaptation history if start was called and we reserved a slot
1302  Adaptation::History::Pointer ah = virginRequest().adaptLogHistory();
1303  if (ah != nullptr && adaptHistoryId >= 0)
1304  ah->recordXactFinish(adaptHistoryId);
1305 
1307 }
1308 
1310 
1312 {
1313  HttpRequest *adapted_request_ = nullptr;
1314  HttpReply *adapted_reply_ = nullptr;
1315  HttpRequest *virgin_request_ = const_cast<HttpRequest*>(&virginRequest());
1316  if (!(adapted_request_ = dynamic_cast<HttpRequest*>(adapted.header))) {
1317  // if the request was not adapted, use virgin request to simplify
1318  // the code further below
1319  adapted_request_ = virgin_request_;
1320  adapted_reply_ = dynamic_cast<HttpReply*>(adapted.header);
1321  }
1322 
1323  Adaptation::Icap::History::Pointer h = virgin_request_->icapHistory();
1324  Must(h != nullptr); // ICAPXaction::maybeLog calls only if there is a log
1325  al.icp.opcode = ICP_INVALID;
1326  al.url = h->log_uri.termedBuf();
1327  const Adaptation::Icap::ServiceRep &s = service();
1328  al.icap.reqMethod = s.cfg().method;
1329 
1330  al.cache.caddr = virgin_request_->client_addr;
1331 
1332  al.request = virgin_request_;
1333  HTTPMSGLOCK(al.request);
1334  al.adapted_request = adapted_request_;
1335  HTTPMSGLOCK(al.adapted_request);
1336 
1337  // XXX: This reply (and other ALE members!) may have been needed earlier.
1338  al.reply = adapted_reply_;
1339 
1340 #if USE_OPENSSL
1341  if (h->ssluser.size())
1342  al.cache.ssluser = h->ssluser.termedBuf();
1343 #endif
1344  al.cache.code = h->logType;
1345 
1346  const Http::Message *virgin_msg = dynamic_cast<HttpReply*>(virgin.header);
1347  if (!virgin_msg)
1348  virgin_msg = virgin_request_;
1349  assert(virgin_msg != virgin.cause);
1350  al.http.clientRequestSz.header = virgin_msg->hdr_sz;
1351  if (virgin_msg->body_pipe != nullptr)
1352  al.http.clientRequestSz.payloadData = virgin_msg->body_pipe->producedSize();
1353 
1354  // leave al.icap.bodyBytesRead negative if no body
1355  if (replyHttpHeaderSize >= 0 || replyHttpBodySize >= 0) {
1356  const int64_t zero = 0; // to make max() argument types the same
1357  const uint64_t headerSize = max(zero, replyHttpHeaderSize);
1358  const uint64_t bodySize = max(zero, replyHttpBodySize);
1359  al.icap.bodyBytesRead = headerSize + bodySize;
1360  al.http.clientReplySz.header = headerSize;
1361  al.http.clientReplySz.payloadData = bodySize;
1362  }
1363 
1364  if (adapted_reply_) {
1365  al.http.code = adapted_reply_->sline.status();
1366  al.http.content_type = adapted_reply_->content_type.termedBuf();
1367  if (replyHttpBodySize >= 0)
1368  al.cache.highOffset = replyHttpBodySize;
1369  //don't set al.cache.objectSize because it hasn't exist yet
1370  }
1371  prepareLogWithRequestDetails(adapted_request_, alep);
1373 }
1374 
1376 {
1377  char ntoabuf[MAX_IPSTRLEN];
1378  /*
1379  * XXX These should use HttpHdr interfaces instead of Printfs
1380  */
1381  const Adaptation::ServiceConfig &s = service().cfg();
1382  buf.appendf("%s " SQUIDSTRINGPH " ICAP/1.0\r\n", s.methodStr(), SQUIDSTRINGPRINT(s.uri));
1383  buf.appendf("Host: " SQUIDSTRINGPH ":%d\r\n", SQUIDSTRINGPRINT(s.host), s.port);
1384  buf.appendf("Date: %s\r\n", Time::FormatRfc1123(squid_curtime));
1385 
1387  buf.appendf("Connection: close\r\n");
1388 
1389  const HttpRequest *request = &virginRequest();
1390 
1391  // we must forward "Proxy-Authenticate" and "Proxy-Authorization"
1392  // as ICAP headers.
1393  if (virgin.header->header.has(Http::HdrType::PROXY_AUTHENTICATE)) {
1394  String vh=virgin.header->header.getById(Http::HdrType::PROXY_AUTHENTICATE);
1395  buf.appendf("Proxy-Authenticate: " SQUIDSTRINGPH "\r\n",SQUIDSTRINGPRINT(vh));
1396  }
1397 
1398  if (virgin.header->header.has(Http::HdrType::PROXY_AUTHORIZATION)) {
1399  String vh=virgin.header->header.getById(Http::HdrType::PROXY_AUTHORIZATION);
1400  buf.appendf("Proxy-Authorization: " SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(vh));
1401  } else if (request->extacl_user.size() > 0 && request->extacl_passwd.size() > 0) {
1402  struct base64_encode_ctx ctx;
1403  base64_encode_init(&ctx);
1404  char base64buf[base64_encode_len(MAX_LOGIN_SZ)];
1405  size_t resultLen = base64_encode_update(&ctx, base64buf, request->extacl_user.size(), reinterpret_cast<const uint8_t*>(request->extacl_user.rawBuf()));
1406  resultLen += base64_encode_update(&ctx, base64buf+resultLen, 1, reinterpret_cast<const uint8_t*>(":"));
1407  resultLen += base64_encode_update(&ctx, base64buf+resultLen, request->extacl_passwd.size(), reinterpret_cast<const uint8_t*>(request->extacl_passwd.rawBuf()));
1408  resultLen += base64_encode_final(&ctx, base64buf+resultLen);
1409  buf.appendf("Proxy-Authorization: Basic %.*s\r\n", (int)resultLen, base64buf);
1410  }
1411 
1412  // share the cross-transactional database records if needed
1414  Adaptation::History::Pointer ah = request->adaptHistory(false);
1415  if (ah != nullptr) {
1416  String name, value;
1417  if (ah->getXxRecord(name, value)) {
1418  buf.appendf(SQUIDSTRINGPH ": " SQUIDSTRINGPH "\r\n", SQUIDSTRINGPRINT(name), SQUIDSTRINGPRINT(value));
1419  }
1420  }
1421  }
1422 
1423  buf.append("Encapsulated: ", 14);
1424 
1425  MemBuf httpBuf;
1426 
1427  httpBuf.init();
1428 
1429  // build HTTP request header, if any
1430  ICAP::Method m = s.method;
1431 
1432  // to simplify, we could assume that request is always available
1433 
1434  if (request) {
1435  if (ICAP::methodRespmod == m)
1436  encapsulateHead(buf, "req-hdr", httpBuf, request);
1437  else if (ICAP::methodReqmod == m)
1438  encapsulateHead(buf, "req-hdr", httpBuf, virgin.header);
1439  }
1440 
1441  if (ICAP::methodRespmod == m)
1442  if (const Http::Message *prime = virgin.header)
1443  encapsulateHead(buf, "res-hdr", httpBuf, prime);
1444 
1445  if (!virginBody.expected())
1446  buf.appendf("null-body=%d", (int) httpBuf.contentSize());
1447  else if (ICAP::methodReqmod == m)
1448  buf.appendf("req-body=%d", (int) httpBuf.contentSize());
1449  else
1450  buf.appendf("res-body=%d", (int) httpBuf.contentSize());
1451 
1452  buf.append(ICAP::crlf, 2); // terminate Encapsulated line
1453 
1454  if (preview.enabled()) {
1455  buf.appendf("Preview: %d\r\n", (int)preview.ad());
1456  if (!virginBody.expected()) // there is no body to preview
1457  finishNullOrEmptyBodyPreview(httpBuf);
1458  }
1459 
1460  makeAllowHeader(buf);
1461 
1462  if (TheConfig.send_client_ip && request) {
1463  Ip::Address client_addr;
1464 #if FOLLOW_X_FORWARDED_FOR
1466  client_addr = request->indirect_client_addr;
1467  } else
1468 #endif
1469  client_addr = request->client_addr;
1470  if (!client_addr.isAnyAddr() && !client_addr.isNoAddr())
1471  buf.appendf("X-Client-IP: %s\r\n", client_addr.toStr(ntoabuf,MAX_IPSTRLEN));
1472  }
1473 
1474  if (TheConfig.send_username && request)
1475  makeUsernameHeader(request, buf);
1476 
1477  // Adaptation::Config::metaHeaders
1478  for (const auto &h: Adaptation::Config::metaHeaders()) {
1479  HttpRequest *r = virgin.cause ?
1480  virgin.cause : dynamic_cast<HttpRequest*>(virgin.header);
1481  Must(r);
1482 
1483  HttpReply *reply = dynamic_cast<HttpReply*>(virgin.header);
1484 
1485  SBuf matched;
1486  if (h->match(r, reply, alMaster, matched)) {
1487  buf.append(h->key().rawContent(), h->key().length());
1488  buf.append(": ", 2);
1489  buf.append(matched.rawContent(), matched.length());
1490  buf.append("\r\n", 2);
1491  Adaptation::History::Pointer ah = request->adaptHistory(false);
1492  if (ah != nullptr) {
1493  if (ah->metaHeaders == nullptr)
1494  ah->metaHeaders = new NotePairs;
1495  if (!ah->metaHeaders->hasPair(h->key(), matched))
1496  ah->metaHeaders->add(h->key(), matched);
1497  }
1498  }
1499  }
1500 
1501  // fprintf(stderr, "%s\n", buf.content());
1502 
1503  buf.append(ICAP::crlf, 2); // terminate ICAP header
1504 
1505  // fill icapRequest for logging
1506  Must(icapRequest->parseCharBuf(buf.content(), buf.contentSize()));
1507 
1508  // start ICAP request body with encapsulated HTTP headers
1509  buf.append(httpBuf.content(), httpBuf.contentSize());
1510 
1511  httpBuf.clean();
1512 }
1513 
1514 // decides which Allow values to write and updates the request buffer
1516 {
1517  const bool allow204in = preview.enabled(); // TODO: add shouldAllow204in()
1518  const bool allow204out = state.allowedPostview204 = shouldAllow204();
1519  const bool allow206in = state.allowedPreview206 = shouldAllow206in();
1520  const bool allow206out = state.allowedPostview206 = shouldAllow206out();
1521  const bool allowTrailers = true; // TODO: make configurable
1522 
1523  debugs(93, 9, "Allows: " << allow204in << allow204out <<
1524  allow206in << allow206out << allowTrailers);
1525 
1526  const bool allow204 = allow204in || allow204out;
1527  const bool allow206 = allow206in || allow206out;
1528 
1529  if ((allow204 || allow206) && virginBody.expected())
1530  virginBodySending.plan(); // if there is a virgin body, plan to send it
1531 
1532  // writing Preview:... means we will honor 204 inside preview
1533  // writing Allow/204 means we will honor 204 outside preview
1534  // writing Allow:206 means we will honor 206 inside preview
1535  // writing Allow:204,206 means we will honor 206 outside preview
1536  if (allow204 || allow206 || allowTrailers) {
1537  buf.appendf("Allow: ");
1538  if (allow204out)
1539  buf.appendf("204, ");
1540  if (allow206)
1541  buf.appendf("206, ");
1542  if (allowTrailers)
1543  buf.appendf("trailers");
1544  buf.appendf("\r\n");
1545  }
1546 }
1547 
1549 {
1550 #if USE_AUTH
1551  struct base64_encode_ctx ctx;
1552  base64_encode_init(&ctx);
1553 
1554  const char *value = nullptr;
1555  if (request->auth_user_request != nullptr) {
1556  value = request->auth_user_request->username();
1557  } else if (request->extacl_user.size() > 0) {
1558  value = request->extacl_user.termedBuf();
1559  }
1560 
1561  if (value) {
1563  char base64buf[base64_encode_len(MAX_LOGIN_SZ)];
1564  size_t resultLen = base64_encode_update(&ctx, base64buf, strlen(value), reinterpret_cast<const uint8_t*>(value));
1565  resultLen += base64_encode_final(&ctx, base64buf+resultLen);
1566  buf.appendf("%s: %.*s\r\n", TheConfig.client_username_header, (int)resultLen, base64buf);
1567  } else
1568  buf.appendf("%s: %s\r\n", TheConfig.client_username_header, value);
1569  }
1570 #else
1571  (void)request;
1572  (void)buf;
1573 #endif
1574 }
1575 
1576 void
1577 Adaptation::Icap::ModXact::encapsulateHead(MemBuf &icapBuf, const char *section, MemBuf &httpBuf, const Http::Message *head)
1578 {
1579  // update ICAP header
1580  icapBuf.appendf("%s=%d, ", section, (int) httpBuf.contentSize());
1581 
1582  // begin cloning
1583  Http::MessagePointer headClone;
1584 
1585  if (const HttpRequest* old_request = dynamic_cast<const HttpRequest*>(head)) {
1586  HttpRequest::Pointer new_request(new HttpRequest(old_request->masterXaction));
1587  // copy the request-line details
1588  new_request->method = old_request->method;
1589  new_request->url = old_request->url;
1590  new_request->http_ver = old_request->http_ver;
1591  headClone = new_request.getRaw();
1592  } else if (const HttpReply *old_reply = dynamic_cast<const HttpReply*>(head)) {
1593  HttpReply::Pointer new_reply(new HttpReply);
1594  new_reply->sline = old_reply->sline;
1595  headClone = new_reply.getRaw();
1596  }
1597  Must(headClone);
1598  headClone->inheritProperties(head);
1599 
1601  while (HttpHeaderEntry* p_head_entry = head->header.getEntry(&pos))
1602  headClone->header.addEntry(p_head_entry->clone());
1603 
1604  // end cloning
1605 
1606  // remove all hop-by-hop headers from the clone
1608  headClone->header.removeHopByHopEntries();
1609 
1610  // TODO: modify HttpHeader::removeHopByHopEntries to accept a list of
1611  // excluded hop-by-hop headers
1612  if (head->header.has(Http::HdrType::UPGRADE)) {
1613  const auto upgrade = head->header.getList(Http::HdrType::UPGRADE);
1614  headClone->header.putStr(Http::HdrType::UPGRADE, upgrade.termedBuf());
1615  }
1616 
1617  // pack polished HTTP header
1618  packHead(httpBuf, headClone.getRaw());
1619 
1620  // headClone unlocks and, hence, deletes the message we packed
1621 }
1622 
1623 void
1625 {
1626  head->packInto(&httpBuf, true);
1627 }
1628 
1629 // decides whether to offer a preview and calculates its size
1631 {
1632  if (!TheConfig.preview_enable) {
1633  debugs(93, 5, "preview disabled by squid.conf");
1634  return;
1635  }
1636 
1637  const SBuf urlPath(virginRequest().url.path());
1638  size_t wantedSize;
1639  if (!service().wantsPreview(urlPath, wantedSize)) {
1640  debugs(93, 5, "should not offer preview for " << urlPath);
1641  return;
1642  }
1643 
1644  // we decided to do preview, now compute its size
1645 
1646  // cannot preview more than we can backup
1647  size_t ad = min(wantedSize, TheBackupLimit);
1648 
1649  if (!virginBody.expected())
1650  ad = 0;
1651  else if (virginBody.knownSize())
1652  ad = min(static_cast<uint64_t>(ad), virginBody.size()); // not more than we have
1653 
1654  debugs(93, 5, "should offer " << ad << "-byte preview " <<
1655  "(service wanted " << wantedSize << ")");
1656 
1657  preview.enable(ad);
1658  Must(preview.enabled());
1659 }
1660 
1661 // decides whether to allow 204 responses
1663 {
1664  if (!service().allows204())
1665  return false;
1666 
1667  return canBackupEverything();
1668 }
1669 
1670 // decides whether to allow 206 responses in some mode
1672 {
1673  return TheConfig.allow206_enable && service().allows206() &&
1674  virginBody.expected(); // no need for 206 without a body
1675 }
1676 
1677 // decides whether to allow 206 responses in preview mode
1679 {
1680  return shouldAllow206any() && preview.enabled();
1681 }
1682 
1683 // decides whether to allow 206 responses outside of preview
1685 {
1686  return shouldAllow206any() && canBackupEverything();
1687 }
1688 
1689 // used by shouldAllow204 and decideOnRetries
1691 {
1692  if (!virginBody.expected())
1693  return true; // no body means no problems with backup
1694 
1695  // if there is a body, check whether we can backup it all
1696 
1697  if (!virginBody.knownSize())
1698  return false;
1699 
1700  // or should we have a different backup limit?
1701  // note that '<' allows for 0-termination of the "full" backup buffer
1702  return virginBody.size() < TheBackupLimit;
1703 }
1704 
1705 // Decide whether this transaction can be retried if pconn fails
1706 // Must be called after decideOnPreview and before openConnection()
1708 {
1709  if (!isRetriable)
1710  return; // no, already decided
1711 
1712  if (preview.enabled())
1713  return; // yes, because preview provides enough guarantees
1714 
1715  if (canBackupEverything())
1716  return; // yes, because we can back everything up
1717 
1718  disableRetries(); // no, because we cannot back everything up
1719 }
1720 
1721 // Normally, the body-writing code handles preview body. It can deal with
1722 // bodies of unexpected size, including those that turn out to be empty.
1723 // However, that code assumes that the body was expected and body control
1724 // structures were initialized. This is not the case when there is no body
1725 // or the body is known to be empty, because the virgin message will lack a
1726 // body_pipe. So we handle preview of null-body and zero-size bodies here.
1728 {
1729  Must(!virginBodyWriting.active()); // one reason we handle it here
1730  Must(!virgin.body_pipe); // another reason we handle it here
1731  Must(!preview.ad());
1732 
1733  // do not add last-chunk because our Encapsulated header says null-body
1734  // addLastRequestChunk(buf);
1735  preview.wrote(0, true);
1736 
1737  Must(preview.done());
1738  Must(preview.ieof());
1739 }
1740 
1742 {
1744 
1745  if (state.serviceWaiting)
1746  buf.append("U", 1);
1747 
1748  if (virgin.body_pipe != nullptr)
1749  buf.append("R", 1);
1750 
1751  if (haveConnection() && !doneReading())
1752  buf.append("r", 1);
1753 
1754  if (!state.doneWriting() && state.writing != State::writingInit)
1755  buf.appendf("w(%d)", state.writing);
1756 
1757  if (preview.enabled()) {
1758  if (!preview.done())
1759  buf.appendf("P(%d)", (int) preview.debt());
1760  }
1761 
1762  if (virginBodySending.active())
1763  buf.append("B", 1);
1764 
1765  if (!state.doneParsing() && state.parsing != State::psIcapHeader)
1766  buf.appendf("p(%d)", state.parsing);
1767 
1768  if (!doneSending() && state.sending != State::sendingUndecided)
1769  buf.appendf("S(%d)", state.sending);
1770 
1771  if (state.readyForUob)
1772  buf.append("6", 1);
1773 
1774  if (canStartBypass)
1775  buf.append("Y", 1);
1776 
1777  if (protectGroupBypass)
1778  buf.append("G", 1);
1779 }
1780 
1782 {
1784 
1785  if (!virgin.body_pipe)
1786  buf.append("R", 1);
1787 
1788  if (state.doneWriting())
1789  buf.append("w", 1);
1790 
1791  if (preview.enabled()) {
1792  if (preview.done())
1793  buf.appendf("P%s", preview.ieof() ? "(ieof)" : "");
1794  }
1795 
1796  if (doneReading())
1797  buf.append("r", 1);
1798 
1799  if (state.doneParsing())
1800  buf.append("p", 1);
1801 
1802  if (doneSending())
1803  buf.append("S", 1);
1804 }
1805 
1806 bool Adaptation::Icap::ModXact::gotEncapsulated(const char *section) const
1807 {
1808  return !icapReply->header.getByNameListMember("Encapsulated",
1809  section, ',').isEmpty();
1810 }
1811 
1812 // calculate whether there is a virgin HTTP body and
1813 // whether its expected size is known
1814 // TODO: rename because we do not just estimate
1816 {
1817  // note: lack of size info may disable previews and 204s
1818 
1819  Http::Message *msg = virgin.header;
1820  Must(msg);
1821 
1822  HttpRequestMethod method;
1823 
1824  if (virgin.cause)
1825  method = virgin.cause->method;
1826  else if (HttpRequest *req = dynamic_cast<HttpRequest*>(msg))
1827  method = req->method;
1828  else
1829  method = Http::METHOD_NONE;
1830 
1831  int64_t size;
1832  // expectingBody returns true for zero-sized bodies, but we will not
1833  // get a pipe for that body, so we treat the message as bodyless
1834  if (method != Http::METHOD_NONE && msg->expectingBody(method, size) && size) {
1835  debugs(93, 6, "expects virgin body from " <<
1836  virgin.body_pipe << "; size: " << size);
1837 
1838  virginBody.expect(size);
1839  virginBodyWriting.plan();
1840 
1841  // sign up as a body consumer
1842  Must(msg->body_pipe != nullptr);
1843  Must(msg->body_pipe == virgin.body_pipe);
1844  Must(virgin.body_pipe->setConsumerIfNotLate(this));
1845 
1846  // make sure TheBackupLimit is in-sync with the buffer size
1847  Must(TheBackupLimit <= static_cast<size_t>(msg->body_pipe->buf().max_capacity));
1848  } else {
1849  debugs(93, 6, "does not expect virgin body");
1850  Must(msg->body_pipe == nullptr);
1851  checkConsuming();
1852  }
1853 }
1854 
1856 {
1857  Must(!adapted.body_pipe);
1858  Must(!adapted.header->body_pipe);
1859  adapted.header->body_pipe = new BodyPipe(this);
1860  adapted.body_pipe = adapted.header->body_pipe;
1861  debugs(93, 7, "will supply " << what << " via " <<
1862  adapted.body_pipe << " pipe");
1863 }
1864 
1865 // TODO: Move SizedEstimate and Preview elsewhere
1866 
1868  : theData(dtUnexpected)
1869 {}
1870 
1872 {
1873  theData = (aSize >= 0) ? aSize : (int64_t)dtUnknown;
1874 }
1875 
1877 {
1878  return theData != dtUnexpected;
1879 }
1880 
1882 {
1883  Must(expected());
1884  return theData != dtUnknown;
1885 }
1886 
1888 {
1889  Must(knownSize());
1890  return static_cast<uint64_t>(theData);
1891 }
1892 
1893 Adaptation::Icap::VirginBodyAct::VirginBodyAct(): theStart(0), theState(stUndecided)
1894 {}
1895 
1897 {
1898  Must(!disabled());
1899  Must(!theStart); // not started
1900  theState = stActive;
1901 }
1902 
1904 {
1905  theState = stDisabled;
1906 }
1907 
1909 {
1910  Must(active());
1911 #if SIZEOF_SIZE_T > 4
1912  /* always true for smaller size_t's */
1913  Must(static_cast<int64_t>(size) >= 0);
1914 #endif
1915  theStart += static_cast<int64_t>(size);
1916 }
1917 
1919 {
1920  Must(active());
1921  return static_cast<uint64_t>(theStart);
1922 }
1923 
1924 Adaptation::Icap::Preview::Preview(): theWritten(0), theAd(0), theState(stDisabled)
1925 {}
1926 
1928 {
1929  // TODO: check for anAd not exceeding preview size limit
1930  Must(!enabled());
1931  theAd = anAd;
1932  theState = stWriting;
1933 }
1934 
1936 {
1937  return theState != stDisabled;
1938 }
1939 
1941 {
1942  Must(enabled());
1943  return theAd;
1944 }
1945 
1947 {
1948  Must(enabled());
1949  return theState >= stIeof;
1950 }
1951 
1953 {
1954  Must(enabled());
1955  return theState == stIeof;
1956 }
1957 
1959 {
1960  Must(enabled());
1961  return done() ? 0 : (theAd - theWritten);
1962 }
1963 
1964 void Adaptation::Icap::Preview::wrote(size_t size, bool wroteEof)
1965 {
1966  Must(enabled());
1967 
1968  theWritten += size;
1969 
1970  Must(theWritten <= theAd);
1971 
1972  if (wroteEof)
1973  theState = stIeof; // written size is irrelevant
1974  else if (theWritten >= theAd)
1975  theState = stDone;
1976 }
1977 
1979 {
1980  if (virgin.header == nullptr)
1981  return false;
1982 
1983  virgin.header->firstLineBuf(mb);
1984 
1985  return true;
1986 }
1987 
1989 {
1990  HttpRequest *request = dynamic_cast<HttpRequest*>(adapted.header);
1991  // if no adapted request, update virgin (and inherit its properties later)
1992  // TODO: make this and HttpRequest::detailError constant, like adaptHistory
1993  if (!request)
1994  request = const_cast<HttpRequest*>(&virginRequest());
1995 
1996  if (request)
1997  request->detailError(ERR_ICAP_FAILURE, errDetail);
1998 }
1999 
2001 {
2002  HttpRequest *request = dynamic_cast<HttpRequest*>(adapted.header);
2003  // if no adapted request, update virgin (and inherit its properties later)
2004  if (!request)
2005  request = const_cast<HttpRequest*>(&virginRequest());
2006 
2007  if (request)
2008  request->clearError();
2009 }
2010 
2012 {
2013  Must(adapted.header);
2014  adapted.header->sources |= (service().cfg().connectionEncryption ? Http::Message::srcIcaps : Http::Message::srcIcap);
2015 }
2016 
2017 /* Adaptation::Icap::ModXactLauncher */
2018 
2020  AsyncJob("Adaptation::Icap::ModXactLauncher"),
2021  Adaptation::Icap::Launcher("Adaptation::Icap::ModXactLauncher", aService),
2022  al(alp)
2023 {
2024  virgin.setHeader(virginHeader);
2025  virgin.setCause(virginCause);
2026  updateHistory(true);
2027 }
2028 
2030 {
2032  dynamic_cast<Adaptation::Icap::ServiceRep*>(theService.getRaw());
2033  Must(s != nullptr);
2034  return new Adaptation::Icap::ModXact(virgin.header, virgin.cause, al, s);
2035 }
2036 
2038 {
2039  debugs(93, 5, "swan sings");
2040  updateHistory(false);
2042 }
2043 
2045 {
2046  HttpRequest *r = virgin.cause ?
2047  virgin.cause : dynamic_cast<HttpRequest*>(virgin.header);
2048 
2049  // r should never be NULL but we play safe; TODO: add Should()
2050  if (r) {
2052  if (h != nullptr) {
2053  if (doStart)
2054  h->start("ICAPModXactLauncher");
2055  else
2056  h->stop("ICAPModXactLauncher");
2057  }
2058  }
2059 }
2060 
2061 bool Adaptation::Icap::TrailerParser::parse(const char *buf, int len, int atEnd, Http::StatusCode *error) {
2063  // RFC 7230 section 4.1.2: MUST NOT generate a trailer that contains
2064  // a field necessary for message framing (e.g., Transfer-Encoding and Content-Length)
2065  clen.applyTrailerRules();
2066  const int parsed = trailer.parse(buf, len, atEnd, hdr_sz, clen);
2067  if (parsed < 0)
2068  *error = Http::scInvalidHeader; // TODO: should we add a new Http::scInvalidTrailer?
2069  return parsed > 0;
2070 }
2071 
2072 void
2074 {
2075  if (extName == UseOriginalBodyName) {
2076  useOriginalBody_ = tok.udec64("use-original-body");
2077  assert(useOriginalBody_ >= 0);
2078  } else {
2079  Ignore(tok, extName);
2080  }
2081 }
2082 
void disableBypass(const char *reason, bool includeGroupBypass)
Definition: ModXact.cc:713
const MemBuf & buf() const
Definition: BodyPipe.h:137
void callException(const std::exception &e) override
called when the job throws during an async call
Definition: ModXact.cc:661
int hdr_sz
Definition: Message.h:81
void stop(const char *context)
note the end of an ICAP processing interval
Definition: History.cc:32
void prepPartialBodyEchoing(uint64_t pos)
Definition: ModXact.cc:1018
String content_type
Definition: HttpReply.h:46
SourceLocationId id() const
same-location exceptions have the same ID
Definition: TextException.h:40
#define Here()
source code location of the caller
Definition: Here.h:15
AccessLogEntry::Pointer alMaster
Master transaction AccessLogEntry.
Definition: ModXact.h:373
ModXactLauncher(Http::Message *virginHeader, HttpRequest *virginCause, AccessLogEntry::Pointer &alp, Adaptation::ServicePointer s)
Definition: ModXact.cc:2019
ModXact(Http::Message *virginHeader, HttpRequest *virginCause, AccessLogEntry::Pointer &alp, ServiceRep::Pointer &s)
Definition: ModXact.cc:54
void terminate()
Definition: MemBuf.cc:241
AnyP::ProtocolVersion http_ver
Definition: Message.h:72
#define base64_encode_len(length)
Definition: base64.h:169
void appendf(const char *fmt,...) PRINTF_FORMAT_ARG2
Append operation with printf-style arguments.
Definition: Packable.h:61
@ srvBlock
Definition: Elements.h:19
void openChunk(MemBuf &buf, size_t chunkSize, bool ieof)
Definition: ModXact.cc:376
const char * rawBuf() const
Definition: SquidString.h:86
void applyTrailerRules()
prohibits Content-Length in GET/HEAD requests
AnyP::Uri url
the request URI
Definition: HttpRequest.h:115
common parts of HttpRequest and HttpReply
Definition: Message.h:25
void startShoveling() override
starts sending/receiving ICAP messages
Definition: ModXact.cc:189
void swanSong() override
Definition: ModXact.cc:1289
void progress(size_t size)
Definition: ModXact.cc:1908
BodyPipe::Pointer body_pipe
optional pipeline to receive message body
Definition: Message.h:97
void enable(size_t anAd)
Definition: ModXact.cc:1927
size_t virginContentSize(const VirginBodyAct &act) const
Definition: ModXact.cc:404
virtual bool expectingBody(const HttpRequestMethod &, int64_t &) const =0
void makeRequestHeaders(MemBuf &buf)
Definition: ModXact.cc:1375
virtual void fillDoneStatus(MemBuf &buf) const
Definition: Xaction.cc:667
Config TheConfig
Definition: Config.cc:19
const XactOutcome xoPartEcho
preserved virgin msg part (ICAP 206)
Definition: Elements.cc:24
void packHead(MemBuf &httpBuf, const Http::Message *head)
Definition: ModXact.cc:1624
@ scNone
Definition: StatusCode.h:21
void removeHopByHopEntries()
Definition: HttpHeader.cc:1710
HttpHeader header
Definition: Message.h:74
Http1::TeChunkedParser * bodyParser
Definition: ModXact.h:303
bool virginBodyEndReached(const VirginBodyAct &act) const
Definition: ModXact.cc:395
void noteMoreBodyDataAvailable(BodyPipe::Pointer) override
Definition: ModXact.cc:1229
Parses and stores ICAP trailer header block.
Definition: ModXact.h:110
ssize_t HttpHeaderPos
Definition: HttpHeader.h:45
bool doneAll() const override
whether positive goal has been reached
Definition: ModXact.cc:524
void clearError()
clear error details, useful for retries/repeats
Definition: HttpRequest.cc:464
static int send_client_ip
Definition: Config.h:47
@ srcIcaps
Secure ICAP service.
Definition: Message.h:35
char * client_username_header
Definition: Config.h:36
void updateXxRecord(const char *name, const String &value)
sets or resets a cross-transactional database record
Definition: History.cc:105
size_t base64_encode_final(struct base64_encode_ctx *ctx, char *dst)
Definition: base64.c:308
bool isAnyAddr() const
Definition: Address.cc:190
void updateNextServices(const String &services)
sets or resets next services for the Adaptation::Iterator to notice
Definition: History.cc:121
CBDATA_NAMESPACED_CLASS_INIT(Adaptation::Icap, ModXact)
#define HttpHeaderInitPos
Definition: HttpHeader.h:48
void error(char *format,...)
bool hasContent() const
Definition: MemBuf.h:54
#define SQUIDSTRINGPRINT(s)
Definition: SquidString.h:22
void init(mb_size_t szInit, mb_size_t szMax)
Definition: MemBuf.cc:93
Definition: SBuf.h:93
void setHeader(Header *h)
Definition: InOut.h:48
static constexpr auto TheBackupLimit
Definition: ModXact.cc:45
virtual void fillPendingStatus(MemBuf &buf) const
Definition: Xaction.cc:649
void stopParsing(const bool checkUnparsedData=true)
Definition: ModXact.cc:1209
void swanSong() override
Definition: Launcher.cc:105
const XactOutcome xoEcho
preserved virgin message (ICAP 204)
Definition: Elements.cc:23
static int send_username
Definition: Config.h:48
const A & max(A const &lhs, A const &rhs)
void closeChunk(MemBuf &buf)
Definition: ModXact.cc:381
static int use_indirect_client
Definition: Config.h:49
Auth::UserRequest::Pointer auth_user_request
Definition: HttpRequest.h:127
C * getRaw() const
Definition: RefCount.h:89
String log_uri
the request uri
Definition: History.h:44
TrailerParser * trailerParser
Definition: ModXact.h:321
void handleCommWrote(size_t size) override
Definition: ModXact.cc:208
Http::StatusLine sline
Definition: HttpReply.h:56
void makeAllowHeader(MemBuf &buf)
Definition: ModXact.cc:1515
bool getXxRecord(String &name, String &value) const
returns true and fills the record fields iff there is a db record
Definition: History.cc:111
bool parsePart(Part *part, const char *description)
Definition: ModXact.cc:1087
StatusCode
Definition: StatusCode.h:20
#define SQUIDSTRINGPH
Definition: SquidString.h:21
void detailError(const err_type c, const ErrorDetail::Pointer &d)
sets error detail if no earlier detail was available
Definition: HttpRequest.h:101
void start() override
called by AsyncStart; do not call directly
Definition: ModXact.cc:88
void fillDoneStatus(MemBuf &buf) const override
Definition: ModXact.cc:1781
String extacl_user
Definition: HttpRequest.h:176
int const char size_t
Definition: stub_liblog.cc:83
void addLastRequestChunk(MemBuf &buf)
Definition: ModXact.cc:369
char * toStr(char *buf, const unsigned int blen, int force=AF_UNSPEC) const
Definition: Address.cc:812
void encapsulateHead(MemBuf &icapBuf, const char *section, MemBuf &httpBuf, const Http::Message *head)
Definition: ModXact.cc:1577
LogTags logType
the squid request status (TCP_MISS etc)
Definition: History.h:42
#define TexcHere(msg)
legacy convenience macro; it is not difficult to type Here() now
Definition: TextException.h:63
virtual bool inheritProperties(const Http::Message *)=0
void consume(size_t size)
Definition: BodyPipe.cc:309
void stopSending(bool nicely)
Definition: ModXact.cc:610
mb_size_t max_capacity
Definition: MemBuf.h:142
#define MAX_IPSTRLEN
Length of buffer that needs to be allocated to old a null-terminated IP-string.
Definition: forward.h:25
void recordXactFinish(int hid)
record the end of a xact identified by its history ID
Definition: History.cc:61
bool bodySizeKnown() const
Definition: BodyPipe.h:109
MemBuf & buf
Definition: BodyPipe.h:74
const char * FormatRfc1123(time_t)
Definition: rfc1123.cc:202
void wrote(size_t size, bool wroteEof)
Definition: ModXact.cc:1964
void handleCommRead(size_t size) override
Definition: ModXact.cc:563
bool doneSending() const
Definition: ModXact.cc:604
@ ERR_ICAP_FAILURE
Definition: forward.h:64
void makeAdaptedBodyPipe(const char *what)
Definition: ModXact.cc:1855
void detailError(const ErrorDetail::Pointer &errDetail) override
record error detail in the virgin request if possible
Definition: ModXact.cc:1988
@ ICP_INVALID
Definition: icp_opcode.h:15
bool httpHeaderHasConnDir(const HttpHeader *hdr, const SBuf &directive)
mb_size_t contentSize() const
available data size
Definition: MemBuf.h:47
uint64_t bodySize() const
Definition: BodyPipe.cc:161
int size
Definition: ModDevPoll.cc:69
struct timeval current_time
the current UNIX time in timeval {seconds, microseconds} format
Definition: gadgets.cc:18
void append(const char *c, int sz) override
Definition: MemBuf.cc:209
void parseExtensionValuesWith(ChunkExtensionValueParser *parser)
const char * rawContent() const
Definition: SBuf.cc:509
static Notes & metaHeaders()
The list of configured meta headers.
Definition: Config.cc:35
void stopWriting(bool nicely)
Definition: ModXact.cc:482
Http::StatusCode status() const
retrieve the status code for this status line
Definition: StatusLine.h:45
void parse(Tokenizer &tok, const SBuf &extName) override
Definition: ModXact.cc:2073
#define MAX_LOGIN_SZ
Definition: defines.h:77
void finalizeLogInfo() override
Definition: ModXact.cc:1311
uint64_t producedSize() const
Definition: BodyPipe.h:112
Ip::Address indirect_client_addr
Definition: HttpRequest.h:152
const char * username() const
Definition: UserRequest.cc:32
void start() override
called by AsyncStart; do not call directly
Definition: Xaction.cc:130
bool expectHttpHeader() const
whether ICAP response header indicates HTTP header presence
Definition: ModXact.cc:1114
Definition: MemBuf.h:23
bool expectHttpBody() const
whether ICAP response header indicates HTTP body presence
Definition: ModXact.cc:1119
const char * status() const override
internal cleanup; do not call directly
Definition: Xaction.cc:635
void clean()
Definition: MemBuf.cc:110
void swanSong() override
Definition: Xaction.cc:573
void clearError() override
clear stored error details, if any; used for retries/repeats
Definition: ModXact.cc:2000
void fillPendingStatus(MemBuf &buf) const override
Definition: ModXact.cc:1741
@ scPartialContent
Definition: StatusCode.h:33
int delById(Http::HdrType id)
Definition: HttpHeader.cc:666
@ srcIcap
traditional ICAP service without encryption
Definition: Message.h:41
Adaptation::Icap::History::Pointer icapHistory() const
Returns possibly nil history, creating it if icap logging is enabled.
Definition: HttpRequest.cc:388
@ scCreated
Definition: StatusCode.h:28
void addEntry(HttpHeaderEntry *e)
Definition: HttpHeader.cc:736
void updateSources()
Update the Http::Message sources.
Definition: ModXact.cc:2011
#define assert(EX)
Definition: assert.h:17
void writeSomeBody(const char *label, size_t size)
Definition: ModXact.cc:319
@ scContinue
Definition: StatusCode.h:22
bool fillVirginHttpHeader(MemBuf &) const override
Definition: ModXact.cc:1978
void HTTPMSGLOCK(Http::Message *a)
Definition: Message.h:161
NotePairs::Pointer metaHeaders
Definition: History.h:66
void noteBodyProducerAborted(BodyPipe::Pointer) override
Definition: ModXact.cc:1251
#define JobCallback(dbgSection, dbgLevel, Dialer, job, method)
Convenience macro to create a Dialer-based job callback.
Definition: AsyncJobCalls.h:70
void prepareLogWithRequestDetails(HttpRequest *, const AccessLogEntryPointer &)
Definition: client_side.cc:322
const char * virginContentData(const VirginBodyAct &act) const
Definition: ModXact.cc:416
void base64_encode_init(struct base64_encode_ctx *ctx)
Definition: base64.c:232
size_type length() const
Returns the number of bytes stored in SBuf.
Definition: SBuf.h:419
time_t squid_curtime
Definition: stub_libtime.cc:20
bool isNoAddr() const
Definition: Address.cc:304
void setCause(HttpRequest *r)
Definition: InOut.h:38
Adaptation::History::Pointer adaptHistory(bool createIfNone=false) const
Returns possibly nil history, creating it if requested.
Definition: HttpRequest.cc:403
Adaptation::History::Pointer adaptLogHistory() const
Returns possibly nil history, creating it if adapt. logging is enabled.
Definition: HttpRequest.cc:414
void callException(const std::exception &e) override
called when the job throws during an async call
Definition: Xaction.cc:372
bool canBackupEverything() const
Definition: ModXact.cc:1690
bool expectIcapTrailers() const
whether ICAP response header indicates ICAP trailers presence
Definition: ModXact.cc:1124
static Answer Forward(Http::Message *aMsg)
create an akForward answer
Definition: Answer.cc:26
virtual void finalizeLogInfo()
Definition: Xaction.cc:612
const HttpRequest & virginRequest() const
locates the request, either as a cause or as a virgin message itself
Definition: ModXact.cc:386
HttpRequestMethod method
Definition: HttpRequest.h:114
Xaction * createXaction() override
Definition: ModXact.cc:2029
const char * termedBuf() const
Definition: SquidString.h:92
void start(const char *context)
record the start of an ICAP processing interval
Definition: History.cc:23
@ methodRespmod
Definition: Elements.h:17
String protoPrefix
Definition: HttpReply.h:60
const char * status() const
Definition: BodyPipe.cc:446
void decideWritingAfterPreview(const char *previewKind)
determine state.writing after we wrote the entire preview
Definition: ModXact.cc:292
Definition: parse.c:160
bool hasPair(const SBuf &key, const SBuf &value) const
Definition: Notes.cc:370
static constexpr size_t MaxCapacity
Definition: BodyPipe.h:100
squidaio_request_t * head
Definition: aiops.cc:127
mb_size_t potentialSpaceSize() const
Definition: MemBuf.cc:161
@ scNoContent
Definition: StatusCode.h:31
@ PROXY_AUTHENTICATE
an std::runtime_error with thrower location info
Definition: TextException.h:20
char * content()
start of the added data
Definition: MemBuf.h:41
void noteMoreBodySpaceAvailable(BodyPipe::Pointer) override
Definition: ModXact.cc:1264
void putStr(Http::HdrType id, const char *str)
Definition: HttpHeader.cc:995
size_type size() const
Definition: SquidString.h:73
void noteBodyConsumerAborted(BodyPipe::Pointer) override
Definition: ModXact.cc:1275
@ METHOD_NONE
Definition: MethodType.h:22
#define Must(condition)
Definition: TextException.h:75
const char * methodStr() const
#define DBG_IMPORTANT
Definition: Stream.h:38
size_t base64_encode_update(struct base64_encode_ctx *ctx, char *dst, size_t length, const uint8_t *src)
Definition: base64.c:265
bool gotEncapsulated(const char *section) const
Definition: ModXact.cc:1806
bool doneAll() const override
whether positive goal has been reached
Definition: Xaction.cc:388
void add(const SBuf &key, const SBuf &value)
Definition: Notes.cc:317
@ methodReqmod
Definition: Elements.h:17
static char * masterx_shared_name
Definition: Config.h:45
@ scInvalidHeader
Squid header parsing error.
Definition: StatusCode.h:88
String ssluser
the username from SSL
Definition: History.h:40
const XactOutcome xoSatisfied
request satisfaction
Definition: Elements.cc:26
void finishNullOrEmptyBodyPreview(MemBuf &buf)
Definition: ModXact.cc:1727
void expect(int64_t aSize)
Definition: ModXact.cc:1871
@ scOkay
Definition: StatusCode.h:27
const ServiceConfig & cfg() const
Definition: Service.h:51
HttpReply::Pointer icapReply
received ICAP reply, if any
Definition: Xaction.h:64
void recordMeta(const HttpHeader *lm)
store the last meta header fields received from the adaptation service
Definition: History.cc:140
int recordXactStart(const String &serviceId, const timeval &when, bool retrying)
record the start of a xact, return xact history ID
Definition: History.cc:51
bool parse(const char *buf, int len, int atEnd, Http::StatusCode *error)
Definition: ModXact.cc:2061
bool parseHead(Http::Message *head)
Definition: ModXact.cc:1105
void noteBodyProductionEnded(BodyPipe::Pointer) override
Definition: ModXact.cc:1238
const XactOutcome xoModified
replaced virgin msg with adapted
Definition: Elements.cc:25
int adaptHistoryId
adaptation history slot reservation
Definition: ModXact.h:319
Ip::Address client_addr
Definition: HttpRequest.h:149
void makeUsernameHeader(const HttpRequest *request, MemBuf &buf)
Definition: ModXact.cc:1548
#define debugs(SECTION, LEVEL, CONTENT)
Definition: Stream.h:192
const A & min(A const &lhs, A const &rhs)
String extacl_passwd
Definition: HttpRequest.h:178
size_t debt() const
Definition: ModXact.cc:1958
void updateHistory(bool start)
starts or stops transaction accounting in ICAP history
Definition: ModXact.cc:2044
const CharacterSet crlf("crlf","\r\n")
Definition: Elements.cc:12
ErrorDetail::Pointer MakeNamedErrorDetail(const char *name)
Definition: Detail.cc:54
@ PROXY_AUTHORIZATION

 

Introduction

Documentation

Support

Miscellaneous