Re: [squid-users] Squid 3.3 Reverse Proxy Mode - 502 Errors when uploading files larger than 6MB

Interesting-- somewhat similar problem here , small file uploads work
larger files do not.

 I am running squid 3.3.9 myself on a centos6 VM running reverse proxy
on 443

One note , i am using accel in my https_port .. not vhost
And I am not seeing any 502 status return in my logs.. i just get a 401

In my case both with OWA and Sharepoint IIS backend servers running
on Server 2008 Enterprise and RC2
I can not upload any more than a few kilobytes of file, more than 60 KB
seems to fail immediately, with a page that seems to indicate a TCP
reset. I have not yet turned on any debug in Squid , but direct to
server uploads work fine.

My squid conf looks somewhat similar to your own

############### SQUID CONFiguration ###############

# listen on port 443 and use the listed certificate and key

https_port 443 cert=/etc/squid/conf.d/443/STAR-sprymethods-cert.pem
key=/etc/squid/conf.d/443/STAR-mycompnaydomain-server.key accel

##################

# TAG: request_body_max_size (KB)
# This specifies the maximum size for an HTTP request body.
# In other words, the maximum size of a PUT/POST request.
# A user who attempts to send a request with a body larger
# than this limit receives an "Invalid Request" error message.
# If you set this parameter to a zero (the default), there will
# be no limit imposed.
#

request_body_max_size 0 KB

chunked_request_body_max_size 0 KB

########################################################################
# Note: default configuration gives you an open proxy-server
# if you want to restrict proxy requests to only known
# domains you must uncomment the following three lines
########################################################################

acl trusted_domains dstdomain
www.mycompanydomain.com .comodoca.com .google.com .microsoft.com .mycompanyADdomain.net
http_access allow trusted_domains

acl bad_requests urlpath_regex -i cmd.exe \/bin\/sh \/bin\/bash
http_access deny bad_requests

acl port80 myportname 80

acl site1 dstdomain mgw.mycompanydomain.com
http_access deny port80 site1
deny_info https://mgw.mycompanydomain.com/owa site1

acl site2 dstdomain portal.mycompanydomain.com
http_access deny port80 site2
deny_info https://portal.mycompanydomain.com/ site2

acl site3 dstdomain webmail.mycompanydomain.com
http_access deny port80 site3
deny_info https://webmail.mycompanydomain.com/owa/ site3

###################
# First HTTPS peer

cache_peer 192.168.17.10 parent 80 0 no-query no-digest proxy-only
originserver forceddomain=seaport.mycompanydomain.com front-end-https=on
name=seaport

acl sites_server_1 dstdomain seaport.mycompanydomain.com
cache_peer_access seaport allow sites_server_1
http_access allow sites_server_1

###################

# Second HTTPS peer
cache_peer 192.168.17.10 parent 443 0 no-query no-digest proxy-only
originserver ssl sslflags=DONT_VERIFY_PEER name=sprytime

acl sites_server_2 dstdomain sprytime.mycompanydomain.com
cache_peer_access sprytime allow sites_server_2
http_access allow sites_server_2

###################
# Third HTTPS peer

cache_peer 192.168.17.24 parent 443 0 no-query no-digest proxy-only
originserver forceddomain=portal.mycompanydomain.com front-end-https=on
connection-auth=on login=PASS ssl sslflags=DONT_VERIFY_PEER
sslflags=DONT_VERIFY_DOMAIN name=SMportal

acl sites_server_3 dstdomain portal.mycompanydomain.com
cache_peer_access SMportal allow sites_server_3
acl TrustedNamePortal url_regex ^https://portal.mycompanydomain.com/
sslproxy_cert_error allow TrustedNamePortal
http_access allow sites_server_3

####################
# Need RPC for OWA webmail and MGW outlook and android clients
# need for for MS ActiveSync over OWA
extension_methods RPC_IN_DATA RPC_OUT_DATA

###################
# Fourth HTTPS peer

cache_peer 192.168.17.18 parent 443 0 no-query no-digest proxy-only
originserver login=PASS ssl sslflags=DONT_VERIFY_PEER front-end-https=on
name=mgw

acl sites_server_4 dstdomain mgw.mycompanydomain.com
cache_peer_access mgw allow sites_server_4
http_access allow sites_server_4

###################
# Fifth HTTPS peer (Exchange 2010 OWA)

cache_peer 192.168.17.18 parent 443 0 no-query no-digest proxy-only
originserver login=PASS ssl sslflags=DONT_VERIFY_PEER front-end-https=on
name=webmail-owa

acl sites_server_5 dstdomain webmail.mycompanydomain.com
cache_peer_access webmail-owa allow sites_server_5
http_access allow sites_server_5

#################################

# Forward proxy

http_port 80 accel

# TO BE CUSTOMIZED
cache_effective_user squid
cache_effective_group squid

cache_mgr webmaster_at_mycompanydomain.com
mail_from webmaster_at_mycompanydomain.com
visible_hostname proxy-cache.mycompanydomain.com

# cache_mem 32 MB
# cache_swap_low 90
# cache_swap_high 95
# maximum_object_size 524288 KB
# ipcache_size 1024
# ipcache_low 90
# ipcache_high 95
# fqdncache_size 1024
cache_replacement_policy heap LFUDA
memory_replacement_policy lru
cache_dir aufs /var/spool/squid 1024 16 256

httpd_suppress_version_string on
forwarded_for on
icp_port 0

logformat combined %>a %[ui %[un [%tl] "%rm %ru HTTP/%rv" %>Hs %<st
"%{Referer}>h" "%{User-Agent}>h" %Ss:%Sh

#logformat combined %>a %ui %un [%tl] "%rm %ru HTTP/%rv" %<Hs %>Hs %<st
Host:"%{Host}>h" ref:"%{Referer}>h" ua:"%{User-Agent}>h" %Ss:%Sh
access_log /var/log/squid/access.log combined
cache_log /var/log/squid/cache.log
cache_store_log /var/log/squid/store.log
# debug_options ALL,1
coredump_dir /var/spool/squid

mime_table /etc/squid/mime.conf
log_mime_hdrs off
pid_filename /var/run/squid.pid

check_hostnames on
hosts_file /etc/hosts

# ACLs to define what is allowed and what is not
acl all src all
acl localhost src 127.0.0.1/32
acl to_localhost dst 127.0.0.0/8
acl SSL_ports port 443
acl Safe_ports port 80 # http
acl Safe_ports port 443 # https
acl CONNECT method CONNECT

# Enforcing of ACLs
http_access deny !Safe_ports
http_access deny to_localhost
http_access deny CONNECT !SSL_ports
http_access allow localhost

### ExternalAUTHENTICATION ####

### negotiate kerberos and ntlm authentication
auth_param negotiate program /usr/local/bin/negotiate_wrapper -d
--ntlm /usr/bin/ntlm_auth --diagnostics
--helper-protocol=squid-2.5-ntlmssp --domain=MYCOMPANYDOMAIN.NET
--kerberos /usr/lib64/squid/squid_kerb_auth -d -s GSS_C_NO_NAME
auth_param negotiate children 10
auth_param negotiate keep_alive off

### pure ntlm authentication
auth_param ntlm program /usr/bin/ntlm_auth --diagnostics
--helper-protocol=squid-2.5-ntlmssp --domain=MYCOMPANYDOMAIN.NET
auth_param ntlm children 10
auth_param ntlm keep_alive off

### provide basic authentication via ldap for clients not authenticated
via kerberos/ntlm
# auth_param basic program /usr/lib/squid3/squid_ldap_auth -R -b
"dc=example,dc=local" -D squid_at_example.local -W /etc/squid3/ldappass.txt
-f sAMAccountName=%s -h dc1.example.local
# auth_param basic children 10
# auth_param basic realm Internet Proxy
# auth_param basic credentialsttl 1 minute

### acl for proxy auth and ldap authorizations
acl auth proxy_auth REQUIRED

### enforce authentication
# http_access deny !auth
http_access allow auth
#http_access deny all

http_access deny all

# http_access allow all

icp_access deny all

# Routing information to parent caches: everything is forwarded to
www.mycompanydomain.com
cache_peer 192.1.1.1 parent 80 0 no-query name=www
# cache_peer 192.168.17.10 parent 80 0 no-query name=www
cache_peer_access www allow all
# never_direct allow all

acl QUERY urlpath_regex cgi-bin \?
cache deny QUERY
cache deny all

# We strongly recommend the following be uncommented to protect innocent
# web applications running on the proxy server who think the only
# one who can access services on "localhost" is a local user
http_access deny to_localhost

# http_port 3128

# We recommend you to use at least the following line.
hierarchy_stoplist cgi-bin ?

# Uncomment and adjust the following to add a disk cache directory.
#cache_dir ufs /var/spool/squid 100 16 256

# Leave coredumps in the first cache dir
coredump_dir /var/spool/squid

# Add any of your own refresh_pattern entries above these.
refresh_pattern ^ftp: 1440 20% 10080
refresh_pattern ^gopher: 1440 0% 1440
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern . 0 20% 4320

Squid Cache: Version 3.3.9
configure options: '--host=x86_64-redhat-linux-gnu'
'--build=x86_64-redhat-linux-gnu' '--program-prefix=' '--prefix=/usr'
'--exec-prefix=/usr' '--bindir=/usr/bin' '--sbindir=/usr/sbin'
'--sysconfdir=/etc' '--datadir=/usr/share' '--includedir=/usr/include'
'--libdir=/usr/lib64' '--libexecdir=/usr/libexec'
'--sharedstatedir=/var/lib' '--mandir=/usr/share/man'
'--infodir=/usr/share/info' '--exec_prefix=/usr'
'--libexecdir=/usr/lib64/squid' '--localstatedir=/var'
'--datadir=/usr/share/squid' '--sysconfdir=/etc/squid' '--with-logdir=
$(localstatedir)/log/squid' '--with-pidfile=
$(localstatedir)/run/squid.pid' '--disable-dependency-tracking'
'--enable-eui' '--enable-follow-x-forwarded-for' '--enable-auth'
'--enable-auth-basic=DB,LDAP,MSNT,MSNT-multi-domain,NCSA,NIS,PAM,POP3,RADIUS,SASL,SMB,getpwnam' '--enable-auth-ntlm=smb_lm,fake' '--enable-auth-digest=file,LDAP,eDirectory' '--enable-auth-negotiate=kerberos,wrapper' '--enable-external-acl-helpers=wbinfo_group,kerberos_ldap_group,AD_group' '--enable-cache-digests' '--enable-cachemgr-hostname=localhost' '--enable-delay-pools' '--enable-epoll' '--enable-icap-client' '--enable-ident-lookups' '--enable-linux-netfilter' '--enable-removal-policies=heap,lru' '--enable-snmp' '--enable-ssl' '--enable-ssl-crtd' '--enable-storeio=aufs,diskd,ufs' '--enable-wccpv2' '--enable-esi' '--with-aio' '--with-default-user=squid' '--with-filedescriptors=16384' '--with-dl' '--with-openssl' '--with-pthreads' 'build_alias=x86_64-redhat-linux-gnu' 'host_alias=x86_64-redhat-linux-gnu' 'CFLAGS=-O2 -g -fpie' 'LDFLAGS=-pie' 'CXXFLAGS=-O2 -g -fpie' 'PKG_CONFIG_PATH=/usr/lib64/pkgconfig:/usr/share/pkgconfig'

I can provide the src rpm if needed

my
 

-----Original Message-----
From: techguy005-me_at_yahoo.com <techguy005-me_at_yahoo.com>
Reply-to: "techguy005-me_at_yahoo.com" <techguy005-me_at_yahoo.com>
To: squid-users_at_squid-cache.org <squid-users_at_squid-cache.org>
Subject: [squid-users] Squid 3.3 Reverse Proxy Mode - 502 Errors when
uploading files larger than 6MB
Date: Thu, 21 Nov 2013 08:50:55 -0800 (PST)

I am currently running Squid 3.3.9 and 3.3.10 on RedHat
Enterprise Server 5 in a reverse-proxy set-up and have an issue when attempting
to upload a file (i.e. .xls) that is larger than 6MB. A 502 error is
thrown by Squid. The backend parent web server is an IIS 7.5 Win2k8 R2.
The oddity is the site functions just fine under the old Squid 2.6.22
version. However after upgrading to 3.3.9 (and subsequently 3.3.10 to see
if this error resolves), the upload issue of files larger than 6MB still
exists.
 
The 502 Error from the access.log reads:
https://site.com/products/application/SomeThing/Batch.aspx -
FIRSTUP_PARENT/192.168.1.5 text/html
1384868971.758 134798 192.250.4.3 TCP_MISS/502 5065 POST
 
I turned up the debug level to: debug_options ALL,333
 
This produced the output below in the cache.log file
which showed a “(104) Connection reset by peer” error:
 
013/11/20 07:58:09.714 kid1| http.cc(1104)
persistentConnStatus: persistentConnStatus: clen=0
2013/11/20 07:58:10.167 kid1| comm.cc(145)
commHandleRead: comm_read_try: FD 10, size 16383, retval -1, errno 104
2013/11/20 07:58:10.167 kid1| AsyncCall.cc(85)
ScheduleCall: IoCallback.cc(127) will call
HttpStateData::readReply(local=192.250.4.3:55985 remote=192.168.1.5:443 FD 10
flags=1, errno=104, flag=-1, data=0x10edade8, size=0, buf=0x10ee9ff0) [call264]
2013/11/20 07:58:10.167 kid1| AsyncCallQueue.cc(51)
fireNext: entering HttpStateData::readReply(local=192.250.4.3:55985
remote=192.168.1.5:443 FD 10 flags=1, errno=104, flag=-1, data=0x10edade8,
size=0, buf=0x10ee9ff0)
2013/11/20 07:58:10.168 kid1| http.cc(1172) readReply:
local=192.250.4.3:55985 remote=192.168.1.5:443 FD 10 flags=1: read failure:
(104) Connection reset by peer.
2013/11/20 07:58:10.168 kid1| AsyncJob.cc(131) callEnd:
HttpStateData::readReply(local=192.250.4.3:55985 remote=192.168.1.5:443
flags=1, errno=104, flag=-1, data=0x10edade8, size=0, buf=0x10ee9ff0) ends job
[ job25]
2013/11/20 07:58:10.168 kid1| AsyncJob.cc(141) callEnd:
HttpStateData::readReply(local=192.250.4.3:55985 remote=192.168.1.5:443
flags=1, errno=104, flag=-1, data=0x10edade8, size=0, buf=0x10ee9ff0) ended
0x10edaea8
2013/11/20 07:58:10.168 kid1| AsyncCallQueue.cc(53)
fireNext: leaving HttpStateData::readReply(local=192.250.4.3:55985
remote=192.168.1.5:443 flags=1, errno=104, flag=-1, data=0x10edade8, size=0,
buf=0x10ee9ff0)
2013/11/20 07:58:10.169 kid1| errorpage.cc(1120) Convert:
errorConvert: %%E --> '(104) Connection reset by peer'
 
I played around with various parameters in the squid.conf
file to no avail:
 
# Base ACL rules to allow connections on port 80 and 443
acl all src all
acl port80 port 80
acl port443 port 443
http_access allow port80
http_access allow port443
http_access deny all
http_reply_access allow all
 
# Forces no caching of failed requests
negative_ttl 0
 
# Timeout value for closing persistent idle connections
pconn_timeout 1 seconds
 
#request_body_max_size 20 MB
#client_request_buffer_max_size 20 MB
 
# Turns off some HTTP Headers we do not want exposed
via off
#forwarded_for off
request_header_access From deny all
request_header_access Server deny all
request_header_access WWW-Authenticate deny all
request_header_access Link deny all
request_header_access Proxy-Connection deny all
request_header_access X-Cache deny all
request_header_access X-Cache-Lookup deny all
request_header_access Via deny all
request_header_access Keep-Alive deny all
 
# SSL HTTP Listeners
https_port 168.250.1.2:443 accel protocol=https vhost
cert=/apps/squid/etc/ssl/sslcert.pem cafile=/apps/squid/etc/ssl/verisign.pem
defaultsite=site.com
 
# SSL Cache Peer
cache_peer 192.168.1.5 parent 443 0 proxy-only
originserver ssl name=pilot_ssl ssldomain=site.com sslflags=DONT_VERIFY_PEER
 
# SSL ACL
acl pilot_ssl_IP_acl myip
168.250.1.2
 
# SSL Cache Peer Access
cache_peer_access pilot_ssl allow pilot_ssl_IP_acl
port443
 
Squid was compiled in this manner:
 
./configure --prefix=/apps/squid3.3.9 --enable-icmp
--enable-ssl --with-openssl=/apps/ssl --disable-internal-dns --disable-ipv6
--with-large-files
--enable-external-acl-helpers=LDAP,MSNT,NCSA,PAM,SMB,YP,getpwnam,multi-domain-NTLM,DB,squid_radius_auth
 
As I mentioned before, everything works just fine under
Squid 2.6.22 hitting the exact same back-end web server. However it
breaks on Squid 3.3.9 and Squid 3.3.10. Something appears to have changed
in the manner in which the connection/buffer/something functions between Squid
3.3.x and the back-end parent server that causes the hiccup with 6MB or larger
files. Attempted to change the timeout value on the back-end parent
server (IIS Windows) to 240 seconds, however all that did was cause the timeout
to happen at the 240 second mark rather than the default 120 second mark.
 It is as if something causes a stall between Squid and IIS as the file
upload never actually make it to the back-end parent server. Again, this
functions just fine under Squid 2.6.22 so something is amiss.
 
Any assistance would be greatly appreciated to resolve or
further troubleshoot this issue. Thanks!
Received on Thu Nov 21 2013 - 18:00:52 MST