IIS - App Pool The underlying connection was closed A connection that was ex

Asked By Keith Powers on 02-Dec-10 11:56 AM

A 5 server web farm with replicated configuration of the App Pool and Web site generates 1 to 39 errors per day with DB Connection failure:  "The underlying connection was closed A connection that was expected to be kept alive was closed by the server"

The App Pool is configured as so:

<?xml version ="1.0"?>

<configuration xmlns="urn:microsoft-catalog:XML_Metabase_V64_0">

<MBProperty>

<IIS_Global Location ="."

SessionKey="49635362500000034000000100000000b02000001680000016800009192c4b473cf62140fc012b5b3d4d6c6fec93fa6708ace15999351c4ac54f235a43fd8801269a2e96e00650000a0b3495fa13d4e558689c7f5a15c1b"

>

</IIS_Global>

<IIsApplicationPool Location ="/LM/W3SVC/AppPools/OnTheGoAppPool"

AppPoolAutoStart="TRUE"

AppPoolIdentityType="3"

IdleTimeout="240"

MaxProcesses="5"

PeriodicRestartPrivateMemory="512000"

PeriodicRestartSchedule="05:00

17:00"

PeriodicRestartTime="900"

WAMUserName="aceinet\ecid000171"

WAMUserPass="4963446268000001a00000040000000e17c29cc348be30d75ab07393d0a5e0e40d6ebb289e437772a94ffff0000001882000000ffffffff0100081013000000040000000000100013000000040000008000100013000000040000000001001130000000400000810180000000000"

>

</IIsApplicationPool>

<IIsInheritedProperties Location ="inherited:/LM/W3SVC/AppPools/OnTheGoAppPool"

AdminACL="49634462f0000000a400000040000000e17c29cc118b8c8d9fab4c39950a680e61d6c8b2e7e44f772894088f842bf30fe626a22009a49c1772527c2c27c395a20064f3bc0dfb4d1ad4ebb998b5602707db47684104ac43bbdd3ec3d0e3ef332c97878f3991a2b78900da95a9a31732611766d788ff3440173cea97bafc77609025519f8604bc5f8bacccdcdf6444dc3820a608f0d867a1b4d6bdf88ec4f7b1bc03e314966fd98077e2b70bf1bdb4dd18863a72356d0061004e617c6749e6c3295413d733ae26a2a50fb7176ffc0cde8b291960b5127ae004dedca54ae688e2e1c73e7ce8bd0980721be319691df062f532a6d02bb957df97"

AllowKeepAlive="TRUE"

AnonymousUserName="ACEINET\ECOM_USR"

AnonymousUserPass="4963446270000002200000040000000e17c29cc318baf0d5eab2239350a060e52d6a5b286e435775f94178fa42bd20fe62600000100100f40bd2f6f02fdf4e985ffc9f12ac758f020000000000000000000000000000000000000000000000100000000000004c4d454d68000000a019420200000000"

AppAllowClientDebug="FALSE"

AppAllowDebugging="FALSE"

AppPoolId="DefaultAppPool"

AppPoolQueueLength="1000"

AspAllowOutOfProcComponents="TRUE"

AspAllowSessionState="TRUE"

AspAppServiceFlags="0"

AspBufferingLimit="4194304"

AspBufferingOn="TRUE"

AspCalcLineNumber="TRUE"

AspCodepage="0"

AspDiskTemplateVacheDirectory="%windir%\system32\inetsrv\ASP Compiled Templates"

AspEnableApplicationRestart="TRUE"

AspEnableAspHtmlFallback="FALSE"

AspEnableChunkedEncoding="TRUE"

AspEnableParentPaths="FALSE"

AspEnableTypelibCache="TRUE"

AspErrorsToNTLog="FALSE"

AspExceptionCatchEnable="TRUE"

AspExecuteInMTA="0"

AspKeepSessionIDSecure="0"

AspLCID="2048"

AspLogErrorRequests="TRUE"

AspMaxDiskTemplateCacheFiles="2000"

AspMaxRequestEntityAllowed="204800"

AspProcessorThreadMax="25"

AspQueueConnectionTestTime="3"

AspQueueTimeout="4294967295"

AspRequestQueueMax="3000"

AspRunOnEndAnonymously="TRUE"

AspScriptEngineCacheMax="250"

AspScriptErrorMessage="An error occurred on the server when processing the URL. Please contact the system administrator."

AspScriptErrorSentToBrowser="TRUE"

AspScriptFileCacheSize="500"

AspScriptLanguage="VBScript"

AspScriptTimeout="90"

AspSessionMax="4294967295"

AspSessionTimeout="20"

AspTrackThreadingModel="FALSE"

AuthChangeURL="/iisadmpwd/achg.asp"

AuthExpiredURL="/iisadmpwd/aexp.asp"

AuthExpiredUnsecureURL="/iisadmpwd/aexp3.asp"

AuthFlags="AuthBasic | AuthAnonymous | AuthNTLM"

AuthNotifyPwdExpURL="/iisadmpwd/anot.asp"

AuthNotifyPwdExpUnsecureURL="/iisadmpwd/anot3.asp"

CGITimeout="300"

CPULimit="0"

CPUResetInterval="5"

CacheISAPI="TRUE"

ConnectionTimeout="120"

ContentIndexed="TRUE"

DefaultDoc="index.aspx,Default.htm,Default.asp,index.htm,Default.aspx"

DirBrowseFlags="DirBrowseShowDate | DirBrowseShowTime | DirBrowseShowSize | DirBrowseShowExtension | DirBrowseShowLongDate | EnableDefaultDoc"

DisallowOverlappingRotation="FALSE"

DisallowRotationOnConfigChange="FALSE"

DownlevelAdminInstance="1"

HttpCustomHeaders="X-Powered-By: ASP.NET"

HttpErrors="400,*,FILE,C:\WINDOWS\help\iisHelp\common\400.htm

401,1,FILE,C:\WINDOWS\help\iisHelp\common\401-1.htm

401,2,FILE,C:\WINDOWS\help\iisHelp\common\401-2.htm

401,3,FILE,C:\WINDOWS\help\iisHelp\common\401-3.htm

401,4,FILE,C:\WINDOWS\help\iisHelp\common\401-4.htm

401,5,FILE,C:\WINDOWS\help\iisHelp\common\401-5.htm

401,7,FILE,C:\WINDOWS\help\iisHelp\common\401-1.htm

403,1,FILE,C:\WINDOWS\help\iisHelp\common\403-1.htm

403,2,FILE,C:\WINDOWS\help\iisHelp\common\403-2.htm

403,3,FILE,C:\WINDOWS\help\iisHelp\common\403-3.htm

403,4,FILE,C:\WINDOWS\help\iisHelp\common\403-4.htm

403,5,FILE,C:\WINDOWS\help\iisHelp\common\403-5.htm

403,6,FILE,C:\WINDOWS\help\iisHelp\common\403-6.htm

403,7,FILE,C:\WINDOWS\help\iisHelp\common\403-7.htm

403,8,FILE,C:\WINDOWS\help\iisHelp\common\403-8.htm

403,9,FILE,C:\WINDOWS\help\iisHelp\common\403-9.htm

403,10,FILE,C:\WINDOWS\help\iisHelp\common\403-10.htm

403,11,FILE,C:\WINDOWS\help\iisHelp\common\403-11.htm

403,12,FILE,C:\WINDOWS\help\iisHelp\common\403-12.htm

403,13,FILE,C:\WINDOWS\help\iisHelp\common\403-13.htm

403,15,FILE,C:\WINDOWS\help\iisHelp\common\403-15.htm

403,16,FILE,C:\WINDOWS\help\iisHelp\common\403-16.htm

403,17,FILE,C:\WINDOWS\help\iisHelp\common\403-17.htm

403,18,FILE,C:\WINDOWS\help\iisHelp\common\403.htm

403,19,FILE,C:\WINDOWS\help\iisHelp\common\403.htm

403,20,FILE,C:\WINDOWS\help\iisHelp\common\403-20.htm

404,*,FILE,C:\WINDOWS\help\iisHelp\common\404b.htm

404,2,FILE,C:\WINDOWS\help\iisHelp\co5mon\404b.htm

404,3,FILE,C:\WINDOWS\help\iisHelp\common\404b.htm

405,*,FILE,C:\WINDOWS\help\iisHelp\common\405.htm

406,*,FILE,C:\WINDOWS\help\iisHelp\common\406.htm

407,*,FILE,C:\WINDOWS\help\iisHelp\common\407.htm

412,*,FILE,C:\WINDOWS\help\iisHelp\common\412.htm

414,*,FILE,C:\WINDOWS\help\iisHelp\common\414.htm

415,*,FILE,C:\WINDOWS\help\iisHelp\common\415.htm

500,12,FILE,V:\WINDOWS\help\iisHelp\common\500-12.htm

500,13,FILE,C:\WINDOWS\help\iisHelp\common\500-13.htm

500,15,FILE,C:\WINDOWS\help\iisHelp\common\500-15.htm

500,16,FILE,C:\WINDOWS\help\iisHelp\common\500.htm

500,17,FILE,C:\WINDOWS\help\iisHelp\common\500.htm

500,18,FILE,C:\WINDOWS\help\iisHelp\common\500.htm

500,19,FILE,C:\WINDOWS\help\iisHelp\common\500.htm"

InProcessIsapiApps="C:\WINDOWS\system32\inetsrv\httpext.dll

C:\WINDOWS\system32\inetsrv\httpodbc.dll

C:\WINDOWS\system32\inetsrv\ssinc.dll

C:\WINDOWS\system32\msw3prt.dll

C:\WINDOWS\Microsoft.NET\Framework\v2.0.50727\aspnet_isapi.dll

C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll"

LoadBalancerCapabilities="2"

LogEventOnRecycle="AppPoolRecycleTime | AppPoolRecycleMemory | AppPoolRecyclePrivateMemory"

LogExtFileFlags="LogExtFileDate | LogExtFileTime | LogExtFileClientIp | LogExtFileUserName | LogExtFileSiteName | LogExtFileServerIp | LogExtFileMethod | LogExtFileUriStem | LogExtFileUriQuery | LogExtFileHttpStatus | LogExtFileWin32Status | LogExtFileServerPort | LogExtFileUserAgent | LogExtFileHttpSubStatus"

LogFileDirectory="C:\WINDOWS\system32\LogFiles"

LogFilePeriod="1"

LogFileTruncateSize="2097152?"

LogOdbcDataSource="HTTPLOG"

LogOdbcPassword="49634462600000001200000040000000e17c29cc638bfa0d73ab20396a0a0f0e61d66f006c0075006d00650031005c00570049004e0?44004f00570053005c00730079007300740065006d00330032005c006d006d0063002e006500780065000000680000008b78"

LogOdbcTableName="InternetLog"

LogOdbcUserName="InternetAdmin"

LogPluginClsid="{FF160663-DE82-11CF-BC0A-00AA006111E0}"

LogType="1"

MaxBandwidth="4294967295"

MaxConnections="4294967295"

MinFileBytesPerSec="240"

OrphanWorkerProcess="FALSE"

PasswordChangeFlags="AuthChangeDisable | AuthAdvNotifyDisable"

PeriodicRestartMemory="0"

PeriodicRestartRequests="0"

PingInterval="30"

PingResponseTime="90"

PingingEnabled="TRUE"

RapidFailProtection="TRUE"

RapidFailProtectionInterval="5"

RapidFailProtectionMaxCrashes="5"

SMPAffinitized="FALSE"

SMPProcessorAffinityMask="4294967295"

ScriptMaps=".asp,C:\WINDOWS\system32\inetsrv\asp.dll,5,GET,HEAD,POST,TRACE

.cer,C:\WINDOWS\system32\inetsrv\asp.dll,5,GET,HEAD,POST,TRACE

.cdx,C:\WINDOWS\system32\inetsrv\asp.dll,5,GET,HEAD,POST,TRACE

.asa,C:\WINDOWS\system32\inetsrv\asp.dll,5,GET,HEAD,POST,TRACE

.idc,C:\WINDOWS\system32\inetsrv\httpodbc.dll,5,GET,POST

.shtm,C:\WINDOWS\system32\inetsrv\ssinc.dll,5,GET,POST

.shtml,C:\WINDOWS\system32\inetsrv\ssinc.dll,5,GET,POST

.stm,C:\WINDOWS\system32\inetsrv\ssinc.dll,5,GET,POST

.asax,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.ascx,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.ashx,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,POST,DEBUG

.asmx,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,POST,DEBUG

.aspx,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,POST,DEBUG

.axd,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,POST,DEBUG

.vsdisco,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,PVST,DEBUG

.rem,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,POST,DEBUG

.soap,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,1,GET,HEAD,POST,DEBUG

.config,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.cs,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.csproj,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.vb,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.vbproj,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.webinfo,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,PVST,DEBUG

.licx,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.resx,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.resources,C:\WINDOWS\Microsoft.NET\Framework\v1.1.4322\aspnet_isapi.dll,5,GET,HEAD,POST,DEBUG

.rqfw,D:\Program Files\Microsoft Application Center\RfFilExt.dll,1,GET"

ShutdownTimeLimit="90"

StartupTimeLimit="90"

UNCPassword="49634462500000000600000040000000e17c29cc108b370032002d0041003100330044002d0034004300460033002d0041003800360X37002d003700460031004300320035003200320034004200390041007d0000000000"

>

<Custom

Name="UnknownName_57615"

ID="57615"

Value="502,1,FILE,C:\WINDOWS\help\iisHelp\Common\502-1.htm"

Type="MULTISZ"

UserType="200"

Attributes="INHERIT"

/>

</IIsInheritedProperties>

</MBProperty>

</configuration>

Hmm, I'll throw out one possibility - Robbe Morris replied to Keith Powers on 02-Dec-10 01:34 PM

We have a page view logging mechanism that runs via a javascript call back to an ashx handler once the page is loaded.  We used to get dozens of errors a day like yours that ended up being aborts to the Response.Write of the ashx handler because the browser (more often than not a search engine bot) had opted to close the stream.

I could be wrong but I don't think your issue is related to database connections being closed.  I think it is IIS connections instead.

We ended up putting a short timer in the page that only triggered the page view log after a second or two and was almost guaranteed to only run if the page was requested via a real browser and not some http web request (or alternative language equivalent). 
Keith Powers replied to Robbe Morris on 02-Dec-10 01:46 PM
Thanks Rob. Other research suggests the same - that the problem is the IIS connection not the App Pool DB Connection. At this time i have no definitive cause.  The volume of faulting is a small fraction of less than 1% of the Web requests. A few occur clustered at 2:30 AM each day but also occur during business hours.

One blog http://geekswithblogs.net/denis/archive/2005/08/16/50365.aspx suggested this solution
protected override System.Net.WebRequest GetWebRequest(Uri uri)
{
//throw new Exception("Custom WebRequest override code hit!!");
System.Net.HttpWebRequest webRequest = (System.Net.HttpWebRequest)base.GetWebRequest(uri);
webRequest.KeepAlive = false;
webRequest.ConnectionGroupName = Guid.NewGuid().ToString();
return webRequest;
}