Server shuts down (Error 503) on deployed environment after login. on development evironment works as expected

I'm creating an application using NET 6 LTS and Angular 14. On the development environment (using IIS express) the app behaves normally. However when I deploy the application (release) on Windows 2019 and IIS 10, right after the login the server returns error 503. The application pool and the site have appropriate permissions. (I even tried to allow everyone with full access to verify it).

below is the web.config file

<?xml version="1.0" encoding="utf-8"?>
<configuration>
<location path="." inheritInChildApplications="false">
<system.webServer>
<handlers>
<add name="aspNetCore" path="*" verb="*" modules="AspNetCoreModuleV2" resourceType="Unspecified" />
</handlers>
<aspNetCore processPath="dotnet" arguments=".\TheFile.dll" stdoutLogEnabled="false" stdoutLogFile=".\logs\stdout" hostingModel="inprocess" />
</system.webServer>
</location>
</configuration>

and the appsettings.json file is the following

{
"ConnectionStrings": {
"DefaultConnection": "<connection string values>"
},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"Serilog": {
"Using": [ "Serilog.Sinks.MSSqlServer" ],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "MSSqlServer",
"Args": {
"connectionString": "<connection string values>",
"sinkOptionsSection": {
"tableName": "EasymintLog",
"schemaName": "dbo",
"autoCreateSqlTable": true
//"batchPostingLimit": 50,
//"period": "0.00:00:05"
},
//"restrictedToMinimumLevel": "Warning",
"columnOptionsSection": {
"disableTriggers": true,
"clusteredColumnstoreIndex": false,
"primaryKeyColumnName": "Id",
"addStandardColumns": [ "LogEvent" ],
"removeStandardColumns": [ "MessageTemplate", "Properties", "TimeStamp", "Message" ],
"additionalColumns": [
{
"ColumnName": "UserId",
"DataType": "nvarchar",
"AllowNull": true,
"DataLength": 50
},
{
"ColumnName": "ControllerAction",
"DataType": "nvarchar",
"AllowNull": true,
"DataLength": 10
},
{
"ColumnName": "Request",
"DataType": "nvarchar"
},
{
"ColumnName": "Response",
"DataType": "nvarchar"
},
{
"ColumnName": "StatusCode",
"DataType": "int",
"AllowNull": true
},
{
"ColumnName": "LogDate",
"DataType": "datetime2",
"AllowNull": true
},
{
"ColumnName": "ClientAddress",
"DataType": "nvarchar",
"AllowNull": true,
"DataLength": 15
},
{
"ColumnName": "InnerException",
"DataType": "nvarchar"
}
],
"id": { "nonClusteredIndex": true },
"level": {
"columnName": "Severity",
"storeAsEnum": false
},
"timeStamp": {
"columnName": "Timestamp"
//"convertToUtc": true
},
"logEvent": {
"excludeAdditionalProperties": true,
"excludeStandardColumns": true
},
"message": { "columnName": "Message" },
"exception": { "columnName": "Exception" }
//"messageTemplate": { "columnName": "Template" }
}
}
}
]
},
"IdentityServer": {
"Key": {
"Type": "Development"
},
"Clients": {
"NextStepNG": {
"Profile": "IdentityServerSPA"
}
}
},
"AllowedHosts": "*",
//SMTP settings
//"SMTP": {
//  "Host": "",
//  "From": "NextStep Notifications",
//  "Port": 587, //25,
//  "UseSSL": true,
//  "Username": "",
//  "Password": ""
//},
}

I don't know if there is an issue with the app startup.cs file.
Which is the following...

using AutoMapper;
using Duende.IdentityServer.Services;
using IdempotentAPI.Cache.DistributedCache.Extensions.DependencyInjection;
using IdempotentAPI.Cache.FusionCache.Extensions.DependencyInjection;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Identity.UI.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using NextStepNG.BusinessLogic.CommonBLs;
using NextStepNG.BusinessLogic.EshopBLs;
using NextStepNG.Data;
using NextStepNG.Data.EFCore.Common;
using NextStepNG.Data.EFCore.Company;
using NextStepNG.Data.EFCore.Eshop;
using NextStepNG.Data.EFCore.Organization;
using NextStepNG.Data.EFCore.Person;
using NextStepNG.Enums;
using NextStepNG.Models;
using NextStepNG.Services;
using NextStepNG.Utils.Common;
using Serilog;
using System;
using System.Security.Claims;
namespace NextStepNG
{
public class Startup
{
public Startup(IConfiguration configuration)
{
Configuration = configuration;
}
public IConfiguration Configuration { get; }
public void ConfigureServices(IServiceCollection services)
{
services.AddDbContext<ApplicationDbContext>(options =>
options.UseSqlServer(
Configuration.GetConnectionString("DefaultConnection"),
sqlServerOptionsAction: sqlOptions =>
{
sqlOptions.EnableRetryOnFailure();
}));
services.AddDbContext<NEXTSTEPNGContext>(options =>
options.UseSqlServer(
Configuration.GetConnectionString("DefaultConnection"),
sqlServerOptionsAction: sqlOptions =>
{
sqlOptions.EnableRetryOnFailure();
}));
services.AddDistributedMemoryCache();
services.AddIdempotentAPIUsingDistributedCache();
//services.AddIdempotentAPIUsingFusionCache();
//services.AddFusionCacheNewtonsoftJsonSerializer();            
services.AddTransient<IProfileService, ProfileService>();
services.AddDefaultIdentity<ApplicationUser>(options => options.SignIn.RequireConfirmedAccount = true)
.AddRoles<IdentityRole>()
.AddEntityFrameworkStores<ApplicationDbContext>();
services.AddIdentityServer(options =>
{
// new key every 30 days
options.KeyManagement.RotationInterval = TimeSpan.FromDays(30);
// announce new key 2 days in advance in discovery
options.KeyManagement.PropagationTime = TimeSpan.FromDays(2);
// keep old key for 7 days in discovery for validation of tokens
options.KeyManagement.RetentionDuration = TimeSpan.FromDays(7);
// don't delete keys after their retention period is over
options.KeyManagement.DeleteRetiredKeys = false;
})
.AddApiAuthorization<ApplicationUser, ApplicationDbContext>()
.AddProfileService<ProfileService>();
services.AddTransient<IEmailSender, EmailSender>();
services.AddAuthentication()
.AddIdentityServerJwt();
services.AddAuthorization(options =>
{
options.AddPolicy(AuthorizationEnum.Roles.Developer.ToString(), policy =>
policy.RequireClaim(ClaimTypes.Role, AuthorizationEnum.Claims.developer.ToString()));
options.AddPolicy(AuthorizationEnum.Roles.Easymint.ToString(), policy =>
policy.RequireClaim(ClaimTypes.Role, AuthorizationEnum.Claims.easymint.ToString()));
options.AddPolicy(AuthorizationEnum.Roles.Administrator.ToString(), policy =>
policy.RequireClaim(ClaimTypes.Role, AuthorizationEnum.Claims.administrator.ToString()));
options.AddPolicy(AuthorizationEnum.Roles.Manager.ToString(), policy =>
policy.RequireClaim(ClaimTypes.Role, AuthorizationEnum.Claims.manager.ToString()));
options.AddPolicy(AuthorizationEnum.Roles.Tester.ToString(), policy =>
policy.RequireClaim(ClaimTypes.Role, AuthorizationEnum.Claims.tester.ToString()));
options.AddPolicy(AuthorizationEnum.Roles.User.ToString(), policy =>
policy.RequireClaim(ClaimTypes.Role, AuthorizationEnum.Claims.user.ToString()));
});
services.AddControllersWithViews();
services.AddRazorPages();
services.AddControllers();
services.AddMvc();
// In production, the Angular files will be served from this directory
services.AddSpaStaticFiles(configuration =>
{
configuration.RootPath = "ClientApp/dist";
});
services.AddScoped<LoggingActionFilter>();
services.AddMemoryCache();
services.AddMvc().AddJsonOptions(options =>
options.JsonSerializerOptions.PropertyNamingPolicy = null);
services.AddLogging(loggingBuilder =>
loggingBuilder.AddSerilog(dispose: true));
services.Configure<ForwardedHeadersOptions>(options =>
{
options.ForwardedHeaders =
ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto;
});
services.AddDatabaseDeveloperPageExceptionFilter();
services.AddAutoMapper(typeof(Startup));
services.AddScoped<CountryRepository>();
services.AddScoped<LocaleRepository>();
services.AddScoped<RolesRepository>();
services.AddScoped<UserSettingsRepository>();
services.AddScoped<OrganizationLocaleRepository>();
services.AddScoped<OrganizationsRepository>();
services.AddScoped<PersonsRepository>();
services.AddScoped<OrganizationBanksRepository>();
services.AddScoped<AddressesRepository>();
services.AddScoped<BanksRepository>();
services.AddScoped<FullDetailsInfoRepository>();
services.AddScoped<CompaniesRepository>();
services.AddScoped<CompanyBankAccountsRepository>();
services.AddScoped<CompanyDetailsRepository>();
services.AddScoped<CompanyIRSRepository>();
services.AddScoped<CustomerDetailsInfoRepository>();
services.AddScoped<OrganizationDetailsRepository>();
services.AddScoped<OrganizationIRSRepository>();
services.AddScoped<OrganizationBankAccountRepository>();
services.AddScoped<UsersManagementBL>();
services.AddScoped<EmployeeManagementBL>();
services.AddScoped<EmployeeManagementRepository>();
services.AddScoped<PersonsBL>();
services.AddScoped<EshopLicenseTypesRepository>();
services.AddScoped<RepairsRepository>();
//services.AddScoped<EshopUsersBL>();
services.AddSingleton(provider => Configuration);
}
// This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
app.UseForwardedHeaders();
}
else
{
app.UseExceptionHandler("/Error");
app.UseForwardedHeaders();
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
app.UseHsts();
}
app.UseXContentTypeOptions();
app.UseReferrerPolicy(r => r.StrictOriginWhenCrossOrigin());
app.UseXDownloadOptions();
app.UseXRobotsTag(r => r
.NoFollow()
.NoIndex());
app.UseXXssProtection(x => x.Enabled());
//app.UseMiddleware<StackifyMiddleware.RequestTracerMiddleware>(); //debug only
app.UseHttpsRedirection();
app.UseStaticFiles();
//nwebsec section
//app.UseCsp(options => options.DefaultSources(s => s.Self()
//.CustomSources("https://cdn3.devexpress.com/jslib/19.2.6/css/dx.light.css",
//    "https://cdn3.devexpress.com/jslib/19.2.6/css/icons/dxicons.woff",
//    "https://cdn3.devexpress.com/jslib/19.2.6/css/icons/dxicons.woff2",
//    "https://cdn3.devexpress.com/jslib/19.2.6/css/icons/dxicons.ttf")));
app.UseCsp(options => options
.FrameAncestors(f => f.Self()));
//.ScriptSources(f => f.Self())
//.ReportUris(r => r.Uris("/reports")));
if (!env.IsDevelopment())
{
app.UseSpaStaticFiles();
}
app.UseRouting();
app.UseAuthentication();
app.UseIdentityServer();
app.UseAuthorization();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllerRoute(
name: "default",
pattern: "{controller}/{action=Index}/{id?}");
endpoints.MapRazorPages();
endpoints.MapControllerRoute(
name: "default",
pattern: "{controller}/{action=Index}/{operation}/{id?}");
endpoints.MapControllerRoute(
name: "default",
pattern: "{controller}/{action=Index}");
endpoints.MapRazorPages();
});
app.UseSpa(spa =>
{
//To learn more about options for serving an Angular SPA from ASP.NET Core,
//see https://go.microsoft.com/fwlink/?linkid=864501
spa.Options.SourcePath = "ClientApp";
if (env.IsDevelopment())
{
//spa.UseAngularCliServer(npmScript: "start");
//comment the line below before deployment
spa.UseProxyToSpaDevelopmentServer("http://localhost:4200");
}
});
}
}
}

What should I check and how to proceed ? Anyone faced the same issue before ?
Thank you in advance

I tried altering permissions and various settings in IIS based on the info I've found here and google. The result remained the same.

Based on the info I found on the windows event viewer it shows the following error

Category: Duende.IdentityServer.Services.KeyManagement.KeyManager
EventId: 0
SpanId: b248d52022677998
TraceId: 01ac174a5bdcbb9ec7a8aec2e95a16be
ParentId: 0000000000000000
RequestId: 80000042-0001-fc00-b63f-84710c7967bb
RequestPath: /.well-known/openid-configuration
Error unprotecting key with kid FB30056547480314CDD1824BFB8D469F.
Exception: 
System.Security.Cryptography.CryptographicException: The key {a6be54e5-5ff8-4fb4-90aa-eb89fa686bc0} was not found in the key ring. For more information go to http://aka.ms/dataprotectionwarning
at Microsoft.AspNetCore.DataProtection.KeyManagement.KeyRingBasedDataProtector.UnprotectCore(Byte[] protectedData, Boolean allowOperationsOnRevokedKeys, UnprotectStatus& status)
at Microsoft.AspNetCore.DataProtection.KeyManagement.KeyRingBasedDataProtector.Unprotect(Byte[] protectedData)
at Microsoft.AspNetCore.DataProtection.DataProtectionCommonExtensions.Unprotect(IDataProtector protector, String protectedData)
at Duende.IdentityServer.Services.KeyManagement.DataProtectionKeyProtector.Unprotect(SerializedKey key) in /_/src/IdentityServer/Services/Default/KeyManagement/DataProtectionKeyProtector.cs:line 56
at Duende.IdentityServer.Services.KeyManagement.KeyManager.<GetKeysFromStoreAsync>b__20_0(SerializedKey x) in /_/src/IdentityServer/Services/Default/KeyManagement/KeyManager.cs:line 426

Please help, as I haven't encountered this before and I've deployed .NET core apps in the past with no issue.

PS. the app is run locally and there is no issue with firewall or anti virus

when you get this error in the logs, then that means that the Data Protection API is not properly configured.

The key {a6be54e5-5ff8-4fb4-90aa-eb89fa686bc0} was not found in the key ring. For more information go to http://aka.ms/dataprotectionwarning

The issue is that the encryption key is re-generated on each deployment if not properly configured and changing the encryption key means that all previously issued cookies are no longer valid.

The error you get above means that the key to decrypt the session cookie was not found.

I did blog about data protection here for some time ago
https://www.edument.se/post/storing-the-asp-net-core-data-protection-key-ring-in-azure-key-vault?lang=en

The picture below outlines how you can configure the data protection API.