Server cannot authorize it's own tokens - asp.net-core

Question

Here is the setup, I have an auth server that issues tokens to an angular website. I have a controller inside the AuthServer that needs to use the [Authorize] system to only allow JWT tokens that are valid. When I check the User variable in my controller it is always null but when I check the HttpRequestHeaders on the controller I see the token being sent.
I also have an Api server that I implemented using the JWT tokens and [Authorize] system very easily.
Another layer, I am running both Api and Auth servers in docker containers.
My entire Startup.cs file from AuthServer:
var connectionString = Configuration.GetConnectionString("Default");
if (_env.IsDevelopment())
{
try
{
using (AppIdentityDbContext identityDb =
new AppIdentityDbContextFactory(connectionString).Create())
{
int Pendings = identityDb.Database.GetPendingMigrations().Count();
identityDb.Database.Migrate();
}
using (PersistedGrantDbContext persistGrantDb =
new PersistedGrantDbContextFactory(connectionString).Create())
{
int Pendings = persistGrantDb.Database.GetPendingMigrations().Count();
persistGrantDb.Database.Migrate();
}
}
catch (Exception)
{
}
}
services.AddControllersWithViews();
services.AddDbContextPool<AppIdentityDbContext>(options => options.UseSqlServer(connectionString));
services
.AddIdentity<AppUser, IdentityRole>(config=> {
config.User.RequireUniqueEmail = true;
config.SignIn.RequireConfirmedEmail = true;
})
.AddEntityFrameworkStores<AppIdentityDbContext>()
.AddDefaultTokenProviders();
services.AddIdentityServer().AddDeveloperSigningCredential()
// this adds the operational data from DB (codes, tokens, consents)
.AddOperationalStore(options =>
{
options.ConfigureDbContext = builder => builder.UseSqlServer(Configuration.GetConnectionString("Default"));
// this enables automatic token cleanup. this is optional.
options.EnableTokenCleanup = true;
options.TokenCleanupInterval = (int)TimeSpan.FromDays(1).TotalSeconds; // interval in seconds
})
.AddInMemoryIdentityResources(Config.GetIdentityResources())
.AddInMemoryApiResources(Config.GetApiResources())
.AddInMemoryClients(Config.GetClients())
.AddAspNetIdentity<AppUser>()
.AddProfileService<AppUserProfileService>()
.AddJwtBearerClientAuthentication();
services.AddAuthentication(IdentityServerAuthenticationDefaults.AuthenticationScheme)
.AddIdentityServerAuthentication(IdentityServerAuthenticationDefaults.AuthenticationScheme,
jwtOptions =>
{
// jwt bearer options
jwtOptions.Authority = _env.IsDevelopment() ? "https://localhost:5001" : "";
jwtOptions.RequireHttpsMetadata = _env.IsDevelopment() ? false : true;
jwtOptions.Audience = "resourceapi";
jwtOptions.TokenValidationParameters = new Microsoft.IdentityModel.Tokens.TokenValidationParameters()
{
ValidateAudience = false,
ValidateIssuer = _env.IsDevelopment() ? false : true,
ValidateActor = false,
ValidateIssuerSigningKey = false
};
},
referenceOptions =>
{
// oauth2 introspection options
});
services.AddMvc().SetCompatibilityVersion(CompatibilityVersion.Version_3_0);
services.AddCors(options => options.AddPolicy("AllowAll", p => p.AllowAnyOrigin()
.AllowAnyMethod()
.AllowAnyHeader()));
services.Configure<EmailSettings>(Configuration.GetSection("EmailSettings"));
services.AddSingleton<IEmailSender, SmtpSender>();
Configure Section:
public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
{
if (env.IsDevelopment())
{
app.UseDeveloperExceptionPage();
}
else
{
app.UseExceptionHandler("/Home/Error");
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
//app.UseHsts();
app.UseHttpsRedirection();
var forwardOptions = new ForwardedHeadersOptions
{
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto,
RequireHeaderSymmetry = false
};
forwardOptions.KnownNetworks.Clear();
forwardOptions.KnownProxies.Clear();
// ref: https://github.com/aspnet/Docs/issues/2384
app.UseForwardedHeaders(forwardOptions);
}
app.UseCors("AllowAll");
app.UseIdentityServer();
app.UseAuthentication();
app.UseAuthorization();
app.UseStaticFiles();
app.UseRouting();
app.UseEndpoints(endpoints =>
{
endpoints.MapControllerRoute(
name: "default",
pattern: "{controller=Home}/{action=Index}/{id?}");
});
}
Check for user inside of AccountController: Controller
var u = User;
var _user = await _userManager.GetUserAsync(u);
var e = this._httpContextAccessor;