GPU: Clear alpha more consistently from 565.

Before, the backends all did different things.  Now they are more in sync,
but Vulkan still behaves slightly differently.

Fixes #11326.

GPU/D3D11/FramebufferManagerD3D11.cpp

@@ -129,6 +129,21 @@ FramebufferManagerD3D11::FramebufferManagerD3D11(Draw::DrawContext *draw)
 	vb.BindFlags = D3D11_BIND_CONSTANT_BUFFER;
 	ASSERT_SUCCESS(device_->CreateBuffer(&vb, nullptr, &postConstants_));
 
+	D3D11_TEXTURE2D_DESC desc{};
+	desc.CPUAccessFlags = 0;
+	desc.Usage = D3D11_USAGE_DEFAULT;
+	desc.ArraySize = 1;
+	desc.SampleDesc.Count = 1;
+	desc.Width = 1;
+	desc.Height = 1;
+	desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
+	desc.MipLevels = 1;
+	desc.BindFlags = D3D11_BIND_SHADER_RESOURCE;
+	ASSERT_SUCCESS(device_->CreateTexture2D(&desc, nullptr, &nullTexture_));
+	ASSERT_SUCCESS(device_->CreateShaderResourceView(nullTexture_, nullptr, &nullTextureView_));
+	uint32_t nullData[1]{};
+	context_->UpdateSubresource(nullTexture_, 0, nullptr, nullData, 1, 0);
+
 	ShaderTranslationInit();
 
 	CompilePostShader();
@@ -178,6 +193,11 @@ FramebufferManagerD3D11::~FramebufferManagerD3D11() {
 		stencilUploadInputLayout_->Release();
 	if (stencilValueBuffer_)
 		stencilValueBuffer_->Release();
+
+	if (nullTextureView_)
+		nullTextureView_->Release();
+	if (nullTexture_)
+		nullTexture_->Release();
 }
 
 void FramebufferManagerD3D11::SetTextureCache(TextureCacheD3D11 *tc) {
@@ -437,31 +457,33 @@ void FramebufferManagerD3D11::ReformatFramebufferFrom(VirtualFramebuffer *vfb, G
 	// Technically, we should at this point re-interpret the bytes of the old format to the new.
 	// That might get tricky, and could cause unnecessary slowness in some games.
 	// For now, we just clear alpha/stencil from 565, which fixes shadow issues in Kingdom Hearts.
-	// (it uses 565 to write zeros to the buffer, than 4444 to actually render the shadow.)
+	// (it uses 565 to write zeros to the buffer, then 4444 to actually render the shadow.)
 	//
 	// The best way to do this may ultimately be to create a new FBO (combine with any resize?)
 	// and blit with a shader to that, then replace the FBO on vfb.  Stencil would still be complex
 	// to exactly reproduce in 4444 and 8888 formats.
 	if (old == GE_FORMAT_565) {
-		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::CLEAR, Draw::RPAction::KEEP, Draw::RPAction::KEEP });
+		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::KEEP, Draw::RPAction::KEEP, Draw::RPAction::CLEAR });
 
-		// TODO: There's no way this does anything useful :(
+		context_->OMSetDepthStencilState(stockD3D11.depthStencilDisabled, 0xFF);
-		context_->OMSetDepthStencilState(stockD3D11.depthDisabledStencilWrite, 0xFF);
+		context_->OMSetBlendState(stockD3D11.blendStateDisabledWithColorMask[D3D11_COLOR_WRITE_ENABLE_ALPHA], nullptr, 0xFFFFFFFF);
-		context_->OMSetBlendState(stockD3D11.blendStateDisabledWithColorMask[0], nullptr, 0xFFFFFFFF);
 		context_->RSSetState(stockD3D11.rasterStateNoCull);
 		context_->IASetInputLayout(quadInputLayout_);
 		context_->PSSetShader(quadPixelShader_, nullptr, 0);
 		context_->VSSetShader(quadVertexShader_, nullptr, 0);
 		context_->IASetVertexBuffers(0, 1, &fsQuadBuffer_, &quadStride_, &quadOffset_);
+		context_->PSSetSamplers(0, 1, &stockD3D11.samplerPoint2DClamp);
+		context_->PSSetShaderResources(0, 1, &nullTextureView_);
 		shaderManagerD3D11_->DirtyLastShader();
 		D3D11_VIEWPORT vp{ 0.0f, 0.0f, (float)vfb->renderWidth, (float)vfb->renderHeight, 0.0f, 1.0f };
 		context_->RSSetViewports(1, &vp);
 		context_->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
 		context_->Draw(4, 0);
-	}
 
-	RebindFramebuffer();
+		textureCache_->ForgetLastTexture();
-	gstate_c.Dirty(DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE | DIRTY_RASTER_STATE | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_VERTEXSHADER_STATE);
+
+		gstate_c.Dirty(DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE | DIRTY_RASTER_STATE | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_VERTEXSHADER_STATE);
+	}
 }
 
 static void CopyPixelDepthOnly(u32 *dstp, const u32 *srcp, size_t c) {

GPU/D3D11/FramebufferManagerD3D11.h

@@ -107,6 +107,9 @@ private:
 	ID3D11Buffer *stencilValueBuffer_ = nullptr;
 	ID3D11DepthStencilState *stencilMaskStates_[256]{};
 
+	ID3D11Texture2D *nullTexture_ = nullptr;
+	ID3D11ShaderResourceView *nullTextureView_ = nullptr;
+
 	TextureCacheD3D11 *textureCacheD3D11_;
 	ShaderManagerD3D11 *shaderManagerD3D11_;
 	DrawEngineD3D11 *drawEngineD3D11_;

GPU/Directx9/FramebufferDX9.cpp

@@ -100,6 +100,19 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 		}
 
 		device_->CreateVertexDeclaration(g_FramebufferVertexElements, &pFramebufferVertexDecl);
+
+
+		int usage = 0;
+		D3DPOOL pool = D3DPOOL_MANAGED;
+		if (deviceEx_) {
+			pool = D3DPOOL_DEFAULT;
+			usage = D3DUSAGE_DYNAMIC;
+		}
+		HRESULT hr = device_->CreateTexture(1, 1, 1, usage, D3DFMT_A8R8G8B8, pool, &nullTex_, nullptr);
+		D3DLOCKED_RECT rect;
+		nullTex_->LockRect(0, &rect, nullptr, D3DLOCK_DISCARD);
+		memset(rect.pBits, 0, 4);
+		nullTex_->UnlockRect(0);
 	}
 
 	FramebufferManagerDX9::~FramebufferManagerDX9() {
@@ -125,6 +138,8 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 		if (stencilUploadVS_) {
 			stencilUploadVS_->Release();
 		}
+		if (nullTex_)
+			nullTex_->Release();
 	}
 
 	void FramebufferManagerDX9::SetTextureCache(TextureCacheDX9 *tc) {
@@ -307,24 +322,24 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 			return;
 		}
 
-		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::CLEAR, Draw::RPAction::KEEP, Draw::RPAction::KEEP });
-
 		// Technically, we should at this point re-interpret the bytes of the old format to the new.
 		// That might get tricky, and could cause unnecessary slowness in some games.
 		// For now, we just clear alpha/stencil from 565, which fixes shadow issues in Kingdom Hearts.
-		// (it uses 565 to write zeros to the buffer, than 4444 to actually render the shadow.)
+		// (it uses 565 to write zeros to the buffer, then 4444 to actually render the shadow.)
 		//
 		// The best way to do this may ultimately be to create a new FBO (combine with any resize?)
 		// and blit with a shader to that, then replace the FBO on vfb.  Stencil would still be complex
 		// to exactly reproduce in 4444 and 8888 formats.
 
 		if (old == GE_FORMAT_565) {
+			draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::KEEP, Draw::RPAction::KEEP, Draw::RPAction::CLEAR });
+
 			dxstate.scissorTest.disable();
 			dxstate.depthWrite.set(FALSE);
 			dxstate.colorMask.set(false, false, false, true);
 			dxstate.stencilFunc.set(D3DCMP_ALWAYS, 0, 0);
 			dxstate.stencilMask.set(0xFF);
-			gstate_c.Dirty(DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE | DIRTY_VIEWPORTSCISSOR_STATE);
+			gstate_c.Dirty(DIRTY_BLEND_STATE | DIRTY_DEPTHSTENCIL_STATE | DIRTY_RASTER_STATE | DIRTY_VIEWPORTSCISSOR_STATE | DIRTY_TEXTURE_PARAMS | DIRTY_VERTEXSHADER_STATE | DIRTY_FRAGMENTSHADER_STATE);
 
 			float coord[20] = {
 				-1.0f,-1.0f,0, 0,0,
@@ -338,7 +353,7 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 			device_->SetPixelShader(pFramebufferPixelShader);
 			device_->SetVertexShader(pFramebufferVertexShader);
 			shaderManagerDX9_->DirtyLastShader();
-			device_->SetTexture(0, nullptr);
+			device_->SetTexture(0, nullTex_);
 
 			D3DVIEWPORT9 vp{ 0, 0, (DWORD)vfb->renderWidth, (DWORD)vfb->renderHeight, 0.0f, 1.0f };
 			device_->SetViewport(&vp);
@@ -349,9 +364,9 @@ static const D3DVERTEXELEMENT9 g_FramebufferVertexElements[] = {
 				ERROR_LOG_REPORT(G3D, "ReformatFramebufferFrom() failed: %08x", hr);
 			}
 			dxstate.viewport.restore();
-		}
 
-		RebindFramebuffer();
+			textureCache_->ForgetLastTexture();
+		}
 	}
 
 	static void CopyPixelDepthOnly(u32 *dstp, const u32 *srcp, size_t c) {

GPU/Directx9/FramebufferDX9.h

@@ -104,6 +104,8 @@ private:
 	LPDIRECT3DVERTEXSHADER9 stencilUploadVS_ = nullptr;
 	bool stencilUploadFailed_ = false;
 
+	LPDIRECT3DTEXTURE9 nullTex_ = nullptr;
+
 	TextureCacheDX9 *textureCacheDX9_;
 	ShaderManagerDX9 *shaderManagerDX9_;
 	DrawEngineDX9 *drawEngineD3D9_;

GPU/GLES/FramebufferManagerGLES.cpp

@@ -466,20 +466,17 @@ void FramebufferManagerGLES::ReformatFramebufferFrom(VirtualFramebuffer *vfb, GE
 	// Technically, we should at this point re-interpret the bytes of the old format to the new.
 	// That might get tricky, and could cause unnecessary slowness in some games.
 	// For now, we just clear alpha/stencil from 565, which fixes shadow issues in Kingdom Hearts.
-	// (it uses 565 to write zeros to the buffer, than 4444 to actually render the shadow.)
+	// (it uses 565 to write zeros to the buffer, then 4444 to actually render the shadow.)
 	//
 	// The best way to do this may ultimately be to create a new FBO (combine with any resize?)
 	// and blit with a shader to that, then replace the FBO on vfb.  Stencil would still be complex
 	// to exactly reproduce in 4444 and 8888 formats.
 
 	if (old == GE_FORMAT_565) {
-		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::CLEAR, Draw::RPAction::CLEAR, Draw::RPAction::CLEAR });
+		// Clear alpha and stencil.
-		gstate_c.Dirty(DIRTY_BLEND_STATE);
+		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::KEEP, Draw::RPAction::KEEP, Draw::RPAction::CLEAR });
-	} else {
+		render_->Clear(0, 0.0f, 0, GL_COLOR_BUFFER_BIT, 0x8, 0, 0, 0, 0);
-		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::KEEP, Draw::RPAction::KEEP, Draw::RPAction::KEEP });
 	}
-
-	RebindFramebuffer();
 }
 
 void FramebufferManagerGLES::BlitFramebufferDepth(VirtualFramebuffer *src, VirtualFramebuffer *dst) {

GPU/Vulkan/FramebufferVulkan.cpp

@@ -364,16 +364,16 @@ void FramebufferManagerVulkan::ReformatFramebufferFrom(VirtualFramebuffer *vfb,
 	// Technically, we should at this point re-interpret the bytes of the old format to the new.
 	// That might get tricky, and could cause unnecessary slowness in some games.
 	// For now, we just clear alpha/stencil from 565, which fixes shadow issues in Kingdom Hearts.
-	// (it uses 565 to write zeros to the buffer, than 4444 to actually render the shadow.)
+	// (it uses 565 to write zeros to the buffer, then 4444 to actually render the shadow.)
 	//
 	// The best way to do this may ultimately be to create a new FBO (combine with any resize?)
 	// and blit with a shader to that, then replace the FBO on vfb.  Stencil would still be complex
 	// to exactly reproduce in 4444 and 8888 formats.
 
 	if (old == GE_FORMAT_565) {
-		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::CLEAR, Draw::RPAction::CLEAR, Draw::RPAction::CLEAR });
+		// TODO: To match other backends, would be ideal to clear alpha only and not color.
-	} else {
+		// But probably doesn't matter that much...
-		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::KEEP, Draw::RPAction::KEEP, Draw::RPAction::KEEP });
+		draw_->BindFramebufferAsRenderTarget(vfb->fbo, { Draw::RPAction::CLEAR, Draw::RPAction::KEEP, Draw::RPAction::CLEAR });
 	}
 }