Bounds check writing to the index buffer when expanding lines/rects/points

This commit is contained in:
Henrik Rydgård 2023-05-02 13:21:47 +02:00
parent f40f7ed38c
commit 14f9c27b4e
6 changed files with 53 additions and 20 deletions

View file

@ -547,7 +547,7 @@ void SoftwareTransform::DetectOffsetTexture(int maxIndex) {
}
// NOTE: The viewport must be up to date!
void SoftwareTransform::BuildDrawingParams(int prim, int vertexCount, u32 vertType, u16 *inds, int &indsOffset, int &maxIndex, SoftwareTransformResult *result) {
void SoftwareTransform::BuildDrawingParams(int prim, int vertexCount, u32 vertType, u16 *inds, int &indsOffset, int indexBufferSize, int &maxIndex, SoftwareTransformResult *result) {
TransformedVertex *transformed = params_.transformed;
TransformedVertex *transformedExpanded = params_.transformedExpanded;
bool throughmode = (vertType & GE_VTYPE_THROUGH_MASK) != 0;
@ -560,7 +560,11 @@ void SoftwareTransform::BuildDrawingParams(int prim, int vertexCount, u32 vertTy
bool useBufferedRendering = fbman->UseBufferedRendering();
if (prim == GE_PRIM_RECTANGLES) {
ExpandRectangles(vertexCount, maxIndex, inds, indsOffset, transformed, transformedExpanded, numTrans, throughmode);
if (!ExpandRectangles(vertexCount, maxIndex, inds, indsOffset, indexBufferSize, transformed, transformedExpanded, numTrans, throughmode)) {
result->drawIndexed = false;
result->drawNumTrans = 0;
return;
}
result->drawBuffer = transformedExpanded;
result->drawIndexed = true;
@ -578,11 +582,19 @@ void SoftwareTransform::BuildDrawingParams(int prim, int vertexCount, u32 vertTy
}
}
} else if (prim == GE_PRIM_POINTS) {
ExpandPoints(vertexCount, maxIndex, inds, indsOffset, transformed, transformedExpanded, numTrans, throughmode);
if (!ExpandPoints(vertexCount, maxIndex, inds, indsOffset, indexBufferSize, transformed, transformedExpanded, numTrans, throughmode)) {
result->drawIndexed = false;
result->drawNumTrans = 0;
return;
}
result->drawBuffer = transformedExpanded;
result->drawIndexed = true;
} else if (prim == GE_PRIM_LINES) {
ExpandLines(vertexCount, maxIndex, inds, indsOffset, transformed, transformedExpanded, numTrans, throughmode);
if (!ExpandLines(vertexCount, maxIndex, inds, indsOffset, indexBufferSize, transformed, transformedExpanded, numTrans, throughmode)) {
result->drawIndexed = false;
result->drawNumTrans = 0;
return;
}
result->drawBuffer = transformedExpanded;
result->drawIndexed = true;
} else {
@ -674,7 +686,13 @@ void SoftwareTransform::CalcCullParams(float &minZValue, float &maxZValue) {
std::swap(minZValue, maxZValue);
}
void SoftwareTransform::ExpandRectangles(int vertexCount, int &maxIndex, u16 *inds, int &indsOffset, const TransformedVertex *transformed, TransformedVertex *transformedExpanded, int &numTrans, bool throughmode) {
bool SoftwareTransform::ExpandRectangles(int vertexCount, int &maxIndex, u16 *inds, int &indsOffset, int indexBufferSize, const TransformedVertex *transformed, TransformedVertex *transformedExpanded, int &numTrans, bool throughmode) {
// Before we start, do a sanity check - does the output fit?
if ((vertexCount / 2) * 6 > indexBufferSize - indsOffset) {
// Won't fit, kill the draw.
return false;
}
// Rectangles always need 2 vertices, disregard the last one if there's an odd number.
vertexCount = vertexCount & ~1;
numTrans = 0;
@ -735,9 +753,16 @@ void SoftwareTransform::ExpandRectangles(int vertexCount, int &maxIndex, u16 *in
}
indsOffset = newIndsOffset;
return true;
}
void SoftwareTransform::ExpandLines(int vertexCount, int &maxIndex, u16 *inds, int &indsOffset, const TransformedVertex *transformed, TransformedVertex *transformedExpanded, int &numTrans, bool throughmode) {
bool SoftwareTransform::ExpandLines(int vertexCount, int &maxIndex, u16 *inds, int &indsOffset, int indexBufferSize, const TransformedVertex *transformed, TransformedVertex *transformedExpanded, int &numTrans, bool throughmode) {
// Before we start, do a sanity check - does the output fit?
if ((vertexCount / 2) * 6 > indexBufferSize - indsOffset) {
// Won't fit, kill the draw.
return false;
}
// Lines always need 2 vertices, disregard the last one if there's an odd number.
vertexCount = vertexCount & ~1;
numTrans = 0;
@ -860,9 +885,16 @@ void SoftwareTransform::ExpandLines(int vertexCount, int &maxIndex, u16 *inds, i
}
indsOffset = newIndsOffset;
return true;
}
void SoftwareTransform::ExpandPoints(int vertexCount, int &maxIndex, u16 *inds, int &indsOffset, const TransformedVertex *transformed, TransformedVertex *transformedExpanded, int &numTrans, bool throughmode) {
bool SoftwareTransform::ExpandPoints(int vertexCount, int &maxIndex, u16 *inds, int &indsOffset, int indexBufferSize, const TransformedVertex *transformed, TransformedVertex *transformedExpanded, int &numTrans, bool throughmode) {
// Before we start, do a sanity check - does the output fit?
if (vertexCount * 6 > indexBufferSize - indsOffset) {
// Won't fit, kill the draw.
return false;
}
numTrans = 0;
TransformedVertex *trans = &transformedExpanded[0];
@ -929,4 +961,5 @@ void SoftwareTransform::ExpandPoints(int vertexCount, int &maxIndex, u16 *inds,
}
indsOffset = newIndsOffset;
return true;
}