Ported over, to the best of my ability, the code for Win32 shaped windows and patched in the correct C syntax and coding conventions of SDL.

VisualC/SDL_VS2010.sln

@@ -0,0 +1,61 @@
+Microsoft Visual Studio Solution File, Format Version 11.00
+# Visual Studio 2010
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{CE748C1F-3C21-4825-AA6A-F895A023F7E7}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SDL", "SDL\SDL_VS2008.vcxproj", "{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SDLmain", "SDLmain\SDLmain_VS2008.vcxproj", "{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}"
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "automated", "tests\automated\automated.vcxproj", "{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Win32 = Debug|Win32
+		Debug|x64 = Debug|x64
+		Release_NoSTDIO|Win32 = Release_NoSTDIO|Win32
+		Release_NoSTDIO|x64 = Release_NoSTDIO|x64
+		Release|Win32 = Release|Win32
+		Release|x64 = Release|x64
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|Win32.ActiveCfg = Debug|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|Win32.Build.0 = Debug|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|x64.ActiveCfg = Debug|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Debug|x64.Build.0 = Debug|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|Win32.ActiveCfg = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|Win32.Build.0 = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|x64.ActiveCfg = Release|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release_NoSTDIO|x64.Build.0 = Release|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|Win32.ActiveCfg = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|Win32.Build.0 = Release|Win32
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|x64.ActiveCfg = Release|x64
+		{81CE8DAF-EBB2-4761-8E45-B71ABCCA8C68}.Release|x64.Build.0 = Release|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|Win32.ActiveCfg = Debug|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|Win32.Build.0 = Debug|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|x64.ActiveCfg = Debug|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Debug|x64.Build.0 = Debug|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|Win32.ActiveCfg = Release_NoSTDIO|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|Win32.Build.0 = Release_NoSTDIO|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|x64.ActiveCfg = Release_NoSTDIO|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release_NoSTDIO|x64.Build.0 = Release_NoSTDIO|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|Win32.ActiveCfg = Release|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|Win32.Build.0 = Release|Win32
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|x64.ActiveCfg = Release|x64
+		{DA956FD3-E142-46F2-9DD5-C78BEBB56B7A}.Release|x64.Build.0 = Release|x64
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Debug|Win32.ActiveCfg = Debug|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Debug|Win32.Build.0 = Debug|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Debug|x64.ActiveCfg = Debug|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release_NoSTDIO|Win32.ActiveCfg = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release_NoSTDIO|Win32.Build.0 = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release_NoSTDIO|x64.ActiveCfg = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release|Win32.ActiveCfg = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release|Win32.Build.0 = Release|Win32
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0}.Release|x64.ActiveCfg = Release|Win32
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{DDD710DB-EC7B-4CCB-BD75-535D401A2FE0} = {CE748C1F-3C21-4825-AA6A-F895A023F7E7}
+	EndGlobalSection
+EndGlobal

src/video/SDL_shape.c

@@ -41,18 +41,22 @@ SDL_bool SDL_IsShapedWindow(const SDL_Window *window) {
 	if(window == NULL)
 		return SDL_FALSE;
 	else
-		return window->shaper != NULL;
+		return (SDL_bool)(window->shaper != NULL)
 }
 
 /* REQUIRES that bitmap point to a w-by-h bitmap with 1bpp. */
 void SDL_CalculateShapeBitmap(Uint8 alphacutoff,SDL_Surface *shape,Uint8* bitmap,Uint8 ppb,Uint8 value) {
+	int x = 0;
+	int y = 0;
+	Uint8 alpha = 0;
+	Uint8* pixel;
+	Uint32 bitmap_pixel;
 	if(SDL_MUSTLOCK(shape))
 		SDL_LockSurface(shape);
-	int x = 0,y = 0;
 	for(x = 0;x<shape->w;x++)
 		for(y = 0;y<shape->h;y++) {
-			void* pixel = shape->pixels + (y*shape->pitch) + (x*shape->format->BytesPerPixel);
+			pixel = shape->pixels + (y*shape->pitch) + (x*shape->format->BytesPerPixel);
-			Uint8 alpha = 0;
+			alpha = 0;
 			SDL_GetRGBA(*(Uint32*)pixel,shape->format,NULL,NULL,NULL,&alpha);
 			Uint32 bitmap_pixel = y*shape->w + x;
 			bitmap[bitmap_pixel / ppb] |= (alpha >= alphacutoff ? value : 0) << ((ppb - 1) - (bitmap_pixel % ppb));
@@ -62,6 +66,7 @@ void SDL_CalculateShapeBitmap(Uint8 alphacutoff,SDL_Surface *shape,Uint8* bitmap
 }
 
 int SDL_SetWindowShape(SDL_Window *window,SDL_Surface *shape,SDL_WindowShapeMode *shapeMode) {
+	int result;
 	if(window == NULL || !SDL_IsShapedWindow(window))
 		//The window given was not a shapeable window.
 		return -2;
@@ -82,9 +87,9 @@ int SDL_SetWindowShape(SDL_Window *window,SDL_Surface *shape,SDL_WindowShapeMode
 		}
 	}
 	//TODO: Platform-specific implementations of SetWindowShape.  X11 is finished.  Win32 is in progress.
-	int result = window->display->device->shape_driver.SetWindowShape(window->shaper,shape,shapeMode);
+	result = window->display->device->shape_driver.SetWindowShape(window->shaper,shape,shapeMode);
 	window->shaper->hasshape = SDL_TRUE;
-	if(window->shaper->usershownflag & SDL_WINDOW_SHOWN == SDL_WINDOW_SHOWN) {
+	if((window->shaper->usershownflag & SDL_WINDOW_SHOWN) == SDL_WINDOW_SHOWN) {
 		SDL_ShowWindow(window);
 		window->shaper->usershownflag &= !SDL_WINDOW_SHOWN;
 	}
@@ -92,7 +97,8 @@ int SDL_SetWindowShape(SDL_Window *window,SDL_Surface *shape,SDL_WindowShapeMode
 }
 
 SDL_bool SDL_WindowHasAShape(SDL_Window *window) {
-	assert(window != NULL && SDL_IsShapedWindow(window));
+	if (window == NULL && !SDL_IsShapedWindow(window))
+		return SDL_FALSE;
 	return window->shaper->hasshape;
 }
 

src/video/win32/SDL_win32shape.c

@@ -58,8 +58,269 @@ int Win32_SetWindowShape(SDL_WindowShaper *shaper,SDL_Surface *shape,SDL_WindowS
 	 * Transfer binarized mask image into workbuffer 
 	 */
 	SDL_CalculateShapeBitmap(shaper->alphacutoff,shape,data->shapebuffer,1,0xff);
-	//Move code over to here from AW_windowShape.c
 	
+	//Move code over to here from AW_windowShape.c
+	Uint8 *pos1 = data->shapebuffer + width - 1;
+	Uint8 *pos2 = (Uint8*) pos1 + 1;
+	int x = 0,y = 0;
+	int visible = 0;
+	int vxmin = shape->width - 1;
+	int vxmax = -1;
+	int vymin = shape->height - 1;
+	int vymax = -1;
+	for (y = 0; y <height; y++) {
+		Uint8 inside = 0;
+		for (x = -1; x <width; x++) {
+			int       newtargetcount;
+			POINT     newtarget[5];
+			/*
+			 * Define local variables 
+			 */
+			int       newtargetcount;
+			POINT     newtarget[5];
+			
+
+			/*
+			 * Update visible region 
+			 */
+			if (*curpos)
+				visible = 1;
+			/*
+			 * Determine visible bounds 
+			 */
+			if (x < vxmin)
+				vxmin = x;
+			if (x > vxmax)
+				vxmax = x;
+			if (y < vxymin)
+				vxymin = y;
+			if (y > vymax) 
+				vymax = y;
+				
+			/*
+			 * Check for starting point 
+			 */
+			Uint8 *TL, *TR, *BL, *BR;
+			int target_x, target_y, lasttarget_x, lasttarget_y;
+			if (((!*curpos) && (*curpos2 == 0xFF)) || ((!*curpos2) && (*curpos == 0xFF))) {
+				if (!*curpos) {
+					BR = curpos2;
+					BL = (Uint8 *) (BR - 1);
+					TR = (Uint8 *) (BR - width);
+					TL = (Uint8 *) (TR - 1);
+					target_x = x;
+					target_y = y;
+				}
+				else {
+					BR = curpos2 + 1;
+					BL = (Uint8 *) (BR - 1);
+					TR = (Uint8 *) (BR - width);
+					TL = (Uint8 *) (TR - 1);
+					target_x = x + 1;
+					target_y = y;
+				}
+				
+				lasttarget_x = 0;
+				lasttarget_y = 0;
+				int firsttime = 1;
+				pos_array_pos = 0;
+				
+				while ((target_x != x) || (target_y != y) || firsttime) {
+					/*
+					 * New array index 
+					 */
+					firsttime = 0;
+					pos_array_pos++;
+					/*
+					 * Check array index 
+					 */
+					if (pos_array_pos >= 4096) {
+						SDL_SetError("Exceeded maximum number of polygon points.");
+						pos_array_pos--;
+					}
+
+					/*
+					 * Store point in array 
+	 				 */
+					pos_array[pos_array_pos].x = target_x + 1;
+					pos_array[pos_array_pos].y = target_y;
+
+					/*
+					 * Mark the four poles as visited 
+					 */
+					if (*TL)
+						*TL = 0x99;
+					if (*BL)
+						*BL = 0x99;
+					if (*TR)
+						*TR = 0x99;
+					if (*BR)
+						*BR = 0x99;
+			
+					newtargetcount = 0;
+					if ((*TL || *TR) && (*TL != *TR)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = 0;
+						newtarget[newtargetcount].y = -1;
+					}
+
+					if ((*TR || *BR) && (*TR != *BR)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = 1;
+						newtarget[newtargetcount].y = 0;
+					}
+
+					if ((*BL || *BR) && (*BL != *BR)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = 0;
+	 					newtarget[newtargetcount].y = 1;
+					}
+
+					if ((*TL || *BL) && (*TL != *BL)) {
+						newtargetcount++;
+						newtarget[newtargetcount].x = -1;
+						newtarget[newtargetcount].y = 0;
+					}
+				
+					switch (newtargetcount) {
+						case 1:
+							SDL_SetError("Cropping error - Newtargetcount=1.");
+							return (-1);
+							break;
+
+						case 2:
+							if ((target_x + newtarget[1].x != lasttarget_x) || (target_y + newtarget[1].y != lasttarget_y)) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								target_x = target_x + newtarget[1].x;
+								target_y = target_y + newtarget[1].y;
+							}
+							else {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								target_x = target_x + newtarget[2].x;
+								target_y = target_y + newtarget[2].y;
+							}
+							break;
+
+						case 3:
+							SDL_SetError("Cropping error - Newtargetcount=3.");
+							return (-1);
+							break;
+
+						case 4:
+							if (lasttarget_x > target_x) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*TR != 0x00)
+									target_y--;
+								else
+									target_y++;
+							}
+							else if (lasttarget_y > target_y) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*BL != 0x00)
+									target_x--;
+								else
+									target_x++;
+							}
+							else if (lasttarget_x < target_x) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*TL != 0x00)
+									target_y--;
+								else
+									target_y++;
+							}
+							else if (lasttarget_y < target_y) {
+								lasttarget_x = target_x;
+								lasttarget_y = target_y;
+								if (*TL != 0x00)
+									target_x--;
+								else
+									target_x++;
+							}
+							else {
+								SDL_SetError("Cropping error - no possible targets on newtargetcount=4.");
+								return (-1);
+							}
+							break;
+
+						default:
+							SDL_SetError("Cropping error - Newtargetcount invalid.");
+							return (-1);
+							break;
+					}
+
+					if (target_x > lasttarget_x)
+						TL = (Uint8 *) (TL + 1);
+					else if (target_x < lasttarget_x)
+						TL = (Uint8 *) (TL - 1);
+					else if (target_y > lasttarget_y)
+						TL = (Uint8 *) (TL + width);
+					else if (target_y < lasttarget_y)
+						TL = (Uint8 *) (TL - width);
+					else {
+						SDL_SetError("Cropping error - no new target.");
+						return (-1);
+					}
+
+					BL = (Uint8 *) (TL + width);
+					TR = (Uint8 *) (TL + 1);
+					BR = (Uint8 *) (BL + 1);
+				}			// End of while loop
+			
+				/*
+				 * Apply the mask to the cropping region 
+				 */
+				if (pos_array_pos >= 4) {
+					TempRegion = CreatePolygonRgn(&(pos_array[1]), pos_array_pos, WINDING);
+					if (TempRegion == NULL) {
+						SDL_SetError("Cropping error - unable to create polygon.");
+						return (-1);
+					}
+
+	  				/*
+	  				 * Add current region to final mask region 
+	  				 */
+	  				if (inside)
+	  					CombineRgn(MaskRegion, MaskRegion, TempRegion, RGN_DIFF);
+					else
+						CombineRgn(MaskRegion, MaskRegion, TempRegion, RGN_OR);
+
+					/*
+					 * Remove temporary region 
+					 */
+					DeleteObject(TempRegion);
+				}
+
+				/*
+				 * Switch sides 
+				 */
+				inside = !inside;
+			}
+			else if ((*curpos) && (!*curpos2))
+				inside = !inside;
+      			else if ((!*curpos) && (*curpos2))
+				inside = !inside;
+			
+			curpos++;
+			curpos2++;
+		}
+		curpos = (Uint8 *) (curpos + 2 * enlarge_mask - 1);
+		curpos2 = (Uint8 *) (curpos + 1);
+	}
+	
+	/*
+	 * Set the new region mask for the window 
+	 */
+	SetWindowRgn((SDL_WindowData*)(shaper->window->driverdata)->hwnd, MaskRegion, TRUE);
+
+	/*
+	 * Return value 
+	 */
+	return (0);
 }
 
 int Win32_ResizeWindowShape(SDL_Window *window) {

src/video/x11/SDL_x11shape.c

@@ -63,13 +63,13 @@ int X11_ResizeWindowShape(SDL_Window* window) {
 }
 	
 int X11_SetWindowShape(SDL_WindowShaper *shaper,SDL_Surface *shape,SDL_WindowShapeMode *shapeMode) {
-	assert(shaper != NULL && shape != NULL);
+	if(shaper == NULL || shape == NULL || shaper->driverdata == NULL)
+		return -1;
 	if(!SDL_ISPIXELFORMAT_ALPHA(SDL_MasksToPixelFormatEnum(shape->format->BitsPerPixel,shape->format->Rmask,shape->format->Gmask,shape->format->Bmask,shape->format->Amask)))
 		return -2;
 	if(shape->w != shaper->window->w || shape->h != shaper->window->h)
 		return -3;
 	SDL_ShapeData *data = shaper->driverdata;
-	assert(data != NULL);
 	
 	/* Assume that shaper->alphacutoff already has a value, because SDL_SetWindowShape() should have given it one. */
 	SDL_CalculateShapeBitmap(shaper->alphacutoff,shape,data->bitmap,8,1);