Try this:
Global CelTexture
Type CelShaded
Field Mesh,OutLine
End Type
Function LinearGradient (Shades,Dir)
Local Steps
Repeat
If Dir = 0
Color 255-Steps*(255/Shades),255-Steps*(255/Shades),255-Steps*(255/Shades)
Else
Color Steps*(255/Shades),Steps*(255/Shades),Steps*(255/Shades)
EndIf
Steps = Steps + 1
Rect 0,-256/Shades+Steps*256/Shades,256,256/Steps,True
Until Steps = Shades
End Function
Function CelShader (Shades=3)
Local CD.CelShaded
If Shades < 1 Then Return
Local Camera = CreateCamera()
Local Light = CreatePivot()
Local Sphere = CreateSphere (12)
Local Texture = CreateTexture (1,256,16+32)
For CD.CelShaded = Each CelShaded
DeleteCelShaded (CD)
Next
If CelTexture <> 0
FreeTexture CelTexture
CelTexture = 0
EndIf
CelTexture = CreateTexture (256,256,16+32+64+256)
LinearGradient (Shades,0)
CopyRect 0,0,256,256,0,0,BackBuffer(),TextureBuffer(Texture)
Planer (Sphere)
EntityFX Sphere,1
RotateMesh Sphere,90,0,0
PositionEntity Sphere,0,65536,65536;;;;;;;;;;;;;;;;;;;
EntityTexture Sphere,Texture
PositionEntity Light,0,65536+65536,65536-16384;;;;;;;;;
PointEntity Sphere,Light
CameraZoom Camera,2
CameraViewport Camera,0,0,256,256
PositionEntity Camera,0,65536,65536-2.21
Cls
RenderWorld
CopyRect 0,0,256,256,0,0,BackBuffer(),TextureBuffer(CelTexture)
FreeTexture Texture
FreeEntity Camera
FreeEntity Light
FreeEntity Sphere
Cls
End Function
Function CelShade (Mesh,Index=0,Scale#=0,Shade#=0)
Local A,B
Local CD.CelShaded = New CelShaded
CD\Mesh = Mesh
If Scale <> 0
CD\OutLine = CopyMesh (Mesh,Mesh)
UpdateNormals CD\OutLine
For A = 1 To CountSurfaces (CD\OutLine)
Local Surface = GetSurface (CD\OutLine,A)
Local Vertices = CountVertices (Surface)-1
For B = 0 To Vertices-1
Local VX# = VertexX (Surface,B)
Local VY# = VertexY (Surface,B)
Local VZ# = VertexZ (Surface,B)
Local VNX# = VertexNX (Surface,B)
Local VNY# = VertexNY (Surface,B)
Local VNZ# = VertexNZ (Surface,B)
If (VNX<-1000) Or (VNX>1000) Then VNX = 0
If (VNY<-1000) Or (VNY>1000) Then VNY = 0
If (VNZ<-1000) Or (VNZ>1000) Then VNZ = 0
VX = VX + (VNX*Scale)
VY = VY + (VNY*Scale)
VZ = VZ + (VNZ*Scale)
VertexCoords Surface,B,VX,VY,VZ
Next
Next
FlipMesh CD\OutLine
EntityFX CD\OutLine,9
EntityColor CD\OutLine,Shade,Shade,Shade
NameEntity CD\OutLine,"Outline"+Handle(CD)
EndIf
If Index = 0
EntityTexture Mesh,CelTexture,0,1
EntityFX Mesh,1
Else
Local Brush = CreateBrush()
Surface = GetSurface (Mesh,Index)
BrushFX Brush,1
BrushTexture Brush,CelTexture,0,1
PaintSurface Surface,Brush
FreeBrush Brush
EndIf
End Function
Function AnimatedCelShade (Mesh,Scale#=0,Shade#=0)
Local A
If EntityClass (Mesh) = "Mesh"
CelShade (Mesh,0,Scale,Shade)
EndIf
Local Children = CountChildren (Mesh)
For A = 1 To Children
Local Child = GetChild (Mesh,A)
If Instr (EntityName(Child),"Outline") = 0
AnimatedCelShade (Child,Scale,Shade)
EndIf
Next
End Function
Function Planer (Mesh)
Local A,B
Local Height# = MeshHeight (Mesh)
Local Surfaces = CountSurfaces (Mesh)
For A = 1 To Surfaces
Local Surface = GetSurface (Mesh,A)
Local Vertices = CountVertices(Surface)
For B = 0 To Vertices-1
Local Y# = (VertexY(Surface,B)/Height) + 0.5
VertexTexCoords Surface,B,0.5,1.0-Y
Next
Next
End Function
Function DeleteCelShaded (CD.CelShaded)
FreeEntity CD\OutLine
Delete CD
End Function
This is a more advanced version of something I used for my Bongo game. It's based on some code by sswift.
Call the CelShader() function to setup cel shading (or change the number of shades your using) then use CelShade() or AnimatedCelShade() functions to apply it to your models.
For debugging, just call the LinearGradient() function after RenderWorld to see the texture used in the Cel Shading functions. Hope that all makes sense, been awhile since I used it :)