Not to take away from the valuable information already shared, but isn't using the Win32 API for something as trivial as a line count a little over the top?

Another way to impose a line limit is to use the KeyDown event on the UltraTextEditor. Using this event we can call a method to get the line count, and then allow the key press or suppress it.

C#

private int CountLinesInString(string s)
{
    int count = 1, start = 0;
    while ((start = s.IndexOf('\n', start)) != -1)
    {
        count++;
        start++;
    }
    return count;
}

private const int MAXLINES = 6;
private void txtEditor_KeyDown(object sender, KeyEventArgs e)
{
    if (e.KeyCode == Keys.Back)
        return;

    String PendingValue = txtEditor.Text + (char)e.KeyValue;
    if (CountLinesInString(PendingValue) > MAXLINES)
        e.SuppressKeyPress = true;
}

VB

Private Function CountLinesInString(s As String) As Integer
	Dim count As Integer = 1, start As Integer = 0
	While (InlineAssignHelper(start, s.IndexOf(ControlChars.Lf, start))) <> -1
		count += 1
		start += 1
	End While
	Return count
End Function

Private Const MAXLINES As Integer = 6
Private Sub txtEditor_KeyDown(sender As Object, e As KeyEventArgs)
	If e.KeyCode = Keys.Back Then
		Return
	End If

	Dim PendingValue As [String] = txtEditor.Text + CChar(e.KeyValue)
	If CountLinesInString(PendingValue) > MAXLINES Then
		e.SuppressKeyPress = True
	End If
End Sub