Imports System.Net
Imports System.IO

Public Class GoogleSearchResultCrawler

    Private m_threadStop As Boolean = False
    Private m_crawlerState As CrawlerStatusEventArgs

    Private m_failedQueries As Integer = 0
    Private m_templateCount As Integer
    Private m_queriesDone As Integer = 0
    Private m_urlCount As Integer = 0
    Private m_newUrls As Integer = 0
    Private m_duplicates As Integer = 0
    Private m_startTime As Date
    Private m_stopTime As Date
    Private m_prItems As Integer = 0
    Private m_prPoints As Integer = 0
    Private m_isFirstTime As Boolean = True

    Public Event StatusUpdate(sender As Object, e As CrawlerStatusEventArgs)
    Public Event Completed(sender As Object, e As CrawlerCompletedEventArgs)

    Public Property Results() As List(Of LinkSite)
    Public Property Queries() As List(Of SearchQuery)

    Public Sub New()
        m_crawlerState = New CrawlerStatusEventArgs()
        Results = New List(Of LinkSite)()
        Queries = New List(Of SearchQuery)()
    End Sub

    Public Sub StartCrawl()
        Dim t As New Threading.Thread(AddressOf CollectData)
        m_templateCount = Queries.GroupBy(Function(q) q.Template).Count()
        m_queriesDone = 0
        m_urlCount = 0
        m_newUrls = 0
        m_duplicates = 0
        m_startTime = Date.Now
        Binarypros.Report.AddEntry(String.Format("Beginning Backlinks Warrior Search at: {0:M/d/yyyy h:mm:ss tt}", DateTime.Now), FontStyle.Bold)
        Binarypros.Report.AddEntry("Refreshing Queries...", FontStyle.Bold)
        Binarypros.Report.AddEntry("-")
        Binarypros.Report.AddEntry("")
        m_isFirstTime = True
        m_threadStop = False
        t.Start()
    End Sub

    Public Sub StopCrawl()
        m_threadStop = True
        Binarypros.Report.AddEntry("User Clicked Stop!", Color.Red, FontStyle.Bold)
        m_stopTime = Date.Now
    End Sub

    Private Sub CollectData()
        Dim web As New BPWebClient()
        Dim url As String


        For Each query In Queries
            query.Queried = True
            Dim googlePage As New GoogleSearchPage()
            Dim pageResults As New List(Of SearchEngineResult)

            If My.Settings.CourtesyDelay AndAlso Not m_isFirstTime Then
                m_crawlerState.IsWaiting = True
                RaiseUpdateEvent()
                Threading.Thread.Sleep(60000)
                m_crawlerState.IsWaiting = False
            End If
            m_isFirstTime = False
            url = "http://www.google.com/search?num=100&hl=en&safe=off&meta=&q=" + query.ToQueryValue()

            If My.Settings.ForceUS = True Then
                url += "&gl=us"
            End If

            Try
                Dim res = googlePage.ParseFromHTML(web.DownloadString(url), 0)
                If Not res Then
                    Continue For ' Move on to the next query because this one failed to parse
                End If
                m_failedQueries = 0
            Catch ex As Net.WebException

                ' Getting the response text doesn't seem to work for some reason, so just check for a 503
                If CType(ex.Response, HttpWebResponse).StatusCode = HttpStatusCode.ServiceUnavailable Then
                    MessageBox.Show("Google has temporarily blocked your computer from making automated searches. Please try again in a few hours. To avoid this problem in the future it is best to search a smaller keyword list at one time and allow more time between multiple searches.", "Automated Search Limit Exceeded", MessageBoxButtons.OK, MessageBoxIcon.Warning)
                    Binarypros.Report.AddEntry("Search interrupted - automated query limit exceeded. Please try again later.", Color.Red, FontStyle.Bold)
                    RaiseCompletedEvent(CrawlerCompletedState.Failed, CrawlerError.QueryLimitExceeded)
                    Exit Sub
                End If

                ' If we have 3 unknown failures in a row stop it
                If m_failedQueries < 3 Then
                    m_failedQueries += 1
                    Continue For
                End If

                Binarypros.Report.AddEntry("Error During Data Collection: " + ex.Message, Color.Red, FontStyle.Bold)
                MessageBox.Show("Error collection data: " + ex.Message + Environment.NewLine + "Please ensure that your Internet connection is working and that any firewall software is set to allow this application.", "Data Collection Error", MessageBoxButtons.OK, MessageBoxIcon.Error)
                RaiseCompletedEvent(CrawlerCompletedState.Failed, CrawlerError.UnKnown)
                Exit Sub
            End Try

            pageResults.AddRange(googlePage.Results)

            For Each result In pageResults
                Dim r = New LinkSite With {.PageUrl = result.URL, .Keyword = query.Keyword.Keyword, .Notes = "Found at: " & Date.Now, .Status = LinkSiteStatus.New}

                ' Disable Google PR
                'If My.Settings.PREnabled Then
                '    ' PageRanks doesn't seem to like these urls... so just skip them
                '    If Not result.URL.Contains("%") AndAlso Not result.URL.Contains(" ") AndAlso Not result.URL.Contains("+") AndAlso Not result.URL.Contains("N/A") Then
                '        r.PageRank = GooglePageRank.GetGooglePR(result.URL)
                '        If r.PageRank > 0 Then
                '            m_prPoints += r.PageRank
                '            m_prItems += 1
                '        End If
                '    End If
                'End If

                m_urlCount += 1
                If Me.Results.Contains(r) Then
                    Dim oldR = Me.Results.Where(Function(res) res.PageUrl = r.PageUrl).FirstOrDefault()
                    m_duplicates += 1
                    oldR.PageRank = r.PageRank
                Else
                    m_newUrls += 1
                    Me.Results.Add(r)
                End If

                RaiseUpdateEvent()
            Next

            m_queriesDone += 1
            RaiseUpdateEvent()

            If m_threadStop Then
                RaiseCompletedEvent(CrawlerCompletedState.UserStopped)
                Binarypros.Report.AddEntry("Stop Complete!", Color.Red, FontStyle.Bold)
                Exit Sub
            End If

        Next

        RaiseCompletedEvent(CrawlerCompletedState.Success)
        'RaiseEvent Completed(Me, New CrawlerCompletedEventArgs With {.Status = CrawlerCompletedState.Success})
    End Sub

    Private Sub RaiseUpdateEvent()
        m_crawlerState.UrlsInspected = m_urlCount
        m_crawlerState.KeywordsLeft = Queries.Where(Function(q) Not q.Queried).Select(Function(q) q.Keyword).Distinct().Count()
        RaiseEvent StatusUpdate(Me, m_crawlerState)
    End Sub

    Private Sub RaiseCompletedEvent(status As CrawlerCompletedState)
        
        RaiseCompletedEvent(status, CrawlerError.NoError)
    End Sub

    Private Sub RaiseCompletedEvent(status As CrawlerCompletedState, err As CrawlerError)
        Dim c = New CrawlerCompletedEventArgs With {
            .Status = status,
            .TotalUrls = m_urlCount,
            .NewUrls = m_newUrls,
            .DuplicateItems = m_duplicates,
            .StartTime = m_startTime,
            .EndTime = DateTime.Now,
            .ErrorType = err
        }

        Binarypros.Report.AddEntry("-")
        Binarypros.Report.AddEntry("")
        Binarypros.Report.AddEntry("Inspection Summary:", FontStyle.Bold)
        Binarypros.Report.AddEntry("-")
        Binarypros.Report.AddEntry(String.Format("{0:###,###,##0} total pages were inspected!", m_urlCount))
        Binarypros.Report.AddEntry(String.Format("{0:###,###,##0} new and unique URLs found!", m_newUrls))
        Binarypros.Report.AddEntry(String.Format("{0:###,###,##0} duplicate items were found!", m_duplicates))

        ' Disable google PR
        'If My.Settings.PREnabled Then
        '    Binarypros.Report.AddEntry(String.Format("{0:###,###,##0} URLS that had a PageRank were found for a total of {1:###,###,##0} PR Points", m_prItems, m_prPoints))
        'End If

        Binarypros.Report.AddEntry("-")
        Binarypros.Report.AddEntry("")
        Binarypros.Report.AddEntry("")
        Binarypros.Report.AddEntry(String.Format("Inspection Completed at: {0:M/dd/yyyy h:mm:ss tt}", DateTime.Now), FontStyle.Bold)

        RaiseEvent Completed(Me, c)
    End Sub

    Private Function ResponseText(ex As WebException) As String
        Using stream As Stream = ex.Response.GetResponseStream
            Using reader As New StreamReader(stream)
                Return reader.ReadToEnd().Trim()
            End Using
        End Using
    End Function

End Class
  
