feat: scrapping form threads

Author: dd84ai

app/exposer/exposer_test.go

@@ -30,7 +30,7 @@ func FixtureTestWebServer() {
 
 func testQuery(url string) (string, error) {
 	resp, err := http.Get("http://localhost:8080" + url)
-	logus.CheckFatal(err, "query failed")
+	logus.CheckError(err, "query failed")
 	defer resp.Body.Close()
 	body, err := io.ReadAll(resp.Body)
 	return string(body), err

app/scrappy/forum/detailed_post.go

@@ -0,0 +1,83 @@
+package forum
+
+import (
+	"darkbot/app/settings/logus"
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/anaskhan96/soup"
+)
+
+type DetailedPost struct {
+	*LatestThread
+	PostID            PostID
+	PostContent       PostContent
+	PostPermamentLink PostPermamentLink
+
+	requester func(MethodType, Url) (*QueryResult, error)
+}
+type PostID string
+type PostContent string
+type PostPermamentLink Url
+
+type detailedPostParam func(detailedPost *DetailedPost)
+
+func WithMockedRequester(
+	requester func(MethodType, Url) (*QueryResult, error),
+) detailedPostParam {
+	return func(detailedPost *DetailedPost) {
+		detailedPost.requester = requester
+	}
+}
+
+func NewDetailedPost(thread *LatestThread, opts ...detailedPostParam) (*DetailedPost, error) {
+	detailed_post := &DetailedPost{requester: NewQuery}
+	for _, opt := range opts {
+		opt(detailed_post)
+	}
+
+	query, err := detailed_post.requester(GET, thread.ThreadLink.GetUrl())
+	if logus.CheckError(err, "failed to query ThreadLink.GetUrl()="+string(thread.ThreadLink)) {
+		return nil, err
+	}
+
+	doc := soup.HTMLParse(query.GetContent())
+	params, _ := url.ParseQuery(query.ResponseRawQuery)
+	post_id := PostID(params["pid"][0])
+
+	forum := doc.Find("div", "id", "forum")
+	if logus.CheckError(forum.Error, "failed to get forum object") {
+		return nil, forum.Error
+	}
+	thread_header := forum.Find("td", "class", "thead")
+	if logus.CheckError(thread_header.Error, "failed to get thread_header object") {
+		return nil, thread_header.Error
+	}
+	thread_name := thread_header.FullText()
+	thread_name = strings.ReplaceAll(thread_name, "\n", "")
+	thread_name = strings.ReplaceAll(thread_name, "\t", "")
+	logus.Debug("thread_name=" + thread_name)
+
+	post := doc.Find("table", "id", fmt.Sprintf("post_%s", string(post_id)))
+	if logus.CheckError(post.Error, "failed to get post object") {
+		return nil, post.Error
+	}
+	post_body := post.Find("div", "class", "post_body")
+	if logus.CheckError(post_body.Error, "failed to get post_body object") {
+		return nil, post_body.Error
+	}
+	post_content := post_body.FullText()
+
+	post_content = strings.ReplaceAll(post_content, "\t", "")
+	for i := 0; i < 5; i++ {
+		post_content = strings.ReplaceAll(post_content, "\n\n", "\n")
+	}
+
+	detailed_post.LatestThread = thread
+	detailed_post.PostID = post_id
+	detailed_post.PostContent = PostContent(post_content)
+	detailed_post.PostPermamentLink = PostPermamentLink(query.ResponseFullUrl)
+
+	return detailed_post, nil
+}

app/scrappy/forum/detailed_post_test.go

@@ -0,0 +1,45 @@
+package forum
+
+import (
+	"darkbot/app/settings/logus"
+	"darkbot/app/settings/utils"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func FixtureLatestThread() *LatestThread {
+	return &LatestThread{
+		ThreadLink:     "https://discoverygc.com/forums/showthread.php?tid=200175&action=lastpost",
+		ThreadName:     "To: NNroute.../(BDM-Direk...",
+		LastUpdated:    "11-20-2023, 09:35 AM",
+		PostAuthorLink: "https://discoverygc.com/forums/member.php?action=profile&uid=54754",
+		PostAuthorName: "Civil Servant",
+	}
+}
+
+func TestGetDetailedPost(t *testing.T) {
+	thread := FixtureLatestThread()
+
+	detailed_post_content_filepath := filepath.Join(utils.GetCurrrentFolder(), "test_data", "detailed_post_content.html")
+	if _, err := os.Stat(detailed_post_content_filepath); err != nil {
+		query, err := NewQuery("GET", "https://discoverygc.com/forums/showthread.php?tid=200175&action=lastpost")
+		logus.CheckFatal(err, "failed to create mock data")
+		os.WriteFile(detailed_post_content_filepath, []byte(query.GetContent()), 0644)
+	}
+	detailed_post_content, _ := os.ReadFile(detailed_post_content_filepath)
+	mocked_requester := func(mt MethodType, u Url) (*QueryResult, error) {
+		return &QueryResult{
+			content:          string(detailed_post_content),
+			ResponseRawQuery: `tid=200175&pid=2315295`,
+			ResponseFullUrl:  `https://discoverygc.com/forums/showthread.php?tid=200175&pid=2315295#pid2315295`,
+		}, nil
+	}
+	detailed_post, err := NewDetailedPost(thread, WithMockedRequester(mocked_requester))
+	_ = detailed_post
+	fmt.Println("err=", err)
+	assert.Nil(t, err, "expected error to be nil")
+}

app/scrappy/forum/latest_threads.go

@@ -0,0 +1,108 @@
+package forum
+
+import (
+	"darkbot/app/settings/logus"
+	"fmt"
+	"net/url"
+
+	"github.com/anaskhan96/soup"
+)
+
+type Url string
+
+type ThreadLink Url
+
+func (u ThreadLink) GetUrl() Url { return Url(u) }
+
+type ThreadShortName string
+type ThreadID string
+type ForumTimestamp string
+
+type PostAuthorLink Url
+
+func (u PostAuthorLink) GetUrl() Url { return Url(u) }
+
+type PostAuthorName string
+
+type LatestThread struct {
+	ThreadLink     ThreadLink
+	ThreadName     ThreadShortName
+	ThreadID       ThreadID
+	LastUpdated    ForumTimestamp
+	PostAuthorLink PostAuthorLink
+	PostAuthorName PostAuthorName
+}
+
+type ThreadsPage struct {
+	Threads   []LatestThread
+	requester func(MethodType, Url) (*QueryResult, error)
+}
+
+type threadPageParam func(thread_page *ThreadsPage)
+
+func WithMockedPageRequester(
+	requester func(MethodType, Url) (*QueryResult, error),
+) threadPageParam {
+	return func(thread_page *ThreadsPage) {
+		thread_page.requester = requester
+	}
+}
+
+const ThreadPageURL Url = "https://discoverygc.com/forums/portal.php"
+
+func GetLatestThreads(opts ...threadPageParam) (*ThreadsPage, error) {
+	thread_page := &ThreadsPage{
+		requester: NewQuery,
+	}
+	for _, opt := range opts {
+		opt(thread_page)
+	}
+
+	query, err := thread_page.requester(GET, ThreadPageURL)
+	if logus.CheckError(err, "Failed to make query") {
+		return nil, err
+	}
+
+	content := query.GetContent()
+	doc := soup.HTMLParse(content)
+	forum_posts := doc.FindAll("tr", "class", "latestthreads_portal")
+
+	for _, forum_post := range forum_posts {
+		thread := forum_post.Find("strong").Find("a")
+		if logus.CheckError(thread.Error, "failed to get thread object") {
+			return nil, thread.Error
+		}
+
+		thread_link := thread.Attrs()["href"]
+		thread_name := thread.Text()
+		span_section := forum_post.Find("span")
+		if logus.CheckError(span_section.Error, "failed to get span_section object") {
+			return nil, span_section.Error
+		}
+
+		forum_timestamp := span_section.Find("span").Attrs()["title"]
+
+		author := span_section.Find("a")
+		if logus.CheckError(author.Error, "failed to get author object") {
+			return nil, author.Error
+		}
+		author_link := author.Attrs()["href"]
+		author_name := author.Text()
+
+		myUrl, _ := url.Parse(thread_link)
+		params, _ := url.ParseQuery(myUrl.RawQuery)
+
+		latest_thread := LatestThread{
+			ThreadLink:     ThreadLink(thread_link),
+			ThreadName:     ThreadShortName(thread_name),
+			LastUpdated:    ForumTimestamp(forum_timestamp),
+			PostAuthorLink: PostAuthorLink(author_link),
+			PostAuthorName: PostAuthorName(author_name),
+			ThreadID:       ThreadID(params["tid"][0]),
+		}
+		thread_page.Threads = append(thread_page.Threads, latest_thread)
+
+		logus.Debug(fmt.Sprintf("latest_thread=%v", latest_thread))
+	}
+	return thread_page, nil
+}

app/scrappy/forum/latest_threads_test.go

@@ -0,0 +1,32 @@
+package forum
+
+import (
+	"darkbot/app/settings/logus"
+	"darkbot/app/settings/utils"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetPosts(t *testing.T) {
+	thread_post_content_filepath := filepath.Join(utils.GetCurrrentFolder(), "test_data", "latest_threads.html")
+	if _, err := os.Stat(thread_post_content_filepath); err != nil {
+		query, err := NewQuery("GET", ThreadPageURL)
+		logus.CheckFatal(err, "failed to create mock data")
+		os.WriteFile(thread_post_content_filepath, []byte(query.GetContent()), 0644)
+	}
+	thread_post_content, _ := os.ReadFile(thread_post_content_filepath)
+	mocked_requester := func(mt MethodType, u Url) (*QueryResult, error) {
+		return &QueryResult{
+			content:          string(thread_post_content),
+			ResponseRawQuery: ``,
+			ResponseFullUrl:  `https://discoverygc.com/forums/portal.php`,
+		}, nil
+	}
+
+	thread_page, err := GetLatestThreads(WithMockedPageRequester(mocked_requester))
+	assert.Nil(t, err, "expected nil as error")
+	assert.Greater(t, len(thread_page.Threads), 0)
+}

app/scrappy/forum/query.go

@@ -0,0 +1,72 @@
+package forum
+
+import (
+	"darkbot/app/settings/logus"
+	"io"
+	"net/http"
+
+	"golang.org/x/net/html/charset"
+)
+
+type MethodType string
+
+const (
+	GET MethodType = "GET"
+)
+
+type QueryResult struct {
+	content          string
+	ResponseRawQuery string
+	ResponseFullUrl  string
+}
+
+func (q *QueryResult) GetContent() string {
+	return q.content
+}
+
+func NewQuery(method_type MethodType, url Url) (*QueryResult, error) {
+	client := &http.Client{}
+	req, err := http.NewRequest(string(method_type), string(url), nil)
+	if logus.CheckWarn(err, "Failed to create request") {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
+	// req.Header.Set("User-Agent", "curl/7.81.0")
+	req.Header.Set("ACCEPT", "*/*")
+	req.Header.Set("CONTENT-LENGTH", "")
+	req.Header.Set("CONTENT-TYPE", "")
+
+	resp, err := client.Do(req)
+	if logus.CheckWarn(err, "Failed to make query") {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 300 && resp.StatusCode <= 399 {
+		logus.Debug("this request is redirecting!")
+		redirectUrl, err := resp.Location()
+		if logus.CheckError(err, "Error getting redirect location") {
+			return nil, err
+		}
+
+		req.URL = redirectUrl
+		resp, err = client.Do(req)
+		if logus.CheckError(err, "Error sending redirect request:") {
+			return nil, err
+		}
+
+	}
+
+	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, err
+	}
+	bytes, err := io.ReadAll(utf8Body)
+
+	return &QueryResult{
+		content:          string(bytes),
+		ResponseRawQuery: resp.Request.URL.RawQuery,
+		ResponseFullUrl:  resp.Request.URL.String(),
+	}, err
+}