fetch url moved to util package
This commit is contained in:
parent
35f7816137
commit
abc61125ea
30
main.go
30
main.go
@ -2,7 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"NukaNewsBot/commands"
|
||||
"context"
|
||||
"NukaNewsBot/utils"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
@ -12,7 +12,6 @@ import (
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/bwmarrin/discordgo"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
@ -101,7 +100,7 @@ func main() {
|
||||
log.Println("Opened Discord connection")
|
||||
|
||||
// Run the scraping and message sending function at start up
|
||||
//sendNotifications(dg, fetchUrl(url), channelMap, roleMap, date)
|
||||
sendNotifications(dg, utils.FetchUrl(url), channelMap, roleMap, date)
|
||||
|
||||
// Schedule the scraping and message sending function to run once a day
|
||||
ticker := time.NewTicker(24 * time.Hour)
|
||||
@ -112,7 +111,7 @@ func main() {
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
sendNotifications(dg, fetchUrl(url), channelMap, roleMap, date)
|
||||
sendNotifications(dg, utils.FetchUrl(url), channelMap, roleMap, date)
|
||||
}
|
||||
}
|
||||
}()
|
||||
@ -122,29 +121,6 @@ func main() {
|
||||
<-make(chan struct{})
|
||||
}
|
||||
|
||||
func fetchUrl(url string) string {
|
||||
// Create a new context
|
||||
ctx := context.Background()
|
||||
ctx, cancel := chromedp.NewContext(
|
||||
ctx,
|
||||
chromedp.WithLogf(log.Printf),
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
// Navigate to the Fallout news page
|
||||
var html string
|
||||
err := chromedp.Run(ctx, chromedp.Tasks{
|
||||
chromedp.Navigate(url),
|
||||
chromedp.OuterHTML("html", &html),
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Return the HTML content
|
||||
return html
|
||||
}
|
||||
|
||||
// Add a function to extract relevant tags from the HTML content
|
||||
func extractNewsArticles(html string) []map[string]string {
|
||||
var articles []map[string]string
|
||||
|
31
utils/fetch_url.go
Normal file
31
utils/fetch_url.go
Normal file
@ -0,0 +1,31 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
func FetchUrl(url string) string {
|
||||
// Create a new context
|
||||
ctx := context.Background()
|
||||
ctx, cancel := chromedp.NewContext(
|
||||
ctx,
|
||||
chromedp.WithLogf(log.Printf),
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
// Navigate to the Fallout news page
|
||||
var html string
|
||||
err := chromedp.Run(ctx, chromedp.Tasks{
|
||||
chromedp.Navigate(url),
|
||||
chromedp.OuterHTML("html", &html),
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Return the HTML content
|
||||
return html
|
||||
}
|
Loading…
Reference in New Issue
Block a user