From 94856b6f00973dd39a15b7a4c65f04885467131e Mon Sep 17 00:00:00 2001
From: BroodjeAap <david@broodjeaap.net>
Date: Sat, 1 Oct 2022 11:24:30 +0000
Subject: [PATCH] added url caching for viewing/editing watches

---
 main.go     |  8 +++++---
 scraping.go | 24 +++++++++++++++++-------
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/main.go b/main.go
index b6210bc..2d3a945 100644
--- a/main.go
+++ b/main.go
@@ -22,7 +22,8 @@ var newWatchHTML = filepath.Join("templates", "newWatch.html")
 
 type Web struct {
 	//Bot *tgbotapi.BotAPI
-	db *gorm.DB
+	urlCache map[string]string
+	db       *gorm.DB
 }
 
 func (web Web) index(c *gin.Context) {
@@ -76,7 +77,7 @@ func (web Web) watchView(c *gin.Context) {
 	web.db.Model(&FilterOutput{}).Where("watch_id = ?", watch.ID).Find(&values)
 
 	buildFilterTree(filters, connections)
-	processFilters(filters, web.db)
+	processFilters(filters, web.db, web.urlCache, true, true)
 
 	c.HTML(http.StatusOK, "watchView", gin.H{
 		"Watch":       watch,
@@ -174,7 +175,8 @@ func main() {
 
 	web := Web{
 		//bot,
-		db,
+		db:       db,
+		urlCache: make(map[string]string, 5),
 	}
 	router := gin.Default()
 
diff --git a/scraping.go b/scraping.go
index e460ffa..0198b51 100644
--- a/scraping.go
+++ b/scraping.go
@@ -19,7 +19,7 @@ import (
 	"gorm.io/gorm"
 )
 
-func processFilters(filters []Filter, db *gorm.DB) {
+func processFilters(filters []Filter, db *gorm.DB, urlCache map[string]string, useCache bool, setCache bool) {
 	processedMap := make(map[uint]bool, len(filters))
 	for len(filters) > 0 {
 		filter := &filters[0]
@@ -35,20 +35,20 @@ func processFilters(filters []Filter, db *gorm.DB) {
 			filters = append(filters, *filter)
 			continue
 		}
-		getFilterResult(filter, db)
+		getFilterResult(filter, db, urlCache, useCache, setCache)
 		processedMap[filter.ID] = true
 	}
 }
 
-func getFilterResult(filter *Filter, db *gorm.DB) {
+func getFilterResult(filter *Filter, db *gorm.DB, urlCache map[string]string, useCache bool, setCache bool) {
 	switch {
 	case filter.Type == "gurl":
 		{
-			getFilterResultURL(filter)
+			getFilterResultURL(filter, urlCache, useCache, setCache)
 		}
 	case filter.Type == "gurls":
 		{
-			getFilterResultURL(filter)
+			getFilterResultURL(filter, urlCache, useCache, setCache)
 		}
 	case filter.Type == "xpath":
 		{
@@ -145,8 +145,14 @@ func getFilterResult(filter *Filter, db *gorm.DB) {
 	}
 }
 
-func getFilterResultURL(filter *Filter) {
+func getFilterResultURL(filter *Filter, urlCache map[string]string, useCache bool, setCache bool) {
 	url := filter.Var1
+	val, exists := urlCache[url]
+	if useCache && exists {
+		filter.Results = append(filter.Results, val)
+		return
+	}
+
 	resp, err := http.Get(url)
 	if err != nil {
 		log.Println("Could not fetch url", url)
@@ -157,7 +163,11 @@ func getFilterResultURL(filter *Filter) {
 		log.Println("Could not fetch url", url)
 		log.Println("Reason:", err)
 	}
-	filter.Results = append(filter.Results, string(body))
+	str := string(body)
+	filter.Results = append(filter.Results, str)
+	if setCache {
+		urlCache[url] = str
+	}
 }
 
 func getFilterResultXPath(filter *Filter) {