broodjeaap/go-watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

added 'expect' filter

Browse source
This commit is contained in:
BroodjeAap 2023-03-18 12:36:09 +00:00
parent cabd11f2e9
commit cec960e6dd
9 changed files with 523 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
README.md
View file

@ -36,6 +36,7 @@ Some out-of-the-box highlights:
- [Substring](#substring)
- [Contains](#contains)
- [Store](#store)
- [Expect](#expect)
- [Notify](#notify)
- [Math](#math)
- [Sum](#sum)
@ -445,6 +446,11 @@ Inputs pass if they contain the given regex.
Stores each input value in the database under its own name.
It's recommended to do this after reducing inputs to a single value (Minimum/Maximum/Average/etc).
## Expect
Outputs a value when it has no inputs, useful to do something (notify) when something goes wrong with your Watch.
Will only trigger once and can be set to wait multiple times before triggering.
## Notify
Executes the given template and sends the resulting string as a message to the given notifier(s).

10
models/expect.go Normal file
View file

@ -0,0 +1,10 @@
package models
import "time"
type ExpectFail struct {
ID uint `yaml:"expect_fail_id" json:"expect_fail_id"`
WatchID uint `yaml:"expect_fail_watch_id" gorm:"index" json:"expect_fail_watch_id"`
Name string `yaml:"expect_fail_name" json:"expect_fail_name"`
Time time.Time `yaml:"expect_fail_time" json:"expect_fail_time"`
}

2
todo.md
View file

@ -1,5 +1,3 @@
# Todo
- add 'expect' filter, outputs on no inputs
- seperate 'expect' table in db, after x number of expects passing really pass
- add time 'jitter' to schedule filter
- var2 with another schedule string, var1 + (var2 * random) duration

51
web/scraping.go
View file

@ -230,6 +230,10 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, d
{
storeFilterResult(filter, web.db, debug)
}
case filter.Type == "expect":
{
getFilterResultExpect(filter, web, debug)
}
case filter.Type == "notify":
{
notifyFilter(filters, filter, watch, web, debug)
@ -1266,6 +1270,53 @@ func getFilterResultLua(filter *Filter) {
)
}
// getFilterResultExpect outputs once if there is no results from its parents a set number of times
func getFilterResultExpect(filter *Filter, web *Web, debug bool) {
if len(filter.Parents) == 0 {
filter.Logs = append(filter.Logs, "Need Parents")
return
}
for i := range filter.Parents {
parent := filter.Parents[i]
if len(parent.Results) > 0 { // reset/delete expectFails
web.db.Delete(&ExpectFail{}, "watch_id = ? AND name = ?", filter.WatchID, filter.Name)
return
}
}
if debug {
filter.Results = append(filter.Results, "Expected")
return
}
expectThreshold, err := strconv.Atoi(filter.Var1)
if err != nil {
filter.Logs = append(filter.Logs, "Could not parse to int:", filter.Var1)
expectThreshold = 1
}
if expectThreshold <= 0 {
// 0 doesn't really make sense so just set to 1
expectThreshold = 1
}
var expectFails []ExpectFail
web.db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ? AND name = ?", filter.WatchID, filter.Name)
// +1 so this one is already counted
failCount := len(expectFails) + 1
if failCount > expectThreshold {
return
} else if expectThreshold == failCount {
filter.Results = append(filter.Results, "Expected")
}
expectFail := ExpectFail{
WatchID: filter.WatchID,
Name: filter.Name,
Time: time.Now(),
}
web.db.Create(&expectFail)
}
// getFilterResultEcho is a debug filter type, used to bootstrap some tests
func getFilterResultEcho(filter *Filter) {
filter.Results = append(filter.Results, filter.Var1)

395
web/scraping_test.go
View file

@ -699,7 +699,7 @@ func TestFilterRound(t *testing.T) {
func getTestDB() *gorm.DB {
db, _ := gorm.Open(sqlite.Open("./test.db"))
db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{})
db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{}, &ExpectFail{})
return db
}
@ -1765,8 +1765,6 @@ func TestSimpleTriggeredWatch(t *testing.T) {
maxFilter := &filters[6]
storeMaxFilter := &filters[7]
log.Println(scheduleFilter)
connections := []FilterConnection{
{
WatchID: watch.ID,
@ -1806,8 +1804,6 @@ func TestSimpleTriggeredWatch(t *testing.T) {
}
db.Create(&connections)
log.Println(connections[0])
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
var filterOutputs []FilterOutput
@ -1874,3 +1870,392 @@ func TestDontAllowMultipleCronOnSingleFilter(t *testing.T) {
t.Errorf("Expected error message in filter log, found empty log: %s", filter.Logs)
}
}
func TestWatchWithExpectNotTriggering(t *testing.T) {
db := getTestDB()
filters := []Filter{
{
ID: 0,
Name: "Echo",
Type: "echo",
Var1: HTML_STRING,
},
{
ID: 1,
Name: "XPath",
Type: "xpath",
Var1: "//td[@class='price']",
},
{
ID: 2,
Name: "Expect",
Type: "expect",
Var1: "1",
},
}
expectFilter := &filters[2]
connections := []FilterConnection{
{
OutputID: 0,
InputID: 1,
},
{
OutputID: 1,
InputID: 2,
},
}
buildFilterTree(filters, connections)
ProcessFilters(filters, &Web{db: db}, nil, false, nil)
if len(expectFilter.Results) != 0 {
t.Error("Expect has results, should be empty:", expectFilter.Results)
}
err := os.Remove("./test.db")
if err != nil {
log.Println("Could not remove test db:", err)
}
}
func TestWatchWithExpectTriggering(t *testing.T) {
db := getTestDB()
filters := []Filter{
{
ID: 0,
Name: "Echo",
Type: "echo",
Var1: HTML_STRING,
},
{
ID: 1,
Name: "XPath",
Type: "xpath",
Var1: "//div[@class='price']",
},
{
ID: 2,
Name: "Expect",
Type: "expect",
Var1: "1",
},
}
expectFilter := &filters[2]
connections := []FilterConnection{
{
OutputID: 0,
InputID: 1,
},
{
OutputID: 1,
InputID: 2,
},
}
buildFilterTree(filters, connections)
ProcessFilters(filters, &Web{db: db}, nil, false, nil)
if len(expectFilter.Results) != 1 {
t.Error("Expect has no results, should have 'expected'")
}
err := os.Remove("./test.db")
if err != nil {
log.Println("Could not remove test db:", err)
}
}
func TestWatchWithExpect3Triggering(t *testing.T) {
db := getTestDB()
filters := []Filter{
{
ID: 0,
Name: "Echo",
Type: "echo",
Var1: HTML_STRING,
},
{
ID: 1,
Name: "XPath",
Type: "xpath",
Var1: "//div[@class='price']",
},
{
ID: 2,
Name: "Expect",
Type: "expect",
Var1: "3",
},
}
expectFilter := &filters[2]
connections := []FilterConnection{
{
OutputID: 0,
InputID: 1,
},
{
OutputID: 1,
InputID: 2,
},
}
buildFilterTree(filters, connections)
ProcessFilters(filters, &Web{db: db}, nil, false, nil)
if len(expectFilter.Results) != 0 {
t.Error("Expect has results, should be empty:", expectFilter.Results)
}
ProcessFilters(filters, &Web{db: db}, nil, false, nil)
if len(expectFilter.Results) != 0 {
t.Error("Expect has results, should be empty:", expectFilter.Results)
}
ProcessFilters(filters, &Web{db: db}, nil, false, nil)
if len(expectFilter.Results) != 1 {
t.Error("Expect has no results, should have 'expected'")
}
err := os.Remove("./test.db")
if err != nil {
log.Println("Could not remove test db:", err)
}
}
func TestWatchWithExpectNotTriggeringDB(t *testing.T) {
db := getTestDB()
watch := Watch{
Name: "Test",
}
db.Create(&watch)
filters := []Filter{
{
WatchID: watch.ID,
Name: "Schedule",
Type: "cron",
},
{
WatchID: watch.ID,
Name: "Echo",
Type: "echo",
Var1: HTML_STRING,
},
{
WatchID: watch.ID,
Name: "XPath",
Type: "xpath",
Var1: "//td[@class='price']",
},
{
WatchID: watch.ID,
Name: "Expect",
Type: "expect",
Var1: "1",
},
}
db.Create(&filters)
scheduleFilter := &filters[0]
echoFilter := &filters[1]
xpathFilter := &filters[2]
expectFilter := &filters[3]
connections := []FilterConnection{
{
WatchID: watch.ID,
OutputID: scheduleFilter.ID,
InputID: echoFilter.ID,
},
{
WatchID: watch.ID,
OutputID: echoFilter.ID,
InputID: xpathFilter.ID,
},
{
WatchID: watch.ID,
OutputID: xpathFilter.ID,
InputID: expectFilter.ID,
},
}
db.Create(&connections)
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
var expectFails []ExpectFail
db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID)
if len(expectFails) > 0 {
t.Errorf("Found ExpectFail values expected none!")
}
err := os.Remove("./test.db")
if err != nil {
log.Println("Could not remove test db:", err)
}
}
func TestWatchWithExpectTriggeringDB(t *testing.T) {
db := getTestDB()
watch := Watch{
Name: "Test",
}
db.Create(&watch)
filters := []Filter{
{
WatchID: watch.ID,
Name: "Schedule",
Type: "cron",
},
{
WatchID: watch.ID,
Name: "Echo",
Type: "echo",
Var1: HTML_STRING,
},
{
WatchID: watch.ID,
Name: "XPath",
Type: "xpath",
Var1: "//div[@class='price']",
},
{
WatchID: watch.ID,
Name: "Expect",
Type: "expect",
Var1: "1",
},
}
db.Create(&filters)
scheduleFilter := &filters[0]
echoFilter := &filters[1]
xpathFilter := &filters[2]
expectFilter := &filters[3]
connections := []FilterConnection{
{
WatchID: watch.ID,
OutputID: scheduleFilter.ID,
InputID: echoFilter.ID,
},
{
WatchID: watch.ID,
OutputID: echoFilter.ID,
InputID: xpathFilter.ID,
},
{
WatchID: watch.ID,
OutputID: xpathFilter.ID,
InputID: expectFilter.ID,
},
}
db.Create(&connections)
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
var expectFails []ExpectFail
db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID)
if len(expectFails) != 1 {
t.Errorf("Found no ExpectFail values expected 1!")
}
err := os.Remove("./test.db")
if err != nil {
log.Println("Could not remove test db:", err)
}
}
func TestWatchWithExpect3TriggeringDB(t *testing.T) {
db := getTestDB()
watch := Watch{
Name: "Test",
}
db.Create(&watch)
filters := []Filter{
{
WatchID: watch.ID,
Name: "Schedule",
Type: "cron",
},
{
WatchID: watch.ID,
Name: "Echo",
Type: "echo",
Var1: HTML_STRING,
},
{
WatchID: watch.ID,
Name: "XPath",
Type: "xpath",
Var1: "//div[@class='price']",
},
{
WatchID: watch.ID,
Name: "Expect",
Type: "expect",
Var1: "3",
},
}
db.Create(&filters)
scheduleFilter := &filters[0]
echoFilter := &filters[1]
xpathFilter := &filters[2]
expectFilter := &filters[3]
connections := []FilterConnection{
{
WatchID: watch.ID,
OutputID: scheduleFilter.ID,
InputID: echoFilter.ID,
},
{
WatchID: watch.ID,
OutputID: echoFilter.ID,
InputID: xpathFilter.ID,
},
{
WatchID: watch.ID,
OutputID: xpathFilter.ID,
InputID: expectFilter.ID,
},
}
db.Create(&connections)
var expectFails []ExpectFail
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID)
if len(expectFails) != 1 {
t.Errorf("Found %d ExpectFail values, expected 1!", len(expectFails))
log.Println(expectFails)
}
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID)
if len(expectFails) != 2 {
t.Errorf("Found %d ExpectFail values, expected 2!", len(expectFails))
log.Println(expectFails)
}
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID)
if len(expectFails) != 3 {
t.Errorf("Found %d ExpectFail values, expected 3! (1)", len(expectFails))
log.Println(expectFails)
}
TriggerSchedule(watch.ID, &Web{db: db}, &scheduleFilter.ID)
db.Model(&ExpectFail{}).Find(&expectFails, "watch_id = ?", watch.ID)
if len(expectFails) != 3 {
t.Errorf("Found %d ExpectFail values, expected 3! (2)", len(expectFails))
log.Println(expectFails)
}
err := os.Remove("./test.db")
if err != nil {
log.Println("Could not remove test db:", err)
}
}

38
web/static/edit.js
View file

@ -34,6 +34,8 @@ var urlPrefix = getURLPrefix();
function onTypeChange(node) {
var e_1, _a, e_2, _b;
if (node === void 0) { node = null; }
// onTypeChange handles changing of the type of a DiagramNode while editing or creating a new Node
// It removes all input elements and each case is responsible for adding the input it needs
// @ts-ignore
var urlPrefix = getURLPrefix();
var select = document.getElementById("typeInput");
@ -415,6 +417,29 @@ function onTypeChange(node) {
onConditionChange(node);
break;
}
case "expect": {
var var1Input = document.createElement("input");
var1Input.name = "var1";
var1Input.id = "var1Input";
var1Input.type = "number";
var1Input.value = "1";
var1Input.classList.add("form-control");
var1Label.innerHTML = "Threshold";
var1Input.placeholder = "1";
if (var1Value != "") {
var1Input.value = var1Value;
}
var1Div.appendChild(var1Input);
var var2Input = document.createElement("input");
var2Input.name = "var2";
var2Input.id = "var2Input";
var2Input.value = var2Value;
var2Input.classList.add("form-control");
var2Input.disabled = true;
var2Label.innerHTML = "-";
var2Div.appendChild(var2Input);
break;
}
case "notify": {
var var1Input = document.createElement("textarea");
var1Input.name = "var1";
@ -607,6 +632,7 @@ function onTypeChange(node) {
}
function onMathChange(node) {
if (node === void 0) { node = null; }
// onMatchChange handles the changing of the inputs when type == math
var var1Input = document.getElementById("var1Input");
var var1Label = document.getElementById("var1Label");
var var2Input = document.getElementById("var2Input");
@ -633,6 +659,7 @@ function onMathChange(node) {
function onConditionChange(node) {
var e_3, _a;
if (node === void 0) { node = null; }
// onConditionChange handles the changing of the inputs when type == condition
var var1Input = document.getElementById("var1Input");
var var1Label = document.getElementById("var1Label");
var var1Div = document.getElementById("var1Div");
@ -713,6 +740,7 @@ function onConditionChange(node) {
}
}
function onBrowserlessChange(node) {
// onBrowserlessChange handles the changing of the inputs when type == browserless
if (node === void 0) { node = null; }
var var1Input = document.getElementById("var1Input");
var var1Label = document.getElementById("var1Label");
@ -790,7 +818,7 @@ function onBrowserlessChange(node) {
var var2Input_6 = document.createElement("textarea");
var2Input_6.name = "var2Input";
var2Input_6.id = "var2Input";
var2Input_6.value = "module.exports = async ({ page, context }) => {\n const { result } = context;\n await page.goto(result);\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};";
var2Input_6.value = "module.exports = async ({ page, context }) => {\n const { result } = context;\n await page.goto(result);\n\n // click something\n //await page.click(\"#elem\");\n \n // fill input\n //await page.$eval('#elem', el => el.value = 'some text');\n \n // select dropdown\n // await page.select('#elem', 'value')\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};";
var2Input_6.classList.add("form-control");
var2Input_6.rows = 15;
var2Label.innerHTML = "Code";
@ -816,6 +844,7 @@ function onBrowserlessChange(node) {
}
}
function onSubmitNewFilter() {
// onSubmitNewFilter collects all the values from the input elements, and calls _diagram.addNode() with it
var nameInput = document.getElementById("nameInput");
var name = nameInput.value;
var selectType = document.getElementById("typeInput");
@ -830,6 +859,7 @@ function onSubmitNewFilter() {
}
function editNode(node) {
var e_4, _a, e_5, _b;
// editNode resets the edit/new Node modal to reflect the values of 'node'
var addFilterButton = document.getElementById("filterButton");
addFilterButton.click();
var name = node.label;
@ -919,6 +949,7 @@ function editNode(node) {
submitButton.onclick = function () { submitEditNode(node); };
}
function deleteNode(node) {
// deleteNode deletes a node from _diagram and removes all connections to/from it
_diagram.nodes.delete(node.id);
for (var i = 0; i < _diagram.connections.length; i++) {
var connection = _diagram.connections[i];
@ -931,6 +962,7 @@ function deleteNode(node) {
}
}
function submitEditNode(node) {
// submitEditNode saves the changes to the input elements to the underlying node
var nameInput = document.getElementById("nameInput");
node.label = nameInput.value;
var selectType = document.getElementById("typeInput");
@ -950,6 +982,7 @@ function submitEditNode(node) {
}
function saveWatch() {
var e_6, _a, e_7, _b;
// saveWatch collects all the state (nodes/connections), turns it into JSON and submits it through a hidden form
var watchIdInput = document.getElementById("watch_id");
var watchId = Number(watchIdInput.value);
var filters = new Array();
@ -1006,6 +1039,7 @@ function saveWatch() {
saveWatchForm.submit();
}
function addFilterButtonClicked() {
// addFilterButtonClicked opens up the new/edit filter modal and empties it
var submitButton = document.getElementById("submitFilterButton");
submitButton.onclick = onSubmitNewFilter;
submitButton.innerHTML = "Add Filter";
@ -1016,6 +1050,7 @@ function addFilterButtonClicked() {
onTypeChange();
}
function pageInit() {
// pageInit sets all the onclick/onchange trigger events
var select = document.getElementById("typeInput");
select.onchange = function () { onTypeChange(); };
var addFilterButton = document.getElementById("filterButton");
@ -1027,6 +1062,7 @@ function pageInit() {
}
document.addEventListener('DOMContentLoaded', pageInit, false);
function clearCache() {
// POSTs to cache/clear and reloads if clearing the cache was succesful
var confirmed = confirm("Do you want to clear the URL cache?");
if (!confirmed) {
return; // do nothing

25
web/static/edit.ts
View file

@ -398,6 +398,31 @@ function onTypeChange(node: DiagramNode | null = null){
onConditionChange(node);
break;
}
case "expect": {
let var1Input = document.createElement("input");
var1Input.name = "var1";
var1Input.id = "var1Input";
var1Input.type = "number";
var1Input.value = "1";
var1Input.classList.add("form-control")
var1Label.innerHTML = "Threshold";
var1Input.placeholder = "1";
if (var1Value != ""){
var1Input.value = var1Value;
}
var1Div.appendChild(var1Input);
let var2Input = document.createElement("input");
var2Input.name = "var2";
var2Input.id = "var2Input";
var2Input.value = var2Value;
var2Input.classList.add("form-control")
var2Input.disabled = true;
var2Label.innerHTML = "-";
var2Div.appendChild(var2Input);
break;
}
case "notify":{
let var1Input = document.createElement("textarea");
var1Input.name = "var1";

1
web/templates/watch/edit.html
View file

@ -88,6 +88,7 @@ GoWatch Edit {{ .Watch.Name }}
<option value="math">Math</option>
<option value="store">Store</option>
<option value="condition">Condition</option>
<option value="expect">Expect</option>
<option value="notify">Notify</option>
<option value="cron">Schedule</option>
<option value="brow">Browserless</option>

4
web/web.go
View file

@ -165,7 +165,7 @@ func (web *Web) initDB() {
}
break
}
web.db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{})
web.db.AutoMigrate(&Watch{}, &Filter{}, &FilterConnection{}, &FilterOutput{}, ExpectFail{})
}
// initRouer initializes the GoWatch routes, binding web.func to a url path
@ -663,6 +663,7 @@ func (web *Web) deleteWatch(c *gin.Context) {
web.db.Delete(&FilterConnection{}, "watch_id = ?", id)
web.db.Delete(&FilterOutput{}, "watch_id = ?", id)
web.db.Delete(&ExpectFail{}, "watch_id = ?", id)
var cronFilters []Filter
web.db.Model(&Filter{}).Find(&cronFilters, "watch_id = ? AND type = 'cron' AND var2 = 'yes'", id)
@ -808,6 +809,7 @@ func (web *Web) watchUpdate(c *gin.Context) {
}
web.db.Delete(&Filter{}, "watch_id = ?", watch.ID)
web.db.Delete(&ExpectFail{}, "watch_id = ?", watch.ID)
filterMap := make(map[uint]*Filter)
if len(newFilters) > 0 {