added broserless /function support
This commit is contained in:
parent
088cf1e66e
commit
ecac18a0b7
4 changed files with 271 additions and 130 deletions
77
scraping.go
77
scraping.go
|
|
@ -244,6 +244,14 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, d
|
|||
{
|
||||
getFilterResultBrowserlessURLs(filter, web.urlCache, debug)
|
||||
}
|
||||
case "func":
|
||||
{
|
||||
getBrowserlessFunctionResult(filter)
|
||||
}
|
||||
case "funcs":
|
||||
{
|
||||
getBrowserlessFunctionResults(filter)
|
||||
}
|
||||
}
|
||||
}
|
||||
case filter.Type == "echo":
|
||||
|
|
@ -384,6 +392,7 @@ func getBrowserlessURLContent(filter *Filter, fetchURL string) (string, error) {
|
|||
filter.log("Could not marshal url:", err)
|
||||
return "", err
|
||||
}
|
||||
browserlessURL = browserlessURL + "/content"
|
||||
resp, err := http.Post(browserlessURL, "application/json", bytes.NewBuffer(jsn))
|
||||
if err != nil {
|
||||
log.Println("Could not get browserless response content:", err)
|
||||
|
|
@ -399,6 +408,74 @@ func getBrowserlessURLContent(filter *Filter, fetchURL string) (string, error) {
|
|||
return string(body), nil
|
||||
}
|
||||
|
||||
func getBrowserlessFunctionResult(filter *Filter) {
|
||||
result, err := getBrowserlessFunctionContent(filter, "")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
filter.log(err)
|
||||
return
|
||||
}
|
||||
filter.Results = append(filter.Results, result)
|
||||
}
|
||||
|
||||
func getBrowserlessFunctionResults(filter *Filter) {
|
||||
for _, parent := range filter.Parents {
|
||||
for _, result := range parent.Results {
|
||||
fetchURL := result
|
||||
|
||||
str, err := getBrowserlessFunctionContent(filter, fetchURL)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
filter.log(err)
|
||||
continue
|
||||
}
|
||||
filter.Results = append(filter.Results, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type BrowserlessContext struct {
|
||||
Result string `json:"result"`
|
||||
}
|
||||
|
||||
func getBrowserlessFunctionContent(filter *Filter, result string) (string, error) {
|
||||
if !viper.IsSet("browserless.url") {
|
||||
|
||||
return "", errors.New("browserless.url not set")
|
||||
}
|
||||
browserlessURL := viper.GetString("browserless.url")
|
||||
if filter.Var2 == nil {
|
||||
return "", errors.New("filter.Var2 == nil")
|
||||
}
|
||||
code := *filter.Var2
|
||||
data := struct {
|
||||
Code string `json:"code"`
|
||||
Context BrowserlessContext `json:"context"`
|
||||
Detached bool `json:"detached"`
|
||||
}{
|
||||
Code: code,
|
||||
Context: BrowserlessContext{Result: result},
|
||||
Detached: false,
|
||||
}
|
||||
jsn, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
browserlessURL = browserlessURL + "/function"
|
||||
resp, err := http.Post(browserlessURL, "application/json", bytes.NewBuffer(jsn))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func getFilterResultXPath(filter *Filter) {
|
||||
selectType := "node"
|
||||
if filter.Var2 != nil {
|
||||
|
|
|
|||
134
static/edit.js
134
static/edit.js
|
|
@ -626,54 +626,14 @@ function onTypeChange(node) {
|
|||
gurlsOption.value = "gurls";
|
||||
gurlsOption.innerHTML = "Get URLs";
|
||||
browserlessSelect.appendChild(gurlsOption);
|
||||
var startOption = document.createElement("option");
|
||||
startOption.value = "start";
|
||||
startOption.innerHTML = "Start Session";
|
||||
browserlessSelect.appendChild(startOption);
|
||||
var endOption = document.createElement("option");
|
||||
endOption.value = "end";
|
||||
endOption.innerHTML = "End Session";
|
||||
browserlessSelect.appendChild(endOption);
|
||||
var gotoOption = document.createElement("option");
|
||||
gotoOption.value = "go";
|
||||
gotoOption.innerHTML = "Goto URL";
|
||||
browserlessSelect.appendChild(gotoOption);
|
||||
var cssOption = document.createElement("option");
|
||||
cssOption.value = "css";
|
||||
cssOption.innerHTML = "Click CSS Element";
|
||||
browserlessSelect.appendChild(cssOption);
|
||||
var xpathOption = document.createElement("option");
|
||||
xpathOption.value = "xpath";
|
||||
xpathOption.innerHTML = "Click XPath Element";
|
||||
browserlessSelect.appendChild(xpathOption);
|
||||
var authOption = document.createElement("option");
|
||||
authOption.value = "auth";
|
||||
authOption.innerHTML = "Authenticate";
|
||||
browserlessSelect.appendChild(authOption);
|
||||
var selectOption = document.createElement("option");
|
||||
selectOption.value = "select";
|
||||
selectOption.innerHTML = "Set Dropdown";
|
||||
browserlessSelect.appendChild(selectOption);
|
||||
var inputOption = document.createElement("option");
|
||||
inputOption.value = "input";
|
||||
inputOption.innerHTML = "Set Input";
|
||||
browserlessSelect.appendChild(inputOption);
|
||||
var keyOption = document.createElement("option");
|
||||
keyOption.value = "key";
|
||||
keyOption.innerHTML = "Press Key";
|
||||
browserlessSelect.appendChild(keyOption);
|
||||
var waitOption = document.createElement("option");
|
||||
waitOption.value = "wait";
|
||||
waitOption.innerHTML = "Wait";
|
||||
browserlessSelect.appendChild(waitOption);
|
||||
var lineOption = document.createElement("option");
|
||||
lineOption.value = "line";
|
||||
lineOption.innerHTML = "Raw Line";
|
||||
browserlessSelect.appendChild(lineOption);
|
||||
var rawOption = document.createElement("option");
|
||||
rawOption.value = "raw";
|
||||
rawOption.innerHTML = "Raw Function";
|
||||
browserlessSelect.appendChild(rawOption);
|
||||
var funcOption = document.createElement("option");
|
||||
funcOption.value = "func";
|
||||
funcOption.innerHTML = "Function";
|
||||
browserlessSelect.appendChild(funcOption);
|
||||
var funcsOption = document.createElement("option");
|
||||
funcsOption.value = "funcs";
|
||||
funcsOption.innerHTML = "Function on results";
|
||||
browserlessSelect.appendChild(funcsOption);
|
||||
var1Div.appendChild(browserlessSelect);
|
||||
if (var1Value == "") {
|
||||
browserlessSelect.value = "gurl";
|
||||
|
|
@ -754,7 +714,8 @@ function onTypeChange(node) {
|
|||
}
|
||||
finally { if (e_2) throw e_2.error; }
|
||||
}
|
||||
var1Div.appendChild(snippetDiv);
|
||||
var2Label.innerHTML = "Snippets";
|
||||
var2Div.appendChild(snippetDiv);
|
||||
var var2Input = document.createElement("input");
|
||||
var2Input.name = "var2";
|
||||
var2Input.id = "var2Input";
|
||||
|
|
@ -820,7 +781,6 @@ function onConditionChange(node) {
|
|||
var var1Value = "";
|
||||
var var2Value = "";
|
||||
var var3Value = "";
|
||||
console.log(node, var1Input.value);
|
||||
if (node != null) {
|
||||
// @ts-ignore
|
||||
var1Value = node.meta.var1;
|
||||
|
|
@ -903,10 +863,10 @@ function onBrowserlessChange(node) {
|
|||
var var3Input = document.getElementById("var3Input");
|
||||
var var3Label = document.getElementById("var3Label");
|
||||
var var3Div = document.getElementById("var3Div");
|
||||
var3Div.innerHTML = "";
|
||||
var var1Value = "";
|
||||
var var2Value = "";
|
||||
var var3Value = "";
|
||||
console.log(node, var1Input.value);
|
||||
if (node != null) {
|
||||
// @ts-ignore
|
||||
var1Value = node.meta.var1;
|
||||
|
|
@ -932,6 +892,78 @@ function onBrowserlessChange(node) {
|
|||
break;
|
||||
}
|
||||
case "gurls": {
|
||||
var var2Input_4 = document.createElement("input");
|
||||
var2Input_4.name = "var2";
|
||||
var2Input_4.id = "var2Input";
|
||||
var2Input_4.value = "";
|
||||
var2Input_4.disabled = true;
|
||||
var2Input_4.classList.add("form-control");
|
||||
var2Label.innerHTML = "-";
|
||||
var2Div.appendChild(var2Input_4);
|
||||
break;
|
||||
}
|
||||
case "func": {
|
||||
var var2Input_5 = document.createElement("textarea");
|
||||
var2Input_5.name = "var2Input";
|
||||
var2Input_5.id = "var2Input";
|
||||
var2Input_5.value = "module.exports = async ({ page, context }) => {\n await page.goto(\"https://192.168.178.254:8000\");\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};";
|
||||
var2Input_5.classList.add("form-control");
|
||||
var2Input_5.rows = 15;
|
||||
var2Label.innerHTML = "Code";
|
||||
var2Div.appendChild(var2Input_5);
|
||||
if (var2Value != "") {
|
||||
var2Input_5.value = var2Value;
|
||||
}
|
||||
var var3Input_1 = document.createElement("input");
|
||||
var3Input_1.type = "hidden";
|
||||
var3Input_1.id = "var3Input";
|
||||
var3Input_1.name = "var3Input";
|
||||
var3Div.appendChild(var3Input_1);
|
||||
var3Label.innerHTML = "Help";
|
||||
var helpLink1 = document.createElement("a");
|
||||
helpLink1.href = "https://www.browserless.io/docs/function";
|
||||
helpLink1.innerHTML = "Browserless /Funcion";
|
||||
helpLink1.target = "_blank";
|
||||
helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink1);
|
||||
var helpLink2 = document.createElement("a");
|
||||
helpLink2.href = "https://pptr.dev/api/puppeteer.page";
|
||||
helpLink2.innerHTML = "Puppeteer Page";
|
||||
helpLink2.target = "_blank";
|
||||
helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink2);
|
||||
break;
|
||||
}
|
||||
case "funcs": {
|
||||
var var2Input_6 = document.createElement("textarea");
|
||||
var2Input_6.name = "var2Input";
|
||||
var2Input_6.id = "var2Input";
|
||||
var2Input_6.value = "module.exports = async ({ page, context }) => {\n const { result } = context;\n await page.goto(result);\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};";
|
||||
var2Input_6.classList.add("form-control");
|
||||
var2Input_6.rows = 15;
|
||||
var2Label.innerHTML = "Code";
|
||||
var2Div.appendChild(var2Input_6);
|
||||
if (var2Value != "") {
|
||||
var2Input_6.value = var2Value;
|
||||
}
|
||||
var var3Input_2 = document.createElement("input");
|
||||
var3Input_2.type = "hidden";
|
||||
var3Input_2.id = "var3Input";
|
||||
var3Input_2.name = "var3Input";
|
||||
var3Div.appendChild(var3Input_2);
|
||||
var3Label.innerHTML = "Help";
|
||||
var helpLink1 = document.createElement("a");
|
||||
helpLink1.href = "https://www.browserless.io/docs/function";
|
||||
helpLink1.innerHTML = "Browserless /Funcion";
|
||||
helpLink1.target = "_blank";
|
||||
helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink1);
|
||||
var helpLink2 = document.createElement("a");
|
||||
helpLink2.href = "https://pptr.dev/api/puppeteer.page";
|
||||
helpLink2.innerHTML = "Puppeteer Page";
|
||||
helpLink2.target = "_blank";
|
||||
helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink2);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
173
static/edit.ts
173
static/edit.ts
|
|
@ -614,65 +614,15 @@ function onTypeChange(node: DiagramNode | null = null){
|
|||
gurlsOption.innerHTML = "Get URLs";
|
||||
browserlessSelect.appendChild(gurlsOption);
|
||||
|
||||
let startOption = document.createElement("option");
|
||||
startOption.value = "start"
|
||||
startOption.innerHTML = "Start Session";
|
||||
browserlessSelect.appendChild(startOption);
|
||||
let funcOption = document.createElement("option");
|
||||
funcOption.value = "func"
|
||||
funcOption.innerHTML = "Function";
|
||||
browserlessSelect.appendChild(funcOption);
|
||||
|
||||
let endOption = document.createElement("option");
|
||||
endOption.value = "end"
|
||||
endOption.innerHTML = "End Session";
|
||||
browserlessSelect.appendChild(endOption);
|
||||
|
||||
let gotoOption = document.createElement("option");
|
||||
gotoOption.value = "go"
|
||||
gotoOption.innerHTML = "Goto URL";
|
||||
browserlessSelect.appendChild(gotoOption);
|
||||
|
||||
let cssOption = document.createElement("option");
|
||||
cssOption.value = "css"
|
||||
cssOption.innerHTML = "Click CSS Element";
|
||||
browserlessSelect.appendChild(cssOption);
|
||||
|
||||
let xpathOption = document.createElement("option");
|
||||
xpathOption.value = "xpath"
|
||||
xpathOption.innerHTML = "Click XPath Element";
|
||||
browserlessSelect.appendChild(xpathOption);
|
||||
|
||||
let authOption = document.createElement("option");
|
||||
authOption.value = "auth"
|
||||
authOption.innerHTML = "Authenticate";
|
||||
browserlessSelect.appendChild(authOption);
|
||||
|
||||
let selectOption = document.createElement("option");
|
||||
selectOption.value = "select"
|
||||
selectOption.innerHTML = "Set Dropdown";
|
||||
browserlessSelect.appendChild(selectOption);
|
||||
|
||||
let inputOption = document.createElement("option");
|
||||
inputOption.value = "input"
|
||||
inputOption.innerHTML = "Set Input";
|
||||
browserlessSelect.appendChild(inputOption);
|
||||
|
||||
let keyOption = document.createElement("option");
|
||||
keyOption.value = "key"
|
||||
keyOption.innerHTML = "Press Key";
|
||||
browserlessSelect.appendChild(keyOption);
|
||||
|
||||
let waitOption = document.createElement("option");
|
||||
waitOption.value = "wait"
|
||||
waitOption.innerHTML = "Wait";
|
||||
browserlessSelect.appendChild(waitOption);
|
||||
|
||||
let lineOption = document.createElement("option");
|
||||
lineOption.value = "line"
|
||||
lineOption.innerHTML = "Raw Line";
|
||||
browserlessSelect.appendChild(lineOption);
|
||||
|
||||
let rawOption = document.createElement("option");
|
||||
rawOption.value = "raw"
|
||||
rawOption.innerHTML = "Raw Function";
|
||||
browserlessSelect.appendChild(rawOption);
|
||||
let funcsOption = document.createElement("option");
|
||||
funcsOption.value = "funcs"
|
||||
funcsOption.innerHTML = "Function on results";
|
||||
browserlessSelect.appendChild(funcsOption);
|
||||
|
||||
var1Div.appendChild(browserlessSelect);
|
||||
|
||||
|
|
@ -885,7 +835,8 @@ end
|
|||
gap.classList.add("m-1", "xs");
|
||||
snippetDiv.appendChild(gap);
|
||||
}
|
||||
var1Div.appendChild(snippetDiv);
|
||||
var2Label.innerHTML = "Snippets";
|
||||
var2Div.appendChild(snippetDiv);
|
||||
|
||||
let var2Input = document.createElement("input");
|
||||
var2Input.name = "var2";
|
||||
|
|
@ -954,7 +905,6 @@ function onConditionChange(node: DiagramNode | null = null){
|
|||
let var1Value = "";
|
||||
let var2Value = "";
|
||||
let var3Value = "";
|
||||
console.log(node, var1Input.value)
|
||||
if (node != null){
|
||||
// @ts-ignore
|
||||
var1Value = node.meta.var1;
|
||||
|
|
@ -1029,11 +979,11 @@ function onBrowserlessChange(node: DiagramNode | null = null){
|
|||
let var3Input = document.getElementById("var3Input") as HTMLInputElement;
|
||||
let var3Label = document.getElementById("var3Label") as HTMLLabelElement;
|
||||
let var3Div = document.getElementById("var3Div") as HTMLDivElement;
|
||||
var3Div.innerHTML = "";
|
||||
|
||||
let var1Value = "";
|
||||
let var2Value = "";
|
||||
let var3Value = "";
|
||||
console.log(node, var1Input.value)
|
||||
if (node != null){
|
||||
// @ts-ignore
|
||||
var1Value = node.meta.var1;
|
||||
|
|
@ -1056,10 +1006,109 @@ function onBrowserlessChange(node: DiagramNode | null = null){
|
|||
var2Input.value = var2Value;
|
||||
var2Input.classList.add("form-control")
|
||||
var2Label.innerHTML = "URL";
|
||||
var2Div.appendChild(var2Input)
|
||||
var2Div.appendChild(var2Input);
|
||||
break;
|
||||
}
|
||||
case "gurls": {
|
||||
let var2Input = document.createElement("input");
|
||||
var2Input.name = "var2";
|
||||
var2Input.id = "var2Input";
|
||||
var2Input.value = "";
|
||||
var2Input.disabled = true;
|
||||
var2Input.classList.add("form-control")
|
||||
var2Label.innerHTML = "-";
|
||||
var2Div.appendChild(var2Input);
|
||||
break;
|
||||
}
|
||||
case "func": {
|
||||
let var2Input = document.createElement("textarea");
|
||||
var2Input.name = "var2Input";
|
||||
var2Input.id = "var2Input";
|
||||
var2Input.value = `module.exports = async ({ page, context }) => {
|
||||
await page.goto("https://192.168.178.254:8000");
|
||||
|
||||
const data = await page.content();
|
||||
|
||||
return {
|
||||
data,
|
||||
type: 'text/plain', // 'application/html' 'application/json'
|
||||
};
|
||||
};`;
|
||||
var2Input.classList.add("form-control");
|
||||
var2Input.rows = 15;
|
||||
var2Label.innerHTML = "Code";
|
||||
var2Div.appendChild(var2Input);
|
||||
|
||||
if (var2Value != ""){
|
||||
var2Input.value = var2Value;
|
||||
}
|
||||
|
||||
let var3Input = document.createElement("input");
|
||||
var3Input.type = "hidden";
|
||||
var3Input.id = "var3Input";
|
||||
var3Input.name = "var3Input";
|
||||
var3Div.appendChild(var3Input);
|
||||
|
||||
var3Label.innerHTML = "Help";
|
||||
let helpLink1 = document.createElement("a");
|
||||
helpLink1.href = "https://www.browserless.io/docs/function";
|
||||
helpLink1.innerHTML = "Browserless /Funcion";
|
||||
helpLink1.target = "_blank";
|
||||
helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink1);
|
||||
|
||||
let helpLink2 = document.createElement("a");
|
||||
helpLink2.href = "https://pptr.dev/api/puppeteer.page";
|
||||
helpLink2.innerHTML = "Puppeteer Page";
|
||||
helpLink2.target = "_blank";
|
||||
helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink2);
|
||||
break;
|
||||
}
|
||||
case "funcs":{
|
||||
let var2Input = document.createElement("textarea");
|
||||
var2Input.name = "var2Input";
|
||||
var2Input.id = "var2Input";
|
||||
var2Input.value = `module.exports = async ({ page, context }) => {
|
||||
const { result } = context;
|
||||
await page.goto(result);
|
||||
|
||||
const data = await page.content();
|
||||
|
||||
return {
|
||||
data,
|
||||
type: 'text/plain', // 'application/html' 'application/json'
|
||||
};
|
||||
};`;
|
||||
var2Input.classList.add("form-control");
|
||||
var2Input.rows = 15;
|
||||
var2Label.innerHTML = "Code";
|
||||
var2Div.appendChild(var2Input);
|
||||
|
||||
if (var2Value != ""){
|
||||
var2Input.value = var2Value;
|
||||
}
|
||||
|
||||
let var3Input = document.createElement("input");
|
||||
var3Input.type = "hidden";
|
||||
var3Input.id = "var3Input";
|
||||
var3Input.name = "var3Input";
|
||||
var3Div.appendChild(var3Input);
|
||||
|
||||
var3Label.innerHTML = "Help";
|
||||
let helpLink1 = document.createElement("a");
|
||||
helpLink1.href = "https://www.browserless.io/docs/function";
|
||||
helpLink1.innerHTML = "Browserless /Funcion";
|
||||
helpLink1.target = "_blank";
|
||||
helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink1);
|
||||
|
||||
let helpLink2 = document.createElement("a");
|
||||
helpLink2.href = "https://pptr.dev/api/puppeteer.page";
|
||||
helpLink2.innerHTML = "Puppeteer Page";
|
||||
helpLink2.target = "_blank";
|
||||
helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
|
||||
var3Div.appendChild(helpLink2);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
17
todo.md
17
todo.md
|
|
@ -5,22 +5,5 @@
|
|||
- util.go
|
||||
- edit.ts
|
||||
- diagram.ts
|
||||
- browserless function filters
|
||||
- ~~get url~~
|
||||
- ~~get urls~~
|
||||
- session
|
||||
- start
|
||||
- end
|
||||
- goto url
|
||||
- click elem
|
||||
- xpath
|
||||
- css
|
||||
- basic auth
|
||||
- set select
|
||||
- set text input
|
||||
- press key
|
||||
- wait
|
||||
- raw line
|
||||
- just raw
|
||||
- url path support
|
||||
- refactor project structure
|
||||
Loading…
Add table
Reference in a new issue