Add new file

spider2.py

+import asyncio
+from pyppeteer import launch
+import time
+
+browser, page, frame = None, None, None
+input_list = ["input"]
+click_list = ["label","button","a"]
+link_list = ["a"]
+input_id_list = []
+click_id_list = []
+url_list = []
+
+ip = "192.168.0.1"
+
+async def get_element_list(page,p):
+    global url_list,input_id_list ,click_id_list
+    print("getting",p)
+    element_id_list = []
+    try:
+        element_list = await page.querySelectorAll(p)
+        #print(element_list)
+        for element in element_list:
+            id = await page.evaluate('(element) => element.id', element)#if None
+                       
+            if_input = True
+            try:
+
+                if p == "input":
+                    element_type = await page.evaluate('(element) => element.type', element)
+                    element_click = await page.evaluate('(element) => element.onclick', element)
+                    #print(element_click)
+                    if  element_click != None:
+                        input_click = []
+                        input_click.append("#"+id)
+                        click_id_list.append(input_click)
+                    #print(element_type)
+                    if element_type == "hidden":
+                        if_input = False
+                elif p == "a":
+                    url = await page.evaluate('(element) => element.href', element)
+                    if url[7:18] == ip:
+                        url_list.append(url)
+                    #print(url)
+                else:
+                    element_click = await page.evaluate('(element) => element.onclick', element)
+                    #print(element_click)
+                    if  element_click == None:
+                        if_input = False
+            except:
+                print("error")
+            #print(id,if_input)
+            if if_input:
+                element_id_list.append("#"+id)
+                #print(element_id_list)
+    except:
+        print("no",p)
+    return element_id_list
+
+
+async def force_inputorclick(page):
+    navigationPromise = asyncio.ensure_future(page.waitForNavigation())
+    for In_list in input_id_list:
+        for In in In_list:
+            try:
+                await page.type(In, "")
+            except:
+                print("could not input")
+    for Cl_list in click_id_list:
+        for Cl in Cl_list:
+            try:
+                await page.click(Cl)
+                await navigationPromise
+                print("new page")
+                #await page.waitFor(300)
+                #finalResponse = await page.waitForResponse(lambda res: res.url == page.url and res.status == 200)
+                sshotname = "screenshot/"+ Cl + ".png"
+                await page.screenshot({'path': sshotname})
+            except:
+                print("could not click")
+            
+
+
+    
+      
+
+async def main():
+    global browser, page, frame, input_id_list, click_id_list, url_list
+    browser = await launch({
+
+        'headless': True, # 关闭无头模式
+        'devtools': True, # 打开 chromium 的 devtools
+        'executablePath': '/home/zhw/.local/share/pyppeteer/chrome',
+        'args': [ 
+            '--disable-extensions',
+            '--hide-scrollbars',
+            '--disable-bundled-ppapi-flash',
+            '--mute-audio',
+            '--no-sandbox',
+            '--disable-setuid-sandbox',
+            '--disable-gpu',
+        ],
+        'dumpio': True,  
+    })
+    print("-----------------login-------------------")
+    url_list = ["http://192.168.0.1/"]
+    page = await browser.newPage()
+
+    await page.setViewport({'width': 1366, 'height': 768})
+    await page.goto('http://192.168.0.1/')
+
+    print(".................start get page's element_id.................")
+    
+    for p in input_list:
+        id_list = await get_element_list(page,p)
+        input_id_list.append(id_list)
+    print(input_id_list)       
+
+    for p in click_list:
+        id_list = await get_element_list(page,p)
+        click_id_list.append(id_list)
+    print(click_id_list)
+
+    print(".................start input/click...................")
+
+    await force_inputorclick(page)
+    
+    #input.buttonBig:nth-child(3)#menu > div:nth-child(2) > a:nth-child(2)
+    print("-----------------first step-------------------")
+
+    new_url = page.url
+    print(new_url)
+    if new_url in url_list:
+        print("not goto")
+    else:
+        print("new page")
+        url_list.append(new_url)
+
+
+    print(url_list) 
+    frames = page.mainFrame
+
+    print(frames)
+    if frames.childFrames == []:
+        for p in input_list:
+            id_list = await get_element_list(page,p)
+            input_id_list.append(id_list)
+        print(input_id_list)       
+
+        for p in click_list:
+            id_list = await get_element_list(page,p)
+            click_id_list.append(id_list)
+        print(click_id_list)
+
+    else:
+
+        for frame in frames.childFrames:
+            print(frame)
+            input_id_list = []
+            click_id_list = []
+            for p in input_list:
+                id_list = await get_element_list(frame,p)
+                input_id_list.append(id_list)
+            print(input_id_list)       
+
+            for p in click_list:
+                id_list = await get_element_list(frame,p)
+                click_id_list.append(id_list)
+            print(click_id_list)
+            print(url_list)
+            try:
+
+                element = await frame.querySelectorAll("#a24")
+                print("???",element)
+                # await frame.click("#a22")#input.buttonBig:nth-child(2)
+                # await page.waitFor(3000)
+                # await frame.click("#a24")
+                # new_url = page.url
+                # print(new_url)   
+
+            except:
+                print("----------------------not click")
+
+    # await page.screenshot({'path': 'example.png'})
+    # for frame in frames.childFrames:
+    #     input_id_list = []
+    #     click_id_list = []
+    #     for p in input_list:
+    #         id_list = await get_element_list(frame,p)
+    #         input_id_list.append(id_list)
+    #     print(input_id_list)       
+
+    #     for p in click_list:
+    #         id_list = await get_element_list(frame,p)
+    #         click_id_list.append(id_list)
+    #     print(click_id_list)
+    #finalRequest = await page.waitForRequest(lambda req: req.url == url_list[3] and req.method == 'GET')  
+    #print(finalRequest)  
+    #await page.waitForRequest(url_list[3])
+    await page.waitFor(3000)
+    await page.screenshot({'path': 'example.png'})
+
+
+     
+    await browser.close()
+    #print(cookies)
+
+asyncio.get_event_loop().run_until_complete(main())
\ No newline at end of file