Tuesday, January 8, 2019

Python - Scrape APEX website ( wwv_flow.show)

Reference : https://www.slideshare.net/Enkitec/apex-behind-the-scenes

This kind of website will have several parameters with the same name 'p_arg_names' 
and value 'p_arg_values'. So when doing the post, need to put them in two separate lists.



import requests
from bs4 import BeautifulSoup
from datetime import datetime
from datetime import timedelta

logurl = 'http://xxxxxxxxx/pls/apex/f?p=108:2700'
posturl = 'http://xxxxxxxx/pls/apex/wwv_flow.show'
with requests.Session() as s:
    s.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) 
         AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"}

    res = s.get(logurl)
    soup = BeautifulSoup(res.text,"lxml")

    p_date_start = (datetime.now() - timedelta(days=7)).date().strftime("%d-%b-%Y"),
    p_date_end = datetime.now().strftime("%d-%b-%Y")
    permit_date_begin = str(p_date_start[0])
    permit_date_end = str(p_date_end)

    p_instance = soup.find(id='pInstance').get('value')
    p_flow_id = soup.find(id='pFlowId').get('value')
    p_flow_step_id = soup.find(id='pFlowStepId').get('value')
    p_query_criteria = 'Permitted on or after: ' + permit_date_begin + '; 
                       Permitted before or on: ' + permit_date_end

    p_arg_names = ['P2700_PERMIT_DATE_BEGIN', 'P2700_PERMIT_DATE_END', 
                 'P2700_QUERY_CRITERIA','P2700_QUERY_STOP','P2700_QUERY_DISPLAY']
    p_arg_values = [permit_date_begin, permit_date_end, p_query_criteria, 'N', '']

    values = {
        'p_flow_id': p_flow_id,
        'p_flow_step_id': p_flow_step_id,
        'p_instance': p_instance,
        'p_debug': '',
         ......
        "p_arg_names": p_arg_names,
        "p_arg_values": p_arg_values
    }

    posturl_final = 'http://xxxxxxxxx/pls/apex/f?p=108:2700:' + p_instance + ':CSV::::'    
    r = s.post(posturl, data=values)
    r2 = s.get(posturl_final)
    soup3 = BeautifulSoup(r2.text,"lxml")

    print(soup3)

No comments:

Post a Comment