Wednesday, January 30, 2019

JAVA - Check if coordinates fall in the right county (WGS84)

This code is using concept that point in polygon. to break down US map into each county and to see whether the given coordinate actually falls in the right county.

This is the pom.xml dependency you need to add:

   <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.geotools</groupId>
            <artifactId>gt-referencing</artifactId>
            <version>20.0</version>
        </dependency>
        <dependency>
            <groupId>org.geotools</groupId>
            <artifactId>gt-epsg-hsql</artifactId>
            <version>20.0</version>
        </dependency>
        <dependency>
            <groupId>org.assertj</groupId>
            <artifactId>assertj-core</artifactId>
            <version>3.11.1</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.locationtech.jts</groupId>
            <artifactId>jts-core</artifactId>
            <version>1.16.0</version>
        </dependency>
        <dependency>
            <groupId>org.geotools</groupId>
            <artifactId>gt-main</artifactId>
            <version>20.1</version>
        </dependency>
        <dependency>
            <groupId>org.geotools</groupId>
            <artifactId>gt-shapefile</artifactId>
            <version>20.1</version>
        </dependency>


This is the interface:

public interface CoordinateCountyMatchFlagWGS84LookupProvider {
    String findPolygon(String key);
}

This is the class:

package com.xxxxx.xxxx.xxxx;

import java.io.File;
import java.util.HashMap;

import org.geotools.data.*;
import org.geotools.data.simple.SimpleFeatureSource;
import org.geotools.feature.FeatureCollection;
import org.geotools.feature.FeatureIterator;

import org.locationtech.jts.geom.Point;
import org.locationtech.jts.geom.Coordinate;
import org.locationtech.jts.io.WKTReader;
import org.locationtech.jts.geom.Geometry;
import org.locationtech.jts.geom.GeometryFactory;
import org.locationtech.jts.geom.MultiPolygon;

import org.opengis.feature.simple.SimpleFeature;
import org.opengis.feature.simple.SimpleFeatureType;


public class CoordinateCountyMatchFlagWGS84 {
    private static CoordinateCountyMatchFlagWGS84LookupProvider provider;

    public CoordinateCountyMatchFlagWGS84(CoordinateCountyMatchFlagWGS84LookupProvider provider) {
        CoordinateCountyMatchFlagWGS84.provider = provider;
    }

    private String checkPolygon(String rawKey) {
        return provider.findPolygon(rawKey);
    }

    public Boolean getMatchFlag(String county, String state, Double lat, Double lng) throws Exception {
        String rawKey = county + '-' + state;
        WKTReader rd = new WKTReader();
        //Point
        GeometryFactory fac = new GeometryFactory();
        Point convertPoint = fac.createPoint(new Coordinate(lng, lat));
        Geometry point = rd.read(convertPoint.toString());
        //Shape
        if (checkPolygon(rawKey) != null) {
        /*The units of measurement is based on the underlying spatial reference.
        So, for example, if it is EPSG:4326(WGS84) it is decimal degrees.*/
            Geometry poly = rd.read(checkPolygon(rawKey)).buffer(2); //boundary allowance
            //return true or false
            return poly.contains(point);
        } else return null;
    }

    public static class ShapeProvider implements CoordinateCountyMatchFlagWGS84LookupProvider {
        private WKTReader reader = new WKTReader();
        private HashMap<String, MultiPolygon> County_GEOM = new HashMap<>();

        public ShapeProvider(String countyShapeFile) throws Exception {
            // Get county shape file
            String path = getClass().getResource(countyShapeFile).getPath();
            File file = new File(path);
            FileDataStore myData = FileDataStoreFinder.getDataStore(file);
            SimpleFeatureSource source = myData.getFeatureSource();

            // Get data information query
            SimpleFeatureType schema = source.getSchema();
            Query query = new Query(schema.getTypeName());

            // Get collection of features
            FeatureCollection<SimpleFeatureType, SimpleFeature> collection = source.getFeatures(query);
            FeatureIterator<SimpleFeature> features = collection.features();

            while (features.hasNext()) {
                //Get feature of each polygon
                SimpleFeature feature = features.next();
                String key = feature.getAttribute(1).toString() + "-" + feature.getAttribute(2).toString();
                String WKTString = feature.getAttribute(0).toString();
                MultiPolygon value = (MultiPolygon) reader.read(WKTString);
                County_GEOM.put(key, value);
            }
            features.close();
        }

        public String findPolygon(String key) {
            if (County_GEOM.keySet().contains(key)) {
                //return shape for the county-state key
                return County_GEOM.get(key).toString();
            } else return null;
        }
    }
}


This is the test for the class:

import com.xxxx.xxx.xxx.CoordinateCountyMatchFlagWGS84;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;


public class CoordinateCountyMatchFlagWGS84Test {

    private CoordinateCountyMatchFlagWGS84 lookup;

    private final String CORRECT_STATE = "Texas";
    private final String CORRECT_COUNTY = "Armstrong";
    private final String WRONG_COUNTY = "WrongCounty";
    private final Double CORRECT_LATITUDE = 34.8255435;
    private final Double CORRECT_LONGITUDE = -101.6004429;
    private final Double WRONG_LATITUDE = 39.148469;
    private final Double WRONG_LONGITUDE = -80.102385;

    @Before
    public void setup() throws Exception {

        lookup = new CoordinateCountyMatchFlagWGS84(new CoordinateCountyMatchFlagWGS84.ShapeProvider("/County_WGS84/County_WGS84.shp"));
    }

    @Test
    public void getMatchFlag_ValidCountyStateCoordinate_ReturnTrue() throws Exception {
        Assert.assertEquals(true, lookup.getMatchFlag(CORRECT_COUNTY,CORRECT_STATE,CORRECT_LATITUDE,CORRECT_LONGITUDE));
    }

    @Test
    public void  getMatchFlag_ValidCountyState_InvalidCoordinate_ReturnFalse() throws Exception {
        Assert.assertEquals(false, lookup.getMatchFlag(CORRECT_COUNTY,CORRECT_STATE,WRONG_LATITUDE,WRONG_LONGITUDE));
    }

    @Test
    public void getMatchFlag_InvalidCounty_ReturnFalse() throws Exception {
        Assert.assertNull(lookup.getMatchFlag(WRONG_COUNTY,CORRECT_STATE,CORRECT_LATITUDE,CORRECT_LONGITUDE));
    }

Tuesday, January 8, 2019

Python - Scrape APEX website ( wwv_flow.show)

Reference : https://www.slideshare.net/Enkitec/apex-behind-the-scenes

This kind of website will have several parameters with the same name 'p_arg_names' 
and value 'p_arg_values'. So when doing the post, need to put them in two separate lists.



import requests
from bs4 import BeautifulSoup
from datetime import datetime
from datetime import timedelta

logurl = 'http://xxxxxxxxx/pls/apex/f?p=108:2700'
posturl = 'http://xxxxxxxx/pls/apex/wwv_flow.show'
with requests.Session() as s:
    s.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) 
         AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"}

    res = s.get(logurl)
    soup = BeautifulSoup(res.text,"lxml")

    p_date_start = (datetime.now() - timedelta(days=7)).date().strftime("%d-%b-%Y"),
    p_date_end = datetime.now().strftime("%d-%b-%Y")
    permit_date_begin = str(p_date_start[0])
    permit_date_end = str(p_date_end)

    p_instance = soup.find(id='pInstance').get('value')
    p_flow_id = soup.find(id='pFlowId').get('value')
    p_flow_step_id = soup.find(id='pFlowStepId').get('value')
    p_query_criteria = 'Permitted on or after: ' + permit_date_begin + '; 
                       Permitted before or on: ' + permit_date_end

    p_arg_names = ['P2700_PERMIT_DATE_BEGIN', 'P2700_PERMIT_DATE_END', 
                 'P2700_QUERY_CRITERIA','P2700_QUERY_STOP','P2700_QUERY_DISPLAY']
    p_arg_values = [permit_date_begin, permit_date_end, p_query_criteria, 'N', '']

    values = {
        'p_flow_id': p_flow_id,
        'p_flow_step_id': p_flow_step_id,
        'p_instance': p_instance,
        'p_debug': '',
         ......
        "p_arg_names": p_arg_names,
        "p_arg_values": p_arg_values
    }

    posturl_final = 'http://xxxxxxxxx/pls/apex/f?p=108:2700:' + p_instance + ':CSV::::'    
    r = s.post(posturl, data=values)
    r2 = s.get(posturl_final)
    soup3 = BeautifulSoup(r2.text,"lxml")

    print(soup3)