HTML - Use Httpurlconnection to log into website

Question

I’m trying to log into a site using Httpurlconnection, as shown in the code below, taken from mkyong’s website:

private final static String USER_AGENT = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401";

private void sendPost() throws Exception {

    String url = "http://ts5.travian.com.br/dorf1.php";
    URL obj = new URL(url);
    HttpURLConnection con = (HttpURLConnection) obj.openConnection();

    String charset = "UTF-8";
    String param1 = "teste2";
    String param2 = "987654321";
    String param3 = "Entrar";
    String param4 = "1920:1080";
    String param5 = "1414027525";

    String urlParameters = String.format("name=%s&password=%s&s1=%s&w=%s&login=%s", URLEncoder.encode(param1, charset), URLEncoder.encode(param2, charset),
            URLEncoder.encode(param3, charset), URLEncoder.encode(param4, charset), URLEncoder.encode(param5, charset));

    con.setRequestMethod("POST");
    con.setRequestProperty("Content-Length", "" + Integer.toString(urlParameters.getBytes().length));
    con.setRequestProperty("User-Agent", USER_AGENT);
    con.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
    con.setRequestProperty("Host", "tx3.travian.com.br");
    con.setRequestProperty("Accept", "*/*");

    // Send post request
    con.setDoOutput(true);
    con.connect();
    DataOutputStream wr = new DataOutputStream(con.getOutputStream());
    wr.writeBytes(urlParameters);
    wr.flush();
    wr.close();

    int responseCode = con.getResponseCode();
    System.out.println("\nSending 'POST' request to URL : " + url);
    System.out.println("Post parameters : " + urlParameters);
    System.out.println("Response Code : " + responseCode);

    BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
    String inputLine;
    StringBuffer response = new StringBuffer();

    while ((inputLine = in.readLine()) != null) {
        response.append(inputLine + "\n");
    }
    in.close();

    // print result
    System.out.println(response.toString());

}

When executing the code, returns me again the login screen, I used wireshark to see if the request was made correctly and is correct, the request header is as it should be.

Using a website like Request Make I made the same request, putting the same parameters and properties of the connection and worked, someone knows where the error is?

Answer 1

Most of the time the simple form you used does not work, because:

• A navigation flow from one page to another may be required to create session variables, which are checked by the website.
• Currently websites use a lot of asynchronous data generation and you want to see the resulting source code after these asynchronous calls.
• Sometimes websites use HTTPS.

There is a very easy way to do what you want. Just use the Apache Httpasyncclient library. You need to use the version Httpasyncclient 4.1-beta1.

The following is the copy and paste code: (there are 3 classes)

1 - Testetravian.java:

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.ExecutionException;

public class TesteTravian {

    public static void loginTravian(){
        MyNavigator navigator = null;

        try{
            navigator = new MyNavigator();

            String resp = navigator.navigateTo("http://ts5.travian.com.br");

            String charset = "UTF-8";
            String param_name = "vinmor";
            String param_password = "321321";

            String strLogin = "<input type=\"hidden\" name=\"login\" value=\""; 
            int index1 = resp.indexOf(strLogin)+strLogin.length();
            int index2 = resp.indexOf("\"",index1);
            String param_login = resp.substring(index1,index2); 

            String urlParameters = String.format("name=%s&password=%s&login=%s", 
                    URLEncoder.encode(param_name, charset), 
                    URLEncoder.encode(param_password, charset),
                    URLEncoder.encode(param_login, charset));

            //Ao enviar o post, ele retorna uma mensagem de erro, porém o login foi efetuado com sucesso.
            //A partir daí os novos GETS irão funcionar
            navigator.navigateToPost("http://ts5.travian.com.br/dorf1.php", urlParameters);

            //Vai para a página: Recursos
            resp = navigator.navigateTo("http://ts5.travian.com.br/dorf1.php");                 
            System.out.println(resp);           

            //Como fazer para ir para outras páginas...         
            //Vai para a página: Centro da Aldeia
            //resp = navigator.navigateTo("http://ts5.travian.com.br/dorf2.php");
        }
        catch(InterruptedException | ExecutionException | KeyManagementException | NoSuchAlgorithmException | KeyStoreException | UnsupportedEncodingException e){
            e.printStackTrace();
        }
        finally{
            if(navigator!=null){ try{navigator.close();}catch(IOException e){e.printStackTrace();}}
        }
    }

    public static void main(final String[] args){
        loginTravian();
        System.exit(0);
    }
}

2 - Mynavigator.java

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.nio.client.methods.HttpAsyncMethods;
import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;

public class MyNavigator{
    private CloseableHttpAsyncClient httpclient;

    public MyNavigator() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException{
        SSLContext sslcontext = SSLContexts.custom().useTLS().loadTrustMaterial(null, new TrustStrategy(){
            @Override
            public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException{
                return true;
            }
        }).build();

        SSLIOSessionStrategy sslSessionStrategy = new SSLIOSessionStrategy(sslcontext, new AllowAll());
        this.httpclient = HttpAsyncClients.custom()
                .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
                .setSSLStrategy(sslSessionStrategy)
                .build();

        httpclient.start();
    }

    public String navigateTo(String endereco) throws InterruptedException, ExecutionException{
        MyResponseConsumer myResponseConsumer = new MyResponseConsumer();

        Future<Boolean> future = httpclient.execute(HttpAsyncMethods.createGet(endereco), myResponseConsumer, null);
        Boolean result;

        result = future.get();
        if(result != null && result.booleanValue()){
            return myResponseConsumer.getResp();
        }
        return null;
    }

    public String navigateToPost(String endereco, String postsValues) throws InterruptedException, ExecutionException,
    UnsupportedEncodingException{
        MyResponseConsumer myResponseConsumer = new MyResponseConsumer();

        Future<Boolean> future = httpclient.execute(HttpAsyncMethods.createPost(endereco, postsValues, ContentType.APPLICATION_FORM_URLENCODED), myResponseConsumer, null);
        Boolean result;

        result = future.get();
        if(result != null && result.booleanValue()){
            return myResponseConsumer.getResp();
        }
        return null;
    }

    public void close() throws IOException{
        if(httpclient != null)
            httpclient.close();
    }

    private static class AllowAll implements X509HostnameVerifier{
        @Override
        public void verify(String s, SSLSocket sslSocket) throws IOException{/* Nothing */
        }

        @Override
        public void verify(String s, X509Certificate x509Certificate) throws SSLException{/* Nothing */
        }

        @Override
        public void verify(String s, String[] strings, String[] strings2) throws SSLException{/* Nothing */
        }

        @Override
        public boolean verify(String s, SSLSession sslSession){
            return true;
        }
    }
}

3 - Myresponseconsumer.java

import java.io.IOException;
import java.nio.CharBuffer;
import org.apache.http.HttpResponse;
import org.apache.http.nio.IOControl;
import org.apache.http.nio.client.methods.AsyncCharConsumer;
import org.apache.http.protocol.HttpContext;

public class MyResponseConsumer extends AsyncCharConsumer<Boolean>{

    private String resp;    

    public MyResponseConsumer(){
        this.resp = "";
    }

    @Override
    protected void onResponseReceived(final HttpResponse response) {
        //nothing
    }

    @Override
    protected void onCharReceived(final CharBuffer buf, final IOControl ioctrl) throws IOException {
        while (buf.hasRemaining()) {
            resp += buf.get();
        }
    }

    @Override
    protected void releaseResources() {
        //nothing
    }

    @Override
    protected Boolean buildResult(final HttpContext context) {
        return Boolean.TRUE;
    }

    public String getResp(){
        return this.resp;
    }    
}