Android parse nested tables with Jsoup

I am trying to parse an HTML page online to retrieve data from a table with Jsoup. The page I want to parse contains more than one table.

How can I do that?

Here is a sample page that I want to parse:

https://www.cpu-world.com/info/AMD/AMD_A4-Series.html

The data I want to extract is the Model Name and the URL of the details page.

Edit:

This is some of the code I'm using to extract data from the details page.

            try {
                /**
                 * Works to iterate through the items at the following website
                 * https://www.cpu-world.com/CPUs/K10/AMD-A4-Series%20A4-3300.html
                 */
                URL url = new URL("https://www.cpu-world.com/CPUs/K10/AMD-A4-Series%20A4-3300.html");
                
                Document doc = Jsoup.parse(url, 3000);
                
                // spec_table is the name of the class associated with the table
                Elements table = doc.select("table.spec_table");
                Elements rows = table.select("tr");
                
                Iterator<Element> rowIterator = rows.iterator();
                rowIterator.next();
                boolean wasMatch = false;
                
                // Loop through all items in list
                while (rowIterator.hasNext()) {
                    Element row = rowIterator.next();
                    Elements cols = row.select("td");
                    String rowName = cols.get(0).text();
                }
            } catch (MalformedURLException e) {
                e.printStackTrace();
            } catch (IOException e) {
                e.printStackTrace();
            }

I've been reading some tutorials as well as the documentation and I can't seem to figure out how to navigate the web pages to extract the data I'm looking for. I understand the HTML and CSS, but am just learning about Jsoup.

(I tagged this as Android because that's where I'm using the Java code. Guess it's not necessary to be that specific.)

This looks like what you're after:

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.IOException;
import java.net.URL;

public class CpuWorld {

    public static void main(String[] args) throws IOException {
        URL url = null;
        try {
            /**
             * Works to iterate through the items at the following website
             * https://www.cpu-world.com/CPUs/K10/AMD-A4-Series%20A4-3300.html
             */
             url = new URL("https://www.cpu-world.com/CPUs/K10/AMD-A4-Series%20A4-3300.html");
        } catch (IOException e) {
            e.printStackTrace();
        }

        Document doc = Jsoup.parse(url, 3000);
        // spec_table is the name of the class associated with the table
        String modelNumber = doc.select("table tr:has(td:contains(Model number)) td b a").text();
        String modelUrl = doc.select("table tr:has(td:contains(Model number)) td b a").attr("href");

        System.out.println(modelNumber + " : " + modelUrl);
    }
}

Let me know if this is not what you're after

EDIT: Results:

A4-3300 : https://www.cpu-world.com/CPUs/K10/AMD-A4-Series%20A4-3300.html

Process finished with exit code 0

EDIT:

This is crazier than a box of frogs but here we go... I'll leave you to put 2 and 2 together to iterate through the URLs to get the individual details you're after:

import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.client.RestTemplate;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

public class CpuWorld {

    public static final String CPU_WORLD_COM_URL = "https://www.cpu-world.com/info/AMD/AMD_A4-Series.html";

    public static final String SCRAMBLED_DATA_HEADER = "<!--\r\nfunction JSC_Process () {var bk,xg,qh,k,aj,y,e,cq,u,a,ei;\r\na=\"";
    public static final String SCRAMBLED_DATA_FOOTER = "//- qh=[\"\"];k=[2];cq=[7600];if (CW_AB){if\t((AB_v!='0')&&(AB_v!='X')&&(AB_Gl((AB_v=='')?99:3)==3)){y=1;AB_cb=function(){JSC_Process();};}else{y=2;}}for(aj=e=0;aj<k.length;aj++){ei=cq[aj];bk=qh[aj];if (!bk) bk=\"JSc_\"+aj;u=CW_E(bk);if (u){bk=\" jsc_a\";if (y>=k[aj]){xg=a.substr(e,ei);xg=xg.replace(/(.)(.)/g,\"$2$1\");u.innerHTML=xg.replace(/\\\\n/g,\"\\n\");bk='';}u.className=u.className.replace(/(^| )jsc_\\w+$/,bk);}e+=ei;}}JSC_Process();";

    private static RestTemplate restTemplate = new RestTemplate();

    public static void main(String[] args) throws IOException {
        Document tableData = getTableData(CPU_WORLD_COM_URL);

        List<String> fullUrls = tableData.select("table tr td:contains(a) a").stream()
                .map(e -> "https://www.cpu-world.com/" + e.attr("href"))
                .collect(Collectors.toList());

        List<String> fullModels = tableData.select("table tr td:contains(a) a").stream()
                .map(e -> e.text())
                .collect(Collectors.toList());

        for (int i=0; i< fullUrls.size(); i++) {
            System.out.println(fullModels.get(i) + " : " + fullUrls.get(i));
        }
    }

    private static Document getTableData(String url) {
        Connection.Response response = null;
        try {
            response = Jsoup
                    .connect(url)
                    .headers(getHeaders())
                    .method(Connection.Method.GET)
                    .data()
                    .execute();

        } catch (IOException e) {
            e.printStackTrace();
            System.exit(1);
        }
        Elements script = Jsoup.parse(response.body()).select("script");

        // take substring of the child node from after the header and before the footer (- 6 more chars which seem dynamic)
        // The script tag containing JSC_Process is the one with the data in (but all mangled).
        Optional<String> scrambledData = script.stream()
                .filter(element -> element.data().contains("JSC_Process"))
                .map(node -> node.data().substring(SCRAMBLED_DATA_HEADER.length(), (node.data().length() - SCRAMBLED_DATA_FOOTER.length()-6)))
                .findFirst();

        String tableData = Unscrambler.unscramble(scrambledData.orElseThrow(() -> new RuntimeException("scrambled data not found in relevant script tag")));

        Document doc = Jsoup.parse(tableData);
        return doc;
    }

    private static boolean isNotEmptyString(Element node) {
        return node.data() != null && !node.data().equals("");
    }

    /**
    * trick server into thinking we're not a bot
    * by telling the server we were referred by the server itself
    * and give tell it we're using a Mozilla/Safari browser
    **/
    private static Map<String, String> getHeaders() {
        Map<String, String> headersMap = new HashMap<>();
        headersMap.put("User-Agent", "Mozilla/5.0 Safari/537.36");
        headersMap.put("Referer", CPU_WORLD_COM_URL);
        return headersMap;
    }
}

class Unscrambler {

    public static final String SCRAMBLED_DATA_HEADER = "<!--\r\nfunction JSC_Process () {var bk,xg,qh,k,aj,y,e,cq,u,a,ei;\r\na=\"";
    public static final String SCRAMBLED_DATA_FOOTER = "qh=[\"\"];k=[2];cq=[7600];if (CW_AB){if\t((AB_v!='0')&&(AB_v!='X')&&(AB_Gl((AB_v=='')?99:3)==3)){y=1;AB_cb=function(){JSC_Process();};}else{y=2;}}for(aj=e=0;aj<k.length;aj++){ei=cq[aj];bk=qh[aj];if (!bk) bk=\"JSc_\"+aj;u=CW_E(bk);if (u){bk=\" jsc_a\";if (y>=k[aj]){xg=a.substr(e,ei);xg=xg.replace(/(.)(.)/g,\"$2$1\");u.innerHTML=xg.replace(/\\\\n/g,\"\\n\");bk='';}u.className=u.className.replace(/(^| )jsc_\\w+$/,bk);}e+=ei;}}JSC_Process();";

    public static String unscramble(String data) {
        String a=data.replace("\\\"","'")
                .replace("\\\\", "\\")
                .replace("\\r", "")
                .replace("\\n", "")
                .replace("\"+\r\n\"", ""); // remove gunk that mucks up processing in java
        StringBuffer buffer = new StringBuffer();
        int e = 0;
        int ei = 2;

        // This is effectively what the code in the footer is doing. Heavily un-obfuscated below.
        // swap two chars around - through
        for (int aj=0; aj < a.length()-2; aj+=2) {
            String xg = a.substring(e, ei);
            buffer.append(xg.substring(1,2) + xg.substring(0,1));
            e+=2;
            ei+=2;
        }
        return buffer.toString().replace("\n","");
    }
}