What is the best way to scrape data from from multiple websites?

What I have tried is that I used volley request to scrape from first website and inside it I created multiple threads for each website and inside each thread I used jsoup connect method to scrape rather than volley. It gets the job done, actually faster. But, what the problem is that it freezes the app when scraping the data until it's fully loaded. It freezes the progress bar and I am having problem to find the cause.

Here's the code I have implemented. It's a bit lengthy.

//        Checking the connection
final StringRequest request = new StringRequest("https://www.google.com/", new Response.Listener<String>() {
@Override
public void onResponse(String response) {
relativeLayout.setVisibility(View.GONE);
//                instances for each required website
final HimalayanTimes himalayanTimes = new HimalayanTimes(getContext());
final GsmArena gsmArena = new GsmArena();
final CinemaBlend cinemaBlend = new CinemaBlend();
final KathmanduPost kathmanduPost = new KathmanduPost(getContext());
final GlobalNews globalNews =  new GlobalNews();
final NepaliTimes nepaliTimes = new NepaliTimes(getContext());
final GoalNepal goalNepal = new GoalNepal(getContext());
final GadgetByte gadgetByte = new GadgetByte();
final TechLekh techLekh = new TechLekh();
final OnlineKhabar onlineKhabar = new OnlineKhabar();
final NepaliSansar nepaliSansar = new NepaliSansar();
final CricketingNepal cricketingNepal = new CricketingNepal();
//                thread for each website
//                thread fot thehimalayantimes
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try  {
ArrayList<NewsItem> himalyannews;
himalyannews = himalayanTimes.getNews();
news.addAll(himalyannews);
for(int i=0; i<4; i++){
finalHeadlines.add(himalyannews.get(i));
}
} catch (Exception ignored) {
}
}
});
thread.start();
//                thread for gsmArena
Thread thread1 = new Thread(new Runnable() {
@Override
public void run() {
try  {
ArrayList<NewsItem> gsmarenanews;
gsmarenanews = gsmArena.getNews();
news.addAll(gsmarenanews);
for(int i=0; i<3; i++){
headlines.add(gsmarenanews.get(i));
}
} catch (Exception ignored) {
}
}
});
thread1.start();
//                thread for cinemaBlend
Thread thread2 = new Thread(new Runnable() {
@Override
public void run() {
try  {
ArrayList<NewsItem> cinemablendnews;
cinemablendnews = cinemaBlend.getNews();
news.addAll(cinemablendnews);
for(int i=0; i<4; i++){
headlines.add(cinemablendnews.get(i));
}
} catch (Exception ignored) {
}
}
});
thread2.start();
//                thread for kathmanduPost
Thread thread3 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> kathmandupostnews;
kathmandupostnews = kathmanduPost.getNews();
news.addAll(kathmandupostnews);
for(int i=0; i<3; i++){
finalHeadlines.add(kathmandupostnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread3.start();
//                thread for globalNews
Thread thread4 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> globalnewsnews;
globalnewsnews = globalNews.getNews();
news.addAll(globalnewsnews);
for(int i=0; i<5; i++){
finalHeadlines.add(globalnewsnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread4.start();
//                thread for nepaliTimes
Thread thread5 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> nepalitimesnews;
nepalitimesnews = nepaliTimes.getNews();
news.addAll(nepalitimesnews);
for(int i=0; i<3; i++){
finalHeadlines.add(nepalitimesnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread5.start();
//                thread for GoalNepal
Thread thread6 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> goalNepalNews;
goalNepalNews = goalNepal.getNews();
news.addAll(goalNepalNews);
for (int i=0; i<4; i++){
headlines.add(goalNepalNews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread6.start();
//                thread for GadgetByteNepal
Thread thread7 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> gadgetbytenews;
gadgetbytenews = gadgetByte.getNews();
news.addAll(gadgetbytenews);
for (int i=0; i<3; i++){
headlines.add(gadgetbytenews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread7.start();
//                thread for Techlekh
Thread thread8 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> techlekhnews;
techlekhnews = techLekh.getNews();
news.addAll(techlekhnews);
for (int i=0; i<3; i++){
headlines.add(techlekhnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread8.start();
//                thread for onlinekhabar
Thread thread9 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> onlineKhabarnews;
onlineKhabarnews = onlineKhabar.getNews();
news.addAll(onlineKhabarnews);
for (int i=0; i<4; i++){
finalHeadlines.add(onlineKhabarnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread9.start();
//thread for nepalisansar
Thread thread11 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> nepalisansarnews;
nepalisansarnews = nepaliSansar.getNews();
news.addAll(nepalisansarnews);
for (int i=0; i<4; i++){
finalHeadlines.add(nepalisansarnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread11.start();
//                thread for cricketingNepal
Thread thread12 = new Thread(new Runnable() {
@Override
public void run() {
try {
ArrayList<NewsItem> cricketnews;
cricketnews = cricketingNepal.getNews();
news.addAll(cricketnews);
for (int i=0; i<4; i++){
headlines.add(cricketnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread12.start();
//                main thread wait for each thread to finish
try {
thread.join();
} catch (InterruptedException ignored) {
}
try {
thread1.join();
} catch (InterruptedException ignored) {
}
try {
thread2.join();
} catch (InterruptedException ignored) {
}
try {
thread3.join();
} catch (InterruptedException ignored) {
}
try {
thread4.join();
} catch (InterruptedException ignored) {
}
try {
thread5.join();
} catch (InterruptedException ignored) {
}
try {
thread6.join();
} catch (InterruptedException ignored) {
}
try {
thread7.join();
} catch (InterruptedException ignored) {
}
try {
thread8.join();
} catch (InterruptedException ignored) {
}
try {
thread8.join();
} catch (InterruptedException ignored) {
}
try {
thread9.join();
} catch (InterruptedException ignored) {
}
try {
thread11.join();
} catch (InterruptedException ignored) {
}
try {
thread12.join();
} catch (InterruptedException ignored) {
}
for(NewsItem item : news){
if (item.tag.contains("kathmandu"))
nepal.add(item);
if (item.tag.contains("cricket"))
sports.add(item);
if (item.tag.contains("football"))
sports.add(item);
switch (item.tag) {
case "nepal":
nepal.add(item);
break;
case "world":
world.add(item);
break;
case "sports":
sports.add(item);
break;
case "tech":
tech.add(item);
break;
case "entertainment":
entertainment.add(item);
break;
}
}
//                putting each news item to the main container
Collections.shuffle(headlines);
Collections.shuffle(finalHeadlines);
finalHeadlines.addAll(headlines);
Collections.shuffle(nepal);
Collections.shuffle(world);
Collections.shuffle(sports);
Collections.shuffle(tech);
Collections.shuffle(entertainment);
tab1 t1 = new tab1(finalHeadlines);
t1.setRetainInstance(true);
tab2 t2 = new tab2(nepal);
t2.setRetainInstance(true);
tab3 t3 = new tab3(world);
t3.setRetainInstance(true);
tab4 t4 = new tab4(sports);
t4.setRetainInstance(true);
tab5 t5 = new tab5(tech);
t5.setRetainInstance(true);
tab6 t6 = new tab6(entertainment);
t6.setRetainInstance(true);
assert getFragmentManager() != null;
pagerAdapter = new PageAdapter(finalHeadlines, nepal, world, sports, tech, entertainment, getFragmentManager(), tabLayout.getTabCount());
viewPager.setAdapter(pagerAdapter);
shimmerFrameLayout.setVisibility(View.GONE);
}
}, new Response.ErrorListener() {
@Override
public void onErrorResponse(VolleyError error) {
Toast.makeText(getContext(), "Internet Connection Error!", Toast.LENGTH_SHORT).show();
shimmerFrameLayout.setVisibility(View.GONE);
tabLayout.setVisibility(View.GONE);
}
});
queue.add(request);

For each website, I made class. One of the following class:-

public class CinemaBlend {
ArrayList<NewsItem> news;
public CinemaBlend() {
news = new ArrayList<>();
}
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
public ArrayList<NewsItem> getNews() throws IOException{
String url = "https://www.cinemablend.com/news.php";
OkHttpClient okHttpClient = new OkHttpClient();
Request request = new Request.Builder().url(url).get().build();
Document document = Jsoup.parse(Objects.requireNonNull(okHttpClient.newCall(request).execute().body()).string());
Elements articles = document.select("div.order-of-type-2").select("div.story-related").select("a");
for(Element article : articles)
{
String link = article.attr("href");
String title = article.attr("title");
String img = article.select("div.story-related-content").select("span.story-cover-image").select("img").attr("data-src");
String date = article.select("span.story-related-published-date").text();
NewsItem newsItem = new NewsItem();
newsItem.imgsrc = img;
newsItem.title = title;
newsItem.link = link;
newsItem.tag = "entertainment";
newsItem.publisher = "cinemablend.com";
newsItem.source_logo = "https://image.pitchbook.com/WFQVGYL17V0MevlcfQKlWjC3E8K1447542818374_200x200";
if(!date.equals(""))
{
newsItem.date = date + " ago";
news.add(newsItem);
}
}
return news;
}
}

Look for tutorials explaining how to perform background work. There are lots of different ways to do that: Service, Kotlin Coroutines, simple self-managed Threads, etc.

Just stay away from tutorials about AsyncTasks and Loaders (deprecated).

A good starting point is the Android developer guide: https://developer.android.com/guide/background

But doing the tasks in Async task create another similar problem. When I create similar threads in background, exactly the similar problem arises where it skips the data and the ui doesn't get updated. Any suggestion would be greatly appreciated.

public class DownloadNews extends AsyncTask<Void, Void, Void>
{
@Override
protected void onPreExecute() {
shimmerFrameLayout.setVisibility(View.VISIBLE);
relativeLayout.setVisibility(View.GONE);
tabLayout.setVisibility(View.GONE);
}
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
@Override
protected Void doInBackground(Void... voids) {
tabLayout.setOnTabSelectedListener(new TabLayout.OnTabSelectedListener() {
@Override
public void onTabSelected(TabLayout.Tab tab) {
viewPager.setCurrentItem(tab.getPosition());
}
@Override
public void onTabUnselected(TabLayout.Tab tab) {
}
@Override
public void onTabReselected(TabLayout.Tab tab) {
}
});
viewPager.addOnPageChangeListener(new TabLayout.TabLayoutOnPageChangeListener(tabLayout));
final RequestQueue queue = Volley.newRequestQueue(Objects.requireNonNull(getContext()));
//        Checking the connection
final StringRequest request = new StringRequest("https://www.google.com/", new Response.Listener<String>() {
@Override
public void onResponse(String response) {
//                instances for each required website
final HimalayanTimes himalayanTimes = new HimalayanTimes(getContext());
final GsmArena gsmArena = new GsmArena();
final CinemaBlend cinemaBlend = new CinemaBlend();
final KathmanduPost kathmanduPost = new KathmanduPost(getContext());
final GlobalNews globalNews =  new GlobalNews();
final NepaliTimes nepaliTimes = new NepaliTimes(getContext());
final GoalNepal goalNepal = new GoalNepal(getContext());
final GadgetByte gadgetByte = new GadgetByte();
final TechLekh techLekh = new TechLekh();
final OnlineKhabar onlineKhabar = new OnlineKhabar();
final NepaliSansar nepaliSansar = new NepaliSansar();
final CricketingNepal cricketingNepal = new CricketingNepal();
//                thread for each website
//                thread fot thehimalayantimes
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try  {
himalyannews = himalayanTimes.getNews();
news.addAll(himalyannews);
for(int i=0; i<4; i++){
finalHeadlines.add(himalyannews.get(i));
}
} catch (Exception ignored) {
}
}
});
thread.start();
//                thread for gsmArena
Thread thread1 = new Thread(new Runnable() {
@Override
public void run() {
try  {
gsmarenanews = gsmArena.getNews();
news.addAll(gsmarenanews);
for(int i=0; i<3; i++){
headlines.add(gsmarenanews.get(i));
}
} catch (Exception ignored) {
}
}
});
thread1.start();
//                thread for cinemaBlend
Thread thread2 = new Thread(new Runnable() {
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
@Override
public void run() {
try  {
cinemablendnews = cinemaBlend.getNews();
news.addAll(cinemablendnews);
for(int i=0; i<4; i++){
headlines.add(cinemablendnews.get(i));
}
} catch (Exception ignored) {
}
}
});
thread2.start();
//                thread for kathmanduPost
Thread thread3 = new Thread(new Runnable() {
@RequiresApi(api = Build.VERSION_CODES.KITKAT)
@Override
public void run() {
try {
kathmandupostnews = kathmanduPost.getNews();
news.addAll(kathmandupostnews);
for(int i=0; i<3; i++){
finalHeadlines.add(kathmandupostnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread3.start();
//                thread for globalNews
Thread thread4 = new Thread(new Runnable() {
@Override
public void run() {
try {
globalnewsnews = globalNews.getNews();
news.addAll(globalnewsnews);
for(int i=0; i<5; i++){
finalHeadlines.add(globalnewsnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread4.start();
//                thread for nepaliTimes
Thread thread5 = new Thread(new Runnable() {
@Override
public void run() {
try {
nepalitimesnews = nepaliTimes.getNews();
news.addAll(nepalitimesnews);
for(int i=0; i<3; i++){
finalHeadlines.add(nepalitimesnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread5.start();
//                thread for GoalNepal
Thread thread6 = new Thread(new Runnable() {
@Override
public void run() {
try {
goalNepalNews = goalNepal.getNews();
news.addAll(goalNepalNews);
for (int i=0; i<4; i++){
headlines.add(goalNepalNews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread6.start();
//                thread for GadgetByteNepal
Thread thread7 = new Thread(new Runnable() {
@Override
public void run() {
try {
gadgetbytenews = gadgetByte.getNews();
news.addAll(gadgetbytenews);
for (int i=0; i<3; i++){
headlines.add(gadgetbytenews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread7.start();
//                thread for Techlekh
Thread thread8 = new Thread(new Runnable() {
@Override
public void run() {
try {
techlekhnews = techLekh.getNews();
news.addAll(techlekhnews);
for (int i=0; i<3; i++){
headlines.add(techlekhnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread8.start();
//                thread for onlinekhabar
Thread thread9 = new Thread(new Runnable() {
@Override
public void run() {
try {
onlineKhabarnews = onlineKhabar.getNews();
news.addAll(onlineKhabarnews);
for (int i=0; i<4; i++){
finalHeadlines.add(onlineKhabarnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread9.start();
//thread for nepalisansar
Thread thread11 = new Thread(new Runnable() {
@Override
public void run() {
try {
nepalisansarnews = nepaliSansar.getNews();
news.addAll(nepalisansarnews);
for (int i=0; i<4; i++){
finalHeadlines.add(nepalisansarnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread11.start();
//                thread for cricketingNepal
Thread thread12 = new Thread(new Runnable() {
@Override
public void run() {
try {
cricketnews = cricketingNepal.getNews();
news.addAll(cricketnews);
for (int i=0; i<4; i++){
headlines.add(cricketnews.get(i));
}
} catch (IOException ignored) {
}
}
});
thread12.start();
}
}, new Response.ErrorListener() {
@Override
public void onErrorResponse(VolleyError error) {
Toast.makeText(getContext(), "Internet Connection Error!", Toast.LENGTH_SHORT).show();
shimmerFrameLayout.setVisibility(View.GONE);
relativeLayout.setVisibility(View.VISIBLE);
tabLayout.setVisibility(View.GONE);
}
});
queue.add(request);
return null;
}
@Override
protected void onPostExecute(Void aVoid) {
tabLayout.setVisibility(View.VISIBLE);
shimmerFrameLayout.setVisibility(View.GONE);
for(NewsItem item : news){
if (item.tag.contains("kathmandu"))
nepal.add(item);
switch (item.tag) {
case "nepal":
nepal.add(item);
break;
case "world":
world.add(item);
break;
case "sports":
sports.add(item);
break;
case "tech":
tech.add(item);
break;
case "entertainment":
entertainment.add(item);
break;
}
}
// putting each news item to the main container
Collections.shuffle(headlines);
Collections.shuffle(finalHeadlines);
finalHeadlines.addAll(headlines);
Collections.shuffle(nepal);
Collections.shuffle(world);
Collections.shuffle(sports);
Collections.shuffle(tech);
Collections.shuffle(entertainment);
tab1 t1 = new tab1(finalHeadlines);
t1.setRetainInstance(true);
tab2 t2 = new tab2(nepal);
t2.setRetainInstance(true);
tab3 t3 = new tab3(world);
t3.setRetainInstance(true);
tab4 t4 = new tab4(sports);
t4.setRetainInstance(true);
tab5 t5 = new tab5(tech);
t5.setRetainInstance(true);
tab6 t6 = new tab6(entertainment);
t6.setRetainInstance(true);
shimmerFrameLayout.setVisibility(View.GONE);
assert getFragmentManager() != null;
pagerAdapter = new PageAdapter(finalHeadlines, nepal, world, sports, tech, entertainment, getFragmentManager(), tabLayout.getTabCount());
viewPager.setAdapter(pagerAdapter);
}
}

The code for the adapter is exactly similar to above.